Пример #1
0
 def add_hdfs_path(self, hdfs_path):
     """
     Post to http://atlas.hadoop.svenskaspel.se/api/atlas/v2/entity
     Post data {"entity":{"typeName":"hdfs_path","attributes":{"description":null,"name":"hdfs://svsprod/apps/hive/warehouse/hadoop_out_prod.db/country_d","owner":null,"qualifiedName":"hdfs://svsprod/apps/hive/warehouse/hadoop_out_prod.db/country_d","createTime":1536098400000,"fileSize":null,"group":null,"isFile":null,"isSymlink":null,"modifiedTime":1536098400000,"path":"hdfs://svsprod/apps/hive/warehouse/hadoop_out_prod.db/country_d","clusterName":null,"numberOfReplicas":null},"guid":-1},"referredEntities":{}}
     Response: {"mutatedEntities":{"CREATE":[{"typeName":"hdfs_path","attributes":{"qualifiedName":"hdfs://svsprod/apps/hive/warehouse/hadoop_out_prod.db/country_d"},"guid":"e20823a6-5521-4dc9-b2a7-b5a1d9babecd","status":"ACTIVE"}]},"guidAssignments":{"-1":"e20823a6-5521-4dc9-b2a7-b5a1d9babecd"}}
     :param hdfs_path: Full url to the file or directory hdfs://environment/my/path/
     :return: guid assigned
     """
     cluster_name = urlutil.get_host(hdfs_path)
     name = urlutil.get_path(hdfs_path)
     entity = {
         "entity": {
             "typeName": "hdfs_path",
             "attributes": {
                 "description": "Created/Updated by cobra-policytool.",
                 "name": name,
                 "qualifiedName": hdfs_path,
                 "path": hdfs_path,
                 "clusterName": cluster_name,
             },
             "guid": -1
         },
     }
     response = self._post_entity(entity)
     if response.status_code == 200:
         json_response = response.json()
         if json_response.has_key('guidAssignments'):
             return json_response['guidAssignments']['-1']
         else:
             AtlasError(
                 "Failed to add hdfs path {} content mismatch {}".format(
                     hdfs_path, response.content))
     else:
         raise AtlasError(response.content, response.status_code)
Пример #2
0
 def get_temporary_dirs(self):
     # First try the GConf temporary dir, then try the system TEMP dir
     # finally try on the HOME
     temps = (self.temporary_dir, TEMP_DIR, HOME_DIR)
     for temp in temps:
         if urlutil.is_local(temp) and _is_dir_writable(temp):
             yield urlutil.get_path(temp)
Пример #3
0
 def get_source(self, music):
     if self.use_gnomevfs:
         src = gst.element_factory_make("gnomevfssrc")
         src.set_property("location", music)
     
     else:
         src = gst.element_factory_make("filesrc", "source")
         src.set_property("location", urlutil.get_path(music))
         
     return src
Пример #4
0
    def get_source(self, music):
        if self.use_gnomevfs:
            src = gst.element_factory_make("gnomevfssrc")
            src.set_property("location", music)

        else:
            src = gst.element_factory_make("filesrc", "source")
            src.set_property("location", urlutil.get_path(music))

        return src
Пример #5
0
    def is_available(self, music):
        on_cache = GstMusicPool.is_available(self, music)

        unique_id = self.unique_music_id(music)

        if not on_cache and self.is_local(music) and self.is_wav(unique_id):
            # convert to native filename
            filename = urlutil.get_path(unique_id)
            self.cache[unique_id] = GstCacheEntry(filename, False)
            on_cache = True

        return on_cache
Пример #6
0
    def is_available(self, music):
        on_cache = GstMusicPool.is_available(self, music)
        
        unique_id = self.unique_music_id(music)

        if not on_cache and self.is_local(music) and self.is_wav(unique_id):
            # convert to native filename
            filename = urlutil.get_path(unique_id)
            self.cache[unique_id] = GstCacheEntry(filename, False)
            on_cache = True

        return on_cache
Пример #7
0
 def before_music_fetched(self, evt, music):
     print music
     prog_txt = "Converting "
     prog_txt += path.basename(urlutil.get_path(music['location']))
     
     self.prog.sub_progress_text = prog_txt
     self.prog.progress_fraction = self.oper.progress
     
     if self.music:
         print "Fetched", self.music['cache_location']
         
     print "Fetching", music['location']
     self.music = music
Пример #8
0
        def before_music_fetched(self, evt, music):
            print music
            prog_txt = "Converting "
            prog_txt += path.basename(urlutil.get_path(music['location']))

            self.prog.sub_progress_text = prog_txt
            self.prog.progress_fraction = self.oper.progress

            if self.music:
                print "Fetched", self.music['cache_location']

            print "Fetching", music['location']
            self.music = music
Пример #9
0
    def is_available(self, music):
        on_cache = GstMusicPool.is_available(self, music)
        
        # XXX: when there is no gnomevfdssrc we have a problem because
        #      we are using URI's
        unique_id = self.unique_music_id(music)
        
        if not on_cache and self.is_local(music) and self.is_wav(unique_id):
            # convert to native filename
            filename = urlutil.get_path(unique_id)
            self.cache[unique_id] = GstCacheEntry(filename, False)
            on_cache = True

        return on_cache
Пример #10
0
    def is_available(self, music):
        on_cache = GstMusicPool.is_available(self, music)

        # XXX: when there is no gnomevfdssrc we have a problem because
        #      we are using URI's
        unique_id = self.unique_music_id(music)

        if not on_cache and self.is_local(music) and self.is_wav(unique_id):
            # convert to native filename
            filename = urlutil.get_path(unique_id)
            self.cache[unique_id] = GstCacheEntry(filename, False)
            on_cache = True

        return on_cache
Пример #11
0
def _get_paths_for_database_resources(hive_client, hive_resources):
    try:
        databases = hive_resources["database"]["values"]
        tables = hive_resources["table"]["values"]
    except KeyError as e:
        raise RangerSyncError(
            "Resource lack information about database or table. " + e.message)

    paths = []
    for db in databases:
        for table in tables:
            path = urlutil.get_path(hive_client.get_location(db, table))
            if path is not None:
                paths.append(path)
    return paths
Пример #12
0
    def is_available(self, music):
        on_cache = GstMusicPool.is_available(self, music)
        uri = gnomevfs.URI(music)

        # XXX: when there is no gnomevfdssrc we have a problem because
        #      we are using URI's
        unique_id = self.unique_music_id(music)
        is_pcm = audio.IsWavPcm(self.get_source(unique_id))

        if not on_cache and \
                    uri.is_local and \
                    gnomevfs.get_mime_type (music) == "audio/x-wav" and \
                    operations.syncOperation (is_pcm).id == operations.SUCCESSFUL:

            # convert to native filename
            filename = urlutil.get_path(unique_id)
            self.cache[unique_id] = GstCacheEntry(filename, False)
            on_cache = True
        del uri

        return on_cache
Пример #13
0
    def is_available (self, music):
        on_cache = GstMusicPool.is_available (self, music)
        uri = gnomevfs.URI (music)
        
        # XXX: when there is no gnomevfdssrc we have a problem because
        #      we are using URI's
        unique_id = self.unique_music_id (music)
        is_pcm = audio.IsWavPcm (self.get_source (unique_id))
        
        if not on_cache and \
                    uri.is_local and \
                    gnomevfs.get_mime_type (music) == "audio/x-wav" and \
                    operations.syncOperation (is_pcm).id == operations.SUCCESSFUL:

            # convert to native filename
            filename = urlutil.get_path (unique_id)
            self.cache[unique_id] = GstCacheEntry (filename, False)
            on_cache = True
        del uri

        return on_cache