def add_hdfs_path(self, hdfs_path): """ Post to http://atlas.hadoop.svenskaspel.se/api/atlas/v2/entity Post data {"entity":{"typeName":"hdfs_path","attributes":{"description":null,"name":"hdfs://svsprod/apps/hive/warehouse/hadoop_out_prod.db/country_d","owner":null,"qualifiedName":"hdfs://svsprod/apps/hive/warehouse/hadoop_out_prod.db/country_d","createTime":1536098400000,"fileSize":null,"group":null,"isFile":null,"isSymlink":null,"modifiedTime":1536098400000,"path":"hdfs://svsprod/apps/hive/warehouse/hadoop_out_prod.db/country_d","clusterName":null,"numberOfReplicas":null},"guid":-1},"referredEntities":{}} Response: {"mutatedEntities":{"CREATE":[{"typeName":"hdfs_path","attributes":{"qualifiedName":"hdfs://svsprod/apps/hive/warehouse/hadoop_out_prod.db/country_d"},"guid":"e20823a6-5521-4dc9-b2a7-b5a1d9babecd","status":"ACTIVE"}]},"guidAssignments":{"-1":"e20823a6-5521-4dc9-b2a7-b5a1d9babecd"}} :param hdfs_path: Full url to the file or directory hdfs://environment/my/path/ :return: guid assigned """ cluster_name = urlutil.get_host(hdfs_path) name = urlutil.get_path(hdfs_path) entity = { "entity": { "typeName": "hdfs_path", "attributes": { "description": "Created/Updated by cobra-policytool.", "name": name, "qualifiedName": hdfs_path, "path": hdfs_path, "clusterName": cluster_name, }, "guid": -1 }, } response = self._post_entity(entity) if response.status_code == 200: json_response = response.json() if json_response.has_key('guidAssignments'): return json_response['guidAssignments']['-1'] else: AtlasError( "Failed to add hdfs path {} content mismatch {}".format( hdfs_path, response.content)) else: raise AtlasError(response.content, response.status_code)
def get_temporary_dirs(self): # First try the GConf temporary dir, then try the system TEMP dir # finally try on the HOME temps = (self.temporary_dir, TEMP_DIR, HOME_DIR) for temp in temps: if urlutil.is_local(temp) and _is_dir_writable(temp): yield urlutil.get_path(temp)
def get_source(self, music): if self.use_gnomevfs: src = gst.element_factory_make("gnomevfssrc") src.set_property("location", music) else: src = gst.element_factory_make("filesrc", "source") src.set_property("location", urlutil.get_path(music)) return src
def is_available(self, music): on_cache = GstMusicPool.is_available(self, music) unique_id = self.unique_music_id(music) if not on_cache and self.is_local(music) and self.is_wav(unique_id): # convert to native filename filename = urlutil.get_path(unique_id) self.cache[unique_id] = GstCacheEntry(filename, False) on_cache = True return on_cache
def before_music_fetched(self, evt, music): print music prog_txt = "Converting " prog_txt += path.basename(urlutil.get_path(music['location'])) self.prog.sub_progress_text = prog_txt self.prog.progress_fraction = self.oper.progress if self.music: print "Fetched", self.music['cache_location'] print "Fetching", music['location'] self.music = music
def is_available(self, music): on_cache = GstMusicPool.is_available(self, music) # XXX: when there is no gnomevfdssrc we have a problem because # we are using URI's unique_id = self.unique_music_id(music) if not on_cache and self.is_local(music) and self.is_wav(unique_id): # convert to native filename filename = urlutil.get_path(unique_id) self.cache[unique_id] = GstCacheEntry(filename, False) on_cache = True return on_cache
def _get_paths_for_database_resources(hive_client, hive_resources): try: databases = hive_resources["database"]["values"] tables = hive_resources["table"]["values"] except KeyError as e: raise RangerSyncError( "Resource lack information about database or table. " + e.message) paths = [] for db in databases: for table in tables: path = urlutil.get_path(hive_client.get_location(db, table)) if path is not None: paths.append(path) return paths
def is_available(self, music): on_cache = GstMusicPool.is_available(self, music) uri = gnomevfs.URI(music) # XXX: when there is no gnomevfdssrc we have a problem because # we are using URI's unique_id = self.unique_music_id(music) is_pcm = audio.IsWavPcm(self.get_source(unique_id)) if not on_cache and \ uri.is_local and \ gnomevfs.get_mime_type (music) == "audio/x-wav" and \ operations.syncOperation (is_pcm).id == operations.SUCCESSFUL: # convert to native filename filename = urlutil.get_path(unique_id) self.cache[unique_id] = GstCacheEntry(filename, False) on_cache = True del uri return on_cache
def is_available (self, music): on_cache = GstMusicPool.is_available (self, music) uri = gnomevfs.URI (music) # XXX: when there is no gnomevfdssrc we have a problem because # we are using URI's unique_id = self.unique_music_id (music) is_pcm = audio.IsWavPcm (self.get_source (unique_id)) if not on_cache and \ uri.is_local and \ gnomevfs.get_mime_type (music) == "audio/x-wav" and \ operations.syncOperation (is_pcm).id == operations.SUCCESSFUL: # convert to native filename filename = urlutil.get_path (unique_id) self.cache[unique_id] = GstCacheEntry (filename, False) on_cache = True del uri return on_cache