def get_geo_topic_from_single_json(self, directory): files = get_all_files_in_directory(directory) for entry in files: with open(entry) as input_file: filename = os.path.basename(input_file.name).split('.')[0] metadata = json.load(input_file) self.geo_topic_list.append({filename: metadata})
def get_geo_topic_from_single_json(self, directory): files = get_all_files_in_directory(directory) for entry in files: with open(entry) as input_file: filename = os.path.basename(input_file.name).split('.')[0] metadata = json.load(input_file) self.geo_topic_list.append({filename: metadata})
def get_geo_topic_objects_from(self, directory): files = get_all_files_in_directory(directory) #print len(files) for entry in files: with open(entry) as input_file: #print(input_file.read()) self.geo_topic_list.extend(json.load(input_file))
def get_geo_topic_objects_from(self, directory): files = get_all_files_in_directory(directory) #print len(files) for entry in files: with open(entry) as input_file: #print(input_file.read()) self.geo_topic_list.extend(json.load(input_file))
def move_files(source_folder, destination_folder): files = utility.get_all_files_in_directory(source_folder) if not os.path.exists(destination_folder): os.makedirs(destination_folder) for filepath in files: filename = filepath.split('/')[-1] destination = [destination_folder, '/', filename] # print(filepath, ''.join(destination)) shutil.move(filepath, ''.join(destination)) return
def parse_owl_directory(path): file_list = get_all_files_in_directory(path, suffix='.owl') entologies = dict() for entry in file_list: print(entry) concept_dictionary = parse_owl_file(entry) entologies.update(concept_dictionary) dump(entologies, 'sweet_concepts.json') categories = transform_to_categories(entologies) dump(categories, 'sweet_concept_categories.json') return
def parse_owl_directory(path): file_list = get_all_files_in_directory(path, suffix='.owl') entologies = dict() for entry in file_list: print(entry) concept_dictionary = parse_owl_file(entry) entologies.update(concept_dictionary) dump(entologies, 'sweet_concepts.json') categories = transform_to_categories(entologies) dump(categories, 'sweet_concept_categories.json') return
def get_magic_bytes(folder): files = utility.get_all_files_in_directory(folder) data = {} for json_file in files: try: with open(json_file) as file1: magic_bytes = get_magic_bytes_from_file(file1) pure_name = file1.name.split('/')[-1] data[pure_name] = magic_bytes except Exception as e: sys.stderr.write(e.message) sys.stderr.write('\n') json_data = json.dumps(data, indent=4) print(json_data) return
def generate_files_from_directory(path_to_dir): files_to_process = utility.get_all_files_in_directory(path_to_dir) destination_folder = 'geo-topic-parser-folder' if not os.path.exists(destination_folder): os.makedirs(destination_folder) dir_name = os.path.basename(os.path.normpath(path_to_dir)) with open(destination_folder + '/geo-topic-' + dir_name + '-files.txt', 'w') as output_file2: for line in files_to_process: output_file2.write(line) output_file2.write('\n') # with open(path_to_dir + '_allfiles.txt', 'w') as output_file: # for line in files_to_process: # output_file.write(line) # output_file.write('\n') return
def generate_files_from_directory(path_to_dir): files_to_process = utility.get_all_files_in_directory(path_to_dir) destination_folder = 'geo-topic-parser-folder' if not os.path.exists(destination_folder): os.makedirs(destination_folder) dir_name = os.path.basename(os.path.normpath(path_to_dir)) with open(destination_folder + '/geo-topic-' + dir_name + '-files.txt', 'w') as output_file2: for line in files_to_process: output_file2.write(line) output_file2.write('\n') # with open(path_to_dir + '_allfiles.txt', 'w') as output_file: # for line in files_to_process: # output_file.write(line) # output_file.write('\n') return
def reverse(dir_list, output_name): with open(output_name, 'w') as output_file: files = [] for entry in dir_list: files.extend(get_all_files_in_directory(entry)) json_data = {} for entry in files: with open(entry) as input_json: json_data.update(json.load(input_json)) print(len(json_data)) filename_sweet_dictionary = {} for key in json_data.keys(): for filename in json_data[key].keys(): if filename not in filename_sweet_dictionary: filename_sweet_dictionary[filename] = {} filename_sweet_dictionary[filename][key] = 1 print(len(filename_sweet_dictionary)) # output_file.write(json.dumps(filename_sweet_dictionary)) return
def reverse(dir_list, output_name): with open(output_name, 'w') as output_file: files = [] for entry in dir_list: files.extend(get_all_files_in_directory(entry)) json_data = {} for entry in files: with open(entry) as input_json: json_data.update(json.load(input_json)) print(len(json_data)) filename_sweet_dictionary = {} for key in json_data.keys(): for filename in json_data[key].keys(): if filename not in filename_sweet_dictionary: filename_sweet_dictionary[filename] = {} filename_sweet_dictionary[filename][key] = 1 print(len(filename_sweet_dictionary)) # output_file.write(json.dumps(filename_sweet_dictionary)) return
def get_file_list(dir_list): result_list = [] for entry in dir_list: result_list.extend(utility.get_all_files_in_directory(entry)) return result_list
def get_measurement_objects_from(self, directory): files = get_all_files_in_directory(directory) for entry in files: with open(entry) as input_file: self.measurement_list.extend(json.load(input_file))
def test(): doi = sys.argv[2] doiDict = {} with open(doi, "r") as f: for line in f: key = line.split('/')[-1] doiDict[key.rstrip('\r\n')] = line.rstrip('\r\n') #with open("doiDict.json", 'w') as json_file: # json.dump(doiDict, json_file, indent = 4) time1 = datetime.datetime.now() measurement_storage = MeasurementStorage( '/media/ravirajukrishna/My Passport/yao_files/measurement/') sweet_storage = SweetStorage( '/media/ravirajukrishna/My Passport/yao_files/sweet/filename-sweet.json' ) geo_topic_storage = GeoTopicStorage( '/media/ravirajukrishna/My Passport/yao_geoData_Large/') geo_topic_storage.get_geo_topic_from_single_json( '/media/ravirajukrishna/My Passport/yao_geoData/') total_dictionary = dict() while measurement_storage.has_next_measurement_object(): filename, measurement_metadata = measurement_storage.get_next_measurement_object( ) if filename not in total_dictionary: if filename in doiDict: total_dictionary[filename] = {"doi": doiDict[filename]} else: total_dictionary[filename] = {"doi": filename} total_dictionary[filename].update(measurement_metadata) print "parsed measurement_storage" while sweet_storage.has_next_sweet_object(): filename, sweet_metadata = sweet_storage.get_next_sweet_object() if filename not in total_dictionary: if filename in doiDict: total_dictionary[filename] = {"doi": doiDict[filename]} else: total_dictionary[filename] = {"doi": filename} total_dictionary[filename].update(sweet_metadata) print "parsed sweet_storage" while geo_topic_storage.has_next_geo_topic_object(): filename, geo_topic_metadata = geo_topic_storage.get_next_geo_topic_object( ) if filename not in total_dictionary: if filename in doiDict: total_dictionary[filename] = {"doi": doiDict[filename]} else: total_dictionary[filename] = {"doi": filename} total_dictionary[filename].update(geo_topic_metadata) print "parsed geo_topic_storage" input_dir = sys.argv[1] input_files = get_all_files_in_directory(input_dir) input_files_set = set(input_files) for ip_file in input_files_set: #print ip_file key = ip_file.split('/')[-1] filename_part = key.rstrip(".json") #print filename_part with open(ip_file) as json_file: jsondict = json.load(json_file) #print json.dumps(jsondict, indent=4) if filename_part not in total_dictionary: total_dictionary[filename_part] = {} total_dictionary[filename_part].update(jsondict) print "parsed grobid_scholar" #final write to each json file i = 1 existing_files_set = set(total_dictionary.keys()) for key in existing_files_set: filename = "/media/ravirajukrishna/My Passport/result_json/" + key + ".json" i = i + 1 print i, print ":", print filename with open(filename, 'w') as json_file: json.dump(total_dictionary[key], json_file, indent=4) time2 = datetime.datetime.now() print(time2 - time1) return
def function(): file_list = utility.get_all_files_in_directory('/Users/Frank/PycharmProjects/599assignment1/hw3/visualization/spectrum-grouped-by-domain/') for entry in file_list: print(os.path.basename(entry))
def get_measurement_objects_from(self, directory): files = get_all_files_in_directory(directory) for entry in files: with open(entry) as input_file: self.measurement_list.extend(json.load(input_file))
def test(): doi = sys.argv[2] doiDict = {} with open(doi, "r") as f: for line in f: key = line.split('/')[-1] doiDict[key.rstrip('\r\n')] = line.rstrip('\r\n') #with open("doiDict.json", 'w') as json_file: # json.dump(doiDict, json_file, indent = 4) time1 = datetime.datetime.now() measurement_storage = MeasurementStorage('/media/ravirajukrishna/My Passport/yao_files/measurement/') sweet_storage = SweetStorage('/media/ravirajukrishna/My Passport/yao_files/sweet/filename-sweet.json') geo_topic_storage = GeoTopicStorage('/media/ravirajukrishna/My Passport/yao_geoData_Large/') geo_topic_storage.get_geo_topic_from_single_json('/media/ravirajukrishna/My Passport/yao_geoData/') total_dictionary = dict() while measurement_storage.has_next_measurement_object(): filename, measurement_metadata = measurement_storage.get_next_measurement_object() if filename not in total_dictionary: if filename in doiDict: total_dictionary[filename] = {"doi" : doiDict[filename]} else: total_dictionary[filename] = {"doi" : filename} total_dictionary[filename].update(measurement_metadata) print "parsed measurement_storage" while sweet_storage.has_next_sweet_object(): filename, sweet_metadata = sweet_storage.get_next_sweet_object() if filename not in total_dictionary: if filename in doiDict: total_dictionary[filename] = {"doi" : doiDict[filename]} else: total_dictionary[filename] = {"doi" : filename} total_dictionary[filename].update(sweet_metadata) print "parsed sweet_storage" while geo_topic_storage.has_next_geo_topic_object(): filename, geo_topic_metadata = geo_topic_storage.get_next_geo_topic_object() if filename not in total_dictionary: if filename in doiDict: total_dictionary[filename] = {"doi" : doiDict[filename]} else: total_dictionary[filename] = {"doi" : filename} total_dictionary[filename].update(geo_topic_metadata) print "parsed geo_topic_storage" input_dir = sys.argv[1] input_files = get_all_files_in_directory(input_dir) input_files_set = set(input_files) for ip_file in input_files_set: #print ip_file key = ip_file.split('/')[-1] filename_part = key.rstrip(".json") #print filename_part with open(ip_file) as json_file: jsondict = json.load(json_file) #print json.dumps(jsondict, indent=4) if filename_part not in total_dictionary: total_dictionary[filename_part] = {} total_dictionary[filename_part].update(jsondict) print "parsed grobid_scholar" #final write to each json file i=1 existing_files_set = set(total_dictionary.keys()) for key in existing_files_set: filename = "/media/ravirajukrishna/My Passport/result_json/" + key + ".json" i = i+1 print i, print ":", print filename with open(filename,'w') as json_file: json.dump(total_dictionary[key], json_file, indent = 4) time2 = datetime.datetime.now() print(time2 - time1) return