def run(self): print("\n*********** STARTING... ***********\n") print('ES INDEX: {0}'.format(self.es_index)) page = self.get_page() sid = page['_scroll_id'] scroll_size = page['hits']['total'] progress = ProgressBar(scroll_size, 'completed') progress.print() self.silent_remove(self.file_path) self.write_header_to_csv(self.file_path, parser.get_column_names(self.body)) self.write_data_to_csv(page['hits']['hits']) while (scroll_size > 0): page = self.es.scroll(scroll_id=sid, scroll='2m') sid = page['_scroll_id'] scroll_size = len(page['hits']['hits']) progress.update_current_count(scroll_size) progress.print() self.write_data_to_csv(page['hits']['hits']) print("\n*********** COMPLETED ***********")
def createEdgesThenSaveToFile(self): counter = 0 num_of_iterations = len(self.tmc_ref_data.keys())**2 progressbar = ProgressBar(num_of_iterations) with open(self.edge_output_file, mode='w', newline='') as network_edges_file: writer = csv.writer(network_edges_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) writer.writerow(["start_node", "end_node", "distance", "from_dir", "to_dir"]) tmc_list = self.tmc_ref_data.items() for i, itmc in tmc_list: for j, jtmc in tmc_list: if (itmc["tmc"] != jtmc["tmc"]): dist = self.euclidean_distance( float(itmc["start_latitude"]), float(itmc["start_longitude"]), float(jtmc["start_latitude"]), float(jtmc["start_longitude"])) if (dist <= self.max_tmc_separation and jtmc["direction"][0] not in self.avoid_directions[itmc["direction"][0]]): writer.writerow( [itmc["tmc"], jtmc["tmc"], dist, itmc["direction"][0], jtmc["direction"][0]]) counter += 1 progressbar.print(counter) print("Successfully created the edges and saved them to the file " + self.edge_output_file) return self.edge_output_file