def run(self):
print("\n*********** STARTING... ***********\n")
print('ES INDEX: {0}'.format(self.es_index))
page = self.get_page()
sid = page['_scroll_id']
scroll_size = page['hits']['total']
progress = ProgressBar(scroll_size, 'completed')
progress.print()
self.silent_remove(self.file_path)
self.write_header_to_csv(self.file_path, parser.get_column_names(self.body))
self.write_data_to_csv(page['hits']['hits'])
while (scroll_size > 0):
page = self.es.scroll(scroll_id=sid, scroll='2m')
sid = page['_scroll_id']
scroll_size = len(page['hits']['hits'])
progress.update_current_count(scroll_size)
progress.print()
self.write_data_to_csv(page['hits']['hits'])
print("\n*********** COMPLETED ***********")
def createEdgesThenSaveToFile(self):
counter = 0
num_of_iterations = len(self.tmc_ref_data.keys())**2
progressbar = ProgressBar(num_of_iterations)
with open(self.edge_output_file, mode='w', newline='') as network_edges_file:
writer = csv.writer(network_edges_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
writer.writerow(["start_node", "end_node", "distance", "from_dir", "to_dir"])
tmc_list = self.tmc_ref_data.items()
for i, itmc in tmc_list:
for j, jtmc in tmc_list:
if (itmc["tmc"] != jtmc["tmc"]):
dist = self.euclidean_distance(
float(itmc["start_latitude"]), float(itmc["start_longitude"]),
float(jtmc["start_latitude"]), float(jtmc["start_longitude"]))
if (dist <= self.max_tmc_separation and
jtmc["direction"][0] not in self.avoid_directions[itmc["direction"][0]]):
writer.writerow(
[itmc["tmc"], jtmc["tmc"], dist, itmc["direction"][0], jtmc["direction"][0]])
counter += 1
progressbar.print(counter)
print("Successfully created the edges and saved them to the file " + self.edge_output_file)
return self.edge_output_file
