def run_loader(self):
"""Main function for running loader"""
if self.args.verbose:
self.logger.warn('DEBUG mode enabled!')
time.sleep(3)
data_manager = DataFileManager(self.context_info.config_file_location)
file_transactor = FileTransactor()
file_transactor.start_threads(
data_manager.get_file_transactor_thread_settings())
data_manager.download_and_validate()
self.logger.debug("finished downloading now doing thread")
file_transactor.check_for_thread_errors()
self.logger.debug("finished threads waiting for queues")
file_transactor.wait_for_queues()
self.logger.debug("finished queues waiting for shutdown")
file_transactor.shutdown()
neo_transactor = Neo4jTransactor()
neo_transactor.start_threads(
data_manager.get_neo_transactor_thread_settings())
self.logger.debug("finished starting neo threads ")
if not self.context_info.env["USING_PICKLE"]:
self.logger.info("Creating indices.")
Neo4jHelper.create_indices()
etl_time_tracker_list = self.run_etl_groups(self.logger, data_manager,
neo_transactor)
neo_transactor.shutdown()
elapsed_time = time.time() - self.start_time
for time_item in etl_time_tracker_list:
self.logger.info(time_item)
self.logger.info('Loader finished. Elapsed time: %s' %
time.strftime("%H:%M:%S", time.gmtime(elapsed_time)))
def test_rel_exists(node1, node2):
"""Test Relationship Exists"""
query = """MATCH (n:%s)-[]-(m:%s)
RETURN DISTINCT COUNT(n) AS count""" % (node1, node2)
result = Neo4jHelper.run_single_query(query)
for record in result:
assert record["count"] > 0
def execute_transaction(query):
"""Execute Transaction"""
return Neo4jHelper.run_single_query(query)