def __init__(self, query_profile, ref_connection, test_connection):
'''query_profile should be an instance of one of the profiles in query_profile.py'''
self.query_profile = query_profile
self.ref_connection = ref_connection
self.test_connection = test_connection
self.common_tables = DbConnection.describe_common_tables(
[ref_connection, test_connection])
def start(self):
try:
self.prepare()
self.query_generator = QueryGenerator(self.query_profile)
self.common_tables = DbConnection.describe_common_tables(
[self.ref_connection, self.test_connection])
for query_model in self.queries_to_be_executed():
LOG.info('About to execute query.')
result_dict = self.run_query(query_model)
LOG.info('Query Executed successfully.')
if result_dict:
self.num_queries_executed += 1
self.result_list.append(result_dict)
LOG.info('Time Left: {0}'.format(self.target_stop_time - time()))
if time() > self.target_stop_time:
break
self.stop_time = time()
self.save_pickle()
self.generate_report()
LOG.info('Generated Report')
except:
LOG.exception('Unexpected Exception in start')
raise
finally:
self.impala_env.stop_docker()
LOG.info('Docker Stopped')
try:
os.remove(join_path(PATH_TO_SCHEDULE, self.job_id))
LOG.info('Schedule file removed')
except OSError:
LOG.info('Unable to remove schedule file.')
def __init__(self, query_profile, ref_connection, test_connection):
'''query_profile should be an instance of one of the profiles in query_profile.py'''
self.query_profile = query_profile
self.ref_connection = ref_connection
self.test_connection = test_connection
self.common_tables = DbConnection.describe_common_tables(
[ref_connection, test_connection])
def start(self):
try:
self.prepare()
self.query_generator = QueryGenerator(self.query_profile)
self.common_tables = DbConnection.describe_common_tables(
[self.ref_connection, self.test_connection])
for query_model in self.queries_to_be_executed():
LOG.info('About to execute query.')
result_dict = self.run_query(query_model)
LOG.info('Query Executed successfully.')
if result_dict:
self.num_queries_executed += 1
self.result_list.append(result_dict)
LOG.info('Time Left: {0}'.format(self.target_stop_time -
time()))
if time() > self.target_stop_time:
break
self.stop_time = time()
self.save_pickle()
self.generate_report()
LOG.info('Generated Report')
except:
LOG.exception('Unexpected Exception in start')
raise
finally:
self.impala_env.stop_docker()
LOG.info('Docker Stopped')
try:
os.remove(join_path(PATH_TO_SCHEDULE, self.job_id))
LOG.info('Schedule file removed')
except OSError:
LOG.info('Unable to remove schedule file.')
def __init__(self, impala_connection, reference_connection, filter_col_types=[]):
self.impala_connection = impala_connection
self.reference_connection = reference_connection
self.common_tables = DbConnection.describe_common_tables(
[impala_connection, reference_connection],
filter_col_types=filter_col_types)
# A file-backed dict of queries that produced a discrepancy, keyed by query number
# (in string form, as required by the dict).
self.query_shelve_path = gettempdir() + '/query.shelve'
# A list of all queries attempted
self.query_log_path = gettempdir() + '/impala_query_log.sql'
def __init__(self,
impala_connection,
reference_connection,
filter_col_types=[]):
self.impala_connection = impala_connection
self.reference_connection = reference_connection
self.common_tables = DbConnection.describe_common_tables(
[impala_connection, reference_connection],
filter_col_types=filter_col_types)
# A file-backed dict of queries that produced a discrepancy, keyed by query number
# (in string form, as required by the dict).
self.query_shelve_path = gettempdir() + '/query.shelve'
# A list of all queries attempted
self.query_log_path = gettempdir() + '/impala_query_log.sql'