def __init__(self, query_profile, ref_connection, test_connection): '''query_profile should be an instance of one of the profiles in query_profile.py''' self.query_profile = query_profile self.ref_connection = ref_connection self.test_connection = test_connection self.common_tables = DbConnection.describe_common_tables( [ref_connection, test_connection])
def start(self): try: self.prepare() self.query_generator = QueryGenerator(self.query_profile) self.common_tables = DbConnection.describe_common_tables( [self.ref_connection, self.test_connection]) for query_model in self.queries_to_be_executed(): LOG.info('About to execute query.') result_dict = self.run_query(query_model) LOG.info('Query Executed successfully.') if result_dict: self.num_queries_executed += 1 self.result_list.append(result_dict) LOG.info('Time Left: {0}'.format(self.target_stop_time - time())) if time() > self.target_stop_time: break self.stop_time = time() self.save_pickle() self.generate_report() LOG.info('Generated Report') except: LOG.exception('Unexpected Exception in start') raise finally: self.impala_env.stop_docker() LOG.info('Docker Stopped') try: os.remove(join_path(PATH_TO_SCHEDULE, self.job_id)) LOG.info('Schedule file removed') except OSError: LOG.info('Unable to remove schedule file.')
def __init__(self, impala_connection, reference_connection, filter_col_types=[]): self.impala_connection = impala_connection self.reference_connection = reference_connection self.common_tables = DbConnection.describe_common_tables( [impala_connection, reference_connection], filter_col_types=filter_col_types) # A file-backed dict of queries that produced a discrepancy, keyed by query number # (in string form, as required by the dict). self.query_shelve_path = gettempdir() + '/query.shelve' # A list of all queries attempted self.query_log_path = gettempdir() + '/impala_query_log.sql'