Beispiel #1
0
 def __init__(self, query_profile, ref_connection, test_connection):
   '''query_profile should be an instance of one of the profiles in query_profile.py'''
   self.query_profile = query_profile
   self.ref_connection = ref_connection
   self.test_connection = test_connection
   self.common_tables = DbConnection.describe_common_tables(
       [ref_connection, test_connection])
Beispiel #2
0
  def start(self):
    try:
      self.prepare()
      self.query_generator = QueryGenerator(self.query_profile)
      self.common_tables = DbConnection.describe_common_tables(
          [self.ref_connection, self.test_connection])

      for query_model in self.queries_to_be_executed():
        LOG.info('About to execute query.')
        result_dict = self.run_query(query_model)
        LOG.info('Query Executed successfully.')
        if result_dict:
          self.num_queries_executed += 1
          self.result_list.append(result_dict)
        LOG.info('Time Left: {0}'.format(self.target_stop_time - time()))
        if time() > self.target_stop_time:
          break
      self.stop_time = time()
      self.save_pickle()
      self.generate_report()
      LOG.info('Generated Report')
    except:
      LOG.exception('Unexpected Exception in start')
      raise
    finally:
      self.impala_env.stop_docker()
      LOG.info('Docker Stopped')
      try:
        os.remove(join_path(PATH_TO_SCHEDULE, self.job_id))
        LOG.info('Schedule file removed')
      except OSError:
        LOG.info('Unable to remove schedule file.')
 def __init__(self, query_profile, ref_connection, test_connection):
     '''query_profile should be an instance of one of the profiles in query_profile.py'''
     self.query_profile = query_profile
     self.ref_connection = ref_connection
     self.test_connection = test_connection
     self.common_tables = DbConnection.describe_common_tables(
         [ref_connection, test_connection])
Beispiel #4
0
    def start(self):
        try:
            self.prepare()
            self.query_generator = QueryGenerator(self.query_profile)
            self.common_tables = DbConnection.describe_common_tables(
                [self.ref_connection, self.test_connection])

            for query_model in self.queries_to_be_executed():
                LOG.info('About to execute query.')
                result_dict = self.run_query(query_model)
                LOG.info('Query Executed successfully.')
                if result_dict:
                    self.num_queries_executed += 1
                    self.result_list.append(result_dict)
                LOG.info('Time Left: {0}'.format(self.target_stop_time -
                                                 time()))
                if time() > self.target_stop_time:
                    break
            self.stop_time = time()
            self.save_pickle()
            self.generate_report()
            LOG.info('Generated Report')
        except:
            LOG.exception('Unexpected Exception in start')
            raise
        finally:
            self.impala_env.stop_docker()
            LOG.info('Docker Stopped')
            try:
                os.remove(join_path(PATH_TO_SCHEDULE, self.job_id))
                LOG.info('Schedule file removed')
            except OSError:
                LOG.info('Unable to remove schedule file.')
  def __init__(self, impala_connection, reference_connection, filter_col_types=[]):
    self.impala_connection = impala_connection
    self.reference_connection = reference_connection
    self.common_tables = DbConnection.describe_common_tables(
        [impala_connection, reference_connection],
        filter_col_types=filter_col_types)

    # A file-backed dict of queries that produced a discrepancy, keyed by query number
    # (in string form, as required by the dict).
    self.query_shelve_path = gettempdir() + '/query.shelve'

    # A list of all queries attempted
    self.query_log_path = gettempdir() + '/impala_query_log.sql'
    def __init__(self,
                 impala_connection,
                 reference_connection,
                 filter_col_types=[]):
        self.impala_connection = impala_connection
        self.reference_connection = reference_connection
        self.common_tables = DbConnection.describe_common_tables(
            [impala_connection, reference_connection],
            filter_col_types=filter_col_types)

        # A file-backed dict of queries that produced a discrepancy, keyed by query number
        # (in string form, as required by the dict).
        self.query_shelve_path = gettempdir() + '/query.shelve'

        # A list of all queries attempted
        self.query_log_path = gettempdir() + '/impala_query_log.sql'