def poke(self, context): table_uri = '{0}:{1}.{2}'.format(self.project_id, self.dataset_id, self.table_id) self.log.info('Sensor checks if the partition %s exists of table: %s', self.expected_partition, table_uri) hook = BigQueryHook(bigquery_conn_id=self.bigquery_conn_id, delegate_to=self.delegate_to) return self.expected_partition in hook.table_list_partition( self.project_id, self.dataset_id, self.table_id)
def execute(self, context): self.log.info('Fetching last partition from tables: {}'.format( str(self.table_lst))) hook = BigQueryHook(bigquery_conn_id=self.bigquery_conn_id, delegate_to=self.delegate_to) return_dict = {} for each_table in self.table_lst: project = each_table.split(':')[0] dataset = each_table.split(':')[1].split('.')[0] table_name = each_table.split(':')[1].split('.')[1] lp = sorted(hook.table_list_partition(project, dataset, table_name))[-1] return_dict[each_table] = lp self.log.info("Table {} has latest partition: {}".format( each_table, lp)) return return_dict