def poke(self, context):
table_uri = '{0}:{1}.{2}'.format(self.project_id, self.dataset_id,
self.table_id)
self.log.info('Sensor checks if the partition %s exists of table: %s',
self.expected_partition, table_uri)
hook = BigQueryHook(bigquery_conn_id=self.bigquery_conn_id,
delegate_to=self.delegate_to)
return self.expected_partition in hook.table_list_partition(
self.project_id, self.dataset_id, self.table_id)
def execute(self, context):
self.log.info('Fetching last partition from tables: {}'.format(
str(self.table_lst)))
hook = BigQueryHook(bigquery_conn_id=self.bigquery_conn_id,
delegate_to=self.delegate_to)
return_dict = {}
for each_table in self.table_lst:
project = each_table.split(':')[0]
dataset = each_table.split(':')[1].split('.')[0]
table_name = each_table.split(':')[1].split('.')[1]
lp = sorted(hook.table_list_partition(project, dataset,
table_name))[-1]
return_dict[each_table] = lp
self.log.info("Table {} has latest partition: {}".format(
each_table, lp))
return return_dict