def metatable_update(dag, tablename, schedule, **kwargs):
dataset = tablename.split('.')[1]
table_name = tablename.split('.')[2]
bq_hook = BigQueryHook(bigquery_conn_id='bigquery_default',
location='europe-west3',
use_legacy_sql=False)
pg_conn = config[schedule][tablename][1]
ti = kwargs['ti']
pgmax_ts = str(
ti.xcom_pull(task_ids='get_maxts_{}_{}'.format(tablename, pg_conn))[1])
bq_hook.run(
sql=
"update bqadmin.tablesync_meta set max_value='{}' where tablename='{}' and datasource_dbconn='{}'"
.format(pgmax_ts, dataset + '.' + table_name, pg_conn))
return 'Executed the update query'
def execute(self, context):
logging.info('Executing: %s', str(self.bql))
hook = BigQueryHook(bigquery_conn_id=self.bigquery_conn_id,
delegate_to=self.delegate_to)
hook.run(self.bql, self.destination_dataset_table,
self.write_disposition)
def execute(self, context):
logging.info('Executing: %s', str(self.bql))
hook = BigQueryHook(bigquery_conn_id=self.bigquery_conn_id)
hook.run(self.bql, self.destination_dataset_table)
def execute(self, context):
logging.info('Executing: %s', str(self.bql))
hook = BigQueryHook(bigquery_conn_id=self.bigquery_conn_id)
hook.run(self.bql, self.destination_dataset_table)
def execute(self, context):
logging.info('Executing: %s', str(self.bql))
hook = BigQueryHook(bigquery_conn_id=self.bigquery_conn_id, delegate_to=self.delegate_to)
hook.run(self.bql, self.destination_dataset_table, self.write_disposition)