def create_data_set(self, big_query_client: BigQueryAdapter) -> bool:
# return True if dataset already exists
if big_query_client.check_dataset(self.definitions['data_set']):
return True
big_query_client.set_data_set_ref(self.definitions['data_set'])
result = big_query_client.create_data_set()
if result:
self.sd_logger.info(
{'message': f"Created {self.definitions['data_set']} Data Set"},
{'class': 'PrefectAddServiceBigQuery', 'method': 'create_data_set'})
else:
self.sd_logger.warning(
big_query_client.errors,
{'class': 'PrefectAddServiceBigQuery', 'method': 'create_data_set'}
)
return result
def create_config_data_set(self,
big_query_client: BigQueryAdapter) -> bool:
# return True if dataset already exists
if big_query_client.check_dataset('mysql_sync'):
return True
big_query_client.set_data_set_ref('mysql_sync')
result = big_query_client.create_data_set()
if result:
self.sd_logger.info({'message': f"Created mysql_sync Data Set"}, {
'class': 'PrefectInstallBigQuery',
'method': 'create_data_set'
})
else:
self.sd_logger.warning(big_query_client.errors, {
'class': 'PrefectInstallBigQuery',
'method': 'create_data_set'
})
return result