def execute(self, context: 'Context') -> None: gcs_hook = GCSHook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, impersonation_chain=self.impersonation_chain, ) presto_hook = PrestoHook(presto_conn_id=self.presto_conn_id) with NamedTemporaryFile("w+") as temp_file: self.log.info("Downloading data from %s", self.source_object) gcs_hook.download( bucket_name=self.source_bucket, object_name=self.source_object, filename=temp_file.name, ) data = list(csv.reader(temp_file)) fields = tuple(data[0]) rows = [] for row in data[1:]: rows.append(tuple(row)) self.log.info("Inserting data into %s", self.presto_table) presto_hook.insert_rows(table=self.presto_table, rows=rows, target_fields=fields)
def execute(self, context: 'Context') -> None: gcs_hook = GCSHook( gcp_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to, impersonation_chain=self.impersonation_chain, ) presto_hook = PrestoHook(presto_conn_id=self.presto_conn_id) with NamedTemporaryFile("w+") as temp_file: self.log.info("Downloading data from %s", self.source_object) gcs_hook.download( bucket_name=self.source_bucket, object_name=self.source_object, filename=temp_file.name, ) data = csv.reader(temp_file) rows = (tuple(row) for row in data) self.log.info("Inserting data into %s", self.presto_table) if self.schema_fields: presto_hook.insert_rows(table=self.presto_table, rows=rows, target_fields=self.schema_fields) elif self.schema_object: blob = gcs_hook.download( bucket_name=self.source_bucket, object_name=self.schema_object, ) schema_fields = json.loads(blob.decode("utf-8")) presto_hook.insert_rows(table=self.presto_table, rows=rows, target_fields=schema_fields) else: presto_hook.insert_rows(table=self.presto_table, rows=rows)