def execute(self, context): snowflake_hook = SnowflakeHook( snowflake_conn_id=self.snowflake_conn_id) # Snowflake won't accept list of files it has to be tuple only. # but in python tuple([1]) = (1,) => which is invalid for snowflake files = str(self.s3_keys) files = files.replace('[', '(') files = files.replace(']', ')') # we can extend this based on stage base_sql = """ FROM @{stage}/ files={files} file_format={file_format} """.format(stage=self.stage, files=files, file_format=self.file_format) if self.columns_array: copy_query = """ COPY INTO {schema}.{table}({columns}) {base_sql} """.format(schema=self.schema, table=self.table, columns=",".join(self.columns_array), base_sql=base_sql) else: copy_query = """ COPY INTO {schema}.{table} {base_sql} """.format(schema=self.schema, table=self.table, base_sql=base_sql) self.log.info('Executing COPY command...') snowflake_hook.run(copy_query, self.autocommit) self.log.info("COPY command completed")
def execute(self, context: Any) -> None: snowflake_hook = SnowflakeHook( snowflake_conn_id=self.snowflake_conn_id, warehouse=self.warehouse, database=self.database, role=self.role, schema=self.schema, authenticator=self.authenticator, session_parameters=self.session_parameters, ) if self.schema: into = f"{self.schema}.{self.table}" else: into = self.table if self.columns_array: into = f"{into}({','.join(self.columns_array)})" sql_parts = [ f"COPY INTO {into}", f"FROM @{self.stage}/{self.prefix or ''}", ] if self.s3_keys: files = ", ".join(f"'{key}'" for key in self.s3_keys) sql_parts.append(f"files=({files})") sql_parts.append(f"file_format={self.file_format}") copy_query = "\n".join(sql_parts) self.log.info('Executing COPY command...') snowflake_hook.run(copy_query, self.autocommit) self.log.info("COPY command completed")
def execute(self, context: Any) -> None: snowflake_hook = SnowflakeHook( snowflake_conn_id=self.snowflake_conn_id, warehouse=self.warehouse, database=self.database, role=self.role, schema=self.schema, authenticator=self.authenticator, session_parameters=self.session_parameters, ) files = "" if self.s3_keys: files = "files=({})".format(", ".join(f"'{key}'" for key in self.s3_keys)) # we can extend this based on stage base_sql = """ FROM @{stage}/{prefix} {files} file_format={file_format} """.format( stage=self.stage, prefix=(self.prefix if self.prefix else ""), files=files, file_format=self.file_format, ) if self.columns_array: copy_query = """ COPY INTO {schema}.{table}({columns}) {base_sql} """.format(schema=self.schema, table=self.table, columns=",".join(self.columns_array), base_sql=base_sql) else: copy_query = f""" COPY INTO {self.schema}.{self.table} {base_sql} """ copy_query = "\n".join(line.strip() for line in copy_query.splitlines()) self.log.info('Executing COPY command...') snowflake_hook.run(copy_query, self.autocommit) self.log.info("COPY command completed")