def execute(self, context): hive = HiveCliHook(hive_cli_conn_id=self.hive_cli_conn_id) vertica = VerticaHook(vertica_conn_id=self.vertica_conn_id) logging.info("Dumping Vertica query results to local file") conn = vertica.get_conn() cursor = conn.cursor() cursor.execute(self.sql) with NamedTemporaryFile("w") as f: csv_writer = csv.writer(f, delimiter=self.delimiter, encoding='utf-8') field_dict = OrderedDict() col_count = 0 for field in cursor.description: col_count += 1 col_position = "Column{position}".format(position=col_count) field_dict[col_position if field[0] == '' else field[0]] = self.type_map(field[1]) csv_writer.writerows(cursor.iterate()) f.flush() cursor.close() conn.close() logging.info("Loading file into Hive") hive.load_file( f.name, self.hive_table, field_dict=field_dict, create=self.create, partition=self.partition, delimiter=self.delimiter, recreate=self.recreate)
def vertica_counter(**kwargs): table = 'table_schema.TABLE_NAME' cur = VerticaHook('vertica').get_cursor() cur.execute('select count(*) from {}'.format(table)) counter = cur.fetchall() counter = counter[0][0] return counter
def execute(self, context): logging.info('Executing: ' + str(self.sql)) hook = VerticaHook(vertica_conn_id=self.vertica_conn_id) hook.run(self.sql)