def vertica_compliant_schema(self): """Transforms mysql table schema into a vertica compliant schema.""" if not self.table_schema: results = get_mysql_query_results( self.db_credentials, self.database, 'describe {}'.format(self.table_name)) for result in results: field_name = result[0].strip() field_type = result[1].strip() field_null = result[2].strip() types_with_parentheses = [ 'tinyint', 'smallint', 'int', 'bigint', 'datetime' ] if any(_type in field_type for _type in types_with_parentheses): field_type = field_type.rsplit('(')[0] elif field_type == 'longtext': field_type = 'LONG VARCHAR' elif field_type == 'double': field_type = 'DOUBLE PRECISION' if field_null == "NO": field_type = field_type + " NOT NULL" field_name = "\"{}\"".format(field_name) self.table_schema.append((field_name, field_type)) return self.table_schema
def vertica_compliant_schema(self): """Transforms mysql table schema into a vertica compliant schema.""" if not self.table_schema: results = get_mysql_query_results(self.db_credentials, self.database, 'describe {}'.format(self.table_name)) for result in results: field_name = result[0].strip() field_type = result[1].strip() field_null = result[2].strip() types_with_parentheses = ['tinyint', 'smallint', 'int', 'bigint', 'datetime'] if any(_type in field_type for _type in types_with_parentheses): field_type = field_type.rsplit('(')[0] elif field_type == 'longtext': field_type = 'LONG VARCHAR' elif field_type == 'double': field_type = 'DOUBLE PRECISION' if field_null == "NO": field_type = field_type + " NOT NULL" field_name = "\"{}\"".format(field_name) self.table_schema.append((field_name, field_type)) return self.table_schema
def requires(self): if not self.table_list: results = get_mysql_query_results(self.db_credentials, self.database, 'show tables') self.table_list = [result[0].strip() for result in results] for table_name in self.table_list: if not self.should_exclude_table(table_name): yield LoadMysqlToVerticaTableTask( credentials=self.credentials, schema=self.schema, db_credentials=self.db_credentials, database=self.database, warehouse_path=self.warehouse_path, table_name=table_name, overwrite=self.overwrite, date=self.date, )