def vertica_compliant_schema(self):
"""Transforms mysql table schema into a vertica compliant schema."""
if not self.table_schema:
results = get_mysql_query_results(
self.db_credentials, self.database,
'describe {}'.format(self.table_name))
for result in results:
field_name = result[0].strip()
field_type = result[1].strip()
field_null = result[2].strip()
types_with_parentheses = [
'tinyint', 'smallint', 'int', 'bigint', 'datetime'
]
if any(_type in field_type
for _type in types_with_parentheses):
field_type = field_type.rsplit('(')[0]
elif field_type == 'longtext':
field_type = 'LONG VARCHAR'
elif field_type == 'double':
field_type = 'DOUBLE PRECISION'
if field_null == "NO":
field_type = field_type + " NOT NULL"
field_name = "\"{}\"".format(field_name)
self.table_schema.append((field_name, field_type))
return self.table_schema
def vertica_compliant_schema(self):
"""Transforms mysql table schema into a vertica compliant schema."""
if not self.table_schema:
results = get_mysql_query_results(self.db_credentials, self.database, 'describe {}'.format(self.table_name))
for result in results:
field_name = result[0].strip()
field_type = result[1].strip()
field_null = result[2].strip()
types_with_parentheses = ['tinyint', 'smallint', 'int', 'bigint', 'datetime']
if any(_type in field_type for _type in types_with_parentheses):
field_type = field_type.rsplit('(')[0]
elif field_type == 'longtext':
field_type = 'LONG VARCHAR'
elif field_type == 'double':
field_type = 'DOUBLE PRECISION'
if field_null == "NO":
field_type = field_type + " NOT NULL"
field_name = "\"{}\"".format(field_name)
self.table_schema.append((field_name, field_type))
return self.table_schema
def requires(self):
if not self.table_list:
results = get_mysql_query_results(self.db_credentials, self.database, 'show tables')
self.table_list = [result[0].strip() for result in results]
for table_name in self.table_list:
if not self.should_exclude_table(table_name):
yield LoadMysqlToVerticaTableTask(
credentials=self.credentials,
schema=self.schema,
db_credentials=self.db_credentials,
database=self.database,
warehouse_path=self.warehouse_path,
table_name=table_name,
overwrite=self.overwrite,
date=self.date,
)
def requires(self):
if not self.table_list:
results = get_mysql_query_results(self.db_credentials,
self.database, 'show tables')
self.table_list = [result[0].strip() for result in results]
for table_name in self.table_list:
if not self.should_exclude_table(table_name):
yield LoadMysqlToVerticaTableTask(
credentials=self.credentials,
schema=self.schema,
db_credentials=self.db_credentials,
database=self.database,
warehouse_path=self.warehouse_path,
table_name=table_name,
overwrite=self.overwrite,
date=self.date,
)
