def test_exception_ddl():
# Create parse object instance
ddlparse = DdlParse()
# Do not set DDL
# Error : DDL is not specified
with pytest.raises(ValueError):
ddlparse.parse()
def test_exception_bq_data_type():
ddl = """
CREATE TABLE Sample_Table (
Col_01 NG_DATA_TYPE,
);
"""
# Parse DDL
table = DdlParse().parse(ddl)
# Error: Unknown data type
with pytest.raises(ValueError):
print(table.columns["col_01"].bigquery_data_type)
def extract_table(ddl):
t = DdlParse().parse(ddl, source_database=DdlParse.DATABASE.oracle)
schema = t.schema.lower()
tbl = t.name.lower()
if schema in table_result:
table_result[schema][tbl] = []
else:
table_result[schema] = {tbl: []}
for col in t.columns.values():
table_result[schema][tbl].append({
'colname': col.name,
'type': combine_type(col)
})
def get_obj(self, environment, database, table_name):
ddl = self.getValue('ddl')
if ddl is None:
imysql.conn(environment)
ddl = imysql.table(table_name, database).get_ddl()
if ddl is False:
return self.error(500)
else:
# 处理英文逗号冲突
ddl = ddl.replace(',\n', '-\n').replace(',',
'，').replace('-\n', ',\n')
match = re.search(r'comment=\'([\s\S]+)\'', ddl, re.I)
table_comment = match.group(1) if match is not None else ''
result = DdlParse().parse(ddl)
obj = iObject(table_name[table_name.find('_'):])
obj.set_comment(table_comment)
return (obj, result, ddl)
def test_bq_ddl(test_case):
# Get test data
data = TEST_DATA_DDL[test_case]
# Parse ddl
table = DdlParse().parse(data["source_ddl"])
# Check generate BigQuery DDL statements of DdlParseTable
assert table.to_bigquery_ddl() == textwrap.dedent(
data["bq_ddl"][DdlParse.NAME_CASE.original])
assert table.to_bigquery_ddl(
DdlParse.NAME_CASE.original) == textwrap.dedent(
data["bq_ddl"][DdlParse.NAME_CASE.original])
assert table.to_bigquery_ddl(DdlParse.NAME_CASE.lower) == textwrap.dedent(
data["bq_ddl"][DdlParse.NAME_CASE.lower])
assert table.to_bigquery_ddl(DdlParse.NAME_CASE.upper) == textwrap.dedent(
data["bq_ddl"][DdlParse.NAME_CASE.upper])
from ddlparse.ddlparse import DdlParse
import stringcase
import json
f = open("./vmdm_public.dmp", "r")
ddl_text=f.read()
ddls = ddl_text.split(";")
for index, ddl in enumerate(ddls):
if ddl.find("CREATE TABLE") != -1:
try:
table = DdlParse().parse(ddl=ddl)
logical_id = stringcase.alphanumcase(
stringcase.capitalcase(stringcase.camelcase(table.name))
)
table_name = table.name
columns = list(map(lambda col: {"Name":col.name.lower(),"Type":"string"}, table.columns.values()))
columns_json = json.dumps(columns, separators=(',', ':'))
print("{logical_id} {table_name} {columns_json}".format(logical_id = logical_id, table_name = table_name, columns_json = columns_json))
except Exception as e:
print(index)
print(value)
print(e)
def test_parse(test_case, parse_pattern):
# Get test ddl script
data = TEST_DATA[test_case]
# Create parse object instance
ddlparse = DdlParse()
# Set source database option
# Set source database option & Parse ddl
if parse_pattern == DDL_SET_PATTERN.method:
if data["database"] is not None:
table = ddlparse.parse(data["ddl"], data["database"])
else:
table = ddlparse.parse(data["ddl"])
else:
if data["database"] is not None:
ddlparse.source_database = data["database"]
ddlparse.ddl = data["ddl"]
table = ddlparse.parse()
# Check DDL
assert ddlparse.ddl == data["ddl"]
# Check source database option
assert ddlparse.source_database == data["database"]
# Check table
assert table.schema == data["table"]["schema"] if data["table"][
"schema"] is not None else table.schema is None
assert table.name == data["table"]["name"]
assert table.is_temp == data["table"]["temp"]
assert table.source_database == data["database"]
# Check columns
data_bq_field_lower = []
data_bq_field_upper = []
i = 0
for col in table.columns.values():
data_col = data["columns"][i]
assert col.name == data_col["name"]
assert col.data_type == data_col["type"]
assert col.length == data_col["length"] if data_col[
"length"] is not None else col.length is None
assert col.precision == data_col["length"] if data_col[
"length"] is not None else col.length is None
assert col.scale == data_col["scale"] if data_col[
"scale"] is not None else col.scale is None
assert col.array_dimensional == data_col["array_dimensional"]
assert col.not_null == data_col["not_null"]
assert col.primary_key == data_col["pk"]
assert col.unique == data_col["unique"]
assert col.constraint == data_col["constraint"]
assert col.source_database == data["database"]
data_bq_field = data["bq_field"][i]
assert col.to_bigquery_field() == data_bq_field
data_bq_field_lower.append(
re.sub(r'({"name": ")Col', r'\1col', data_bq_field))
assert col.to_bigquery_field(
col.NAME_CASE.lower) == data_bq_field_lower[-1]
data_bq_field_upper.append(
re.sub(r'({"name": ")Col', r'\1COL', data_bq_field))
assert col.to_bigquery_field(
col.NAME_CASE.upper) == data_bq_field_upper[-1]
assert col.bigquery_legacy_data_type == col.bigquery_data_type
assert col.bigquery_standard_data_type == data[
"bq_standard_data_type"][i]
i += 1
# Check BigQuery fields format of DdlParseColumnDict
assert table.columns.to_bigquery_fields() == "[{}]".format(",".join(
data["bq_field"]))
assert table.columns.to_bigquery_fields(
col.NAME_CASE.lower) == "[{}]".format(",".join(data_bq_field_lower))
assert table.columns.to_bigquery_fields(
col.NAME_CASE.upper) == "[{}]".format(",".join(data_bq_field_upper))
assert table.columns.source_database == data["database"]
# Check BigQuery fields format of DdlParseTable
assert table.columns.to_bigquery_fields() == table.to_bigquery_fields()
assert table.columns.to_bigquery_fields(
col.NAME_CASE.lower) == table.to_bigquery_fields(col.NAME_CASE.lower)
assert table.columns.to_bigquery_fields(
col.NAME_CASE.upper) == table.to_bigquery_fields(col.NAME_CASE.upper)