def test_array_with_multiple_elements(sql_dialect):
"""
Exactly 1 view should be created with 4 dimensions, and one LATERAL FLATTEN expression
:return:
"""
g = Generator(column_name="data_column",
table_alias="data_table",
handle_null_values_in_sql=False,
primary_key=None,
sql_dialect=sql_dialect)
g.collect_all_paths(
data_object={
ORDERS_TABLE_NAME: [{
"id": 3
}, {
"info": 334
}, {
"email": "*****@*****.**"
}, {
"phone_number": 3344531679
}]
})
assert ORDERS_TABLE_NAME in g.dim_definitions
assert 4 == len(g.dim_definitions["orders"])
__test_lower_cases(g)
assert 1 == len(g.explore_joins)
def test_int_key(self):
"""
Int key in JSON should be ignored
:return:
"""
g = Generator()
g.collect_all_paths(current_dict={1: 2})
self.assertFalse(g.views_dimensions_expr)
self.assertFalse(g.explore_joins)
def test_empty(self):
"""
For empty JSON nothing should be created.
:return:
"""
g = Generator()
g.collect_all_paths(current_dict={})
self.assertFalse(g.views_dimensions_expr)
self.assertFalse(g.explore_joins)
def test_one_array(self):
"""
Simple test exactly 1 view should be created with 1 dimension, and one LATERAL FLATTEN expression
:return:
"""
g = Generator()
g.collect_all_paths(current_dict={"orders": [{"id": 3}, {"id": 334}]})
self.assertIn("orders", g.views_dimensions_expr)
self.assertEqual(1, len(g.views_dimensions_expr["orders"]))
self.assertIn("id", list(g.views_dimensions_expr["orders"])[0])
self.assertEqual(1, len(g.explore_joins))
def test_int_key():
"""
Int key in JSON should be ignored
:return:
"""
g = Generator(column_name="data_column",
table_alias="data_table",
handle_null_values_in_sql=False,
primary_key=None)
g.collect_all_paths(data_object={1: 2})
assert count_dims(g) == 0
assert not g.explore_joins
def test_replaces_nulls_values_in_json(json_data, prefix, suffix):
""""
the appropriate null handling code should be added to all columns in all_non_null_fields
"""
g = Generator(column_name="data_column",
table_alias="data_table",
handle_null_values_in_sql=True,
primary_key=None)
g.collect_all_paths(data_object={ORDERS_TABLE_NAME: json_data})
for column_def in g.dim_sql_definitions[ORDERS_TABLE_NAME].values():
assert column_def.startswith(prefix)
assert suffix in column_def
def test_empty():
"""
For empty JSON nothing should be created.
:return:
"""
g = Generator(column_name="data_column",
table_alias="data_table",
handle_null_values_in_sql=False,
primary_key=None)
g.collect_all_paths(data_object={})
assert count_dims(g) == 0
assert not g.explore_joins
def test_lower_cases_1():
g = Generator(column_name="data_column",
table_alias="data_table",
handle_null_values_in_sql=False,
primary_key=None)
g.collect_all_paths(data_object={
ORDERS_TABLE_NAME: [{
"ID": 3
}, {
"ID": 334
}],
"AMOUNT": 5654.3
})
__test_lower_cases(g)
def test_lower_cases_2(sql_dialect):
g = Generator(column_name="data_column",
table_alias="data_table",
handle_null_values_in_sql=False,
primary_key=None,
sql_dialect=sql_dialect)
g.collect_all_paths(
data_object={
ORDERS_TABLE_NAME: [{
"ID": 3
}, {
"ID": 334
}],
"AMOUNT": {
"value": 5654.3,
"CURRENCY": "EUR"
}
})
__test_lower_cases(g)
def test_one_array():
"""
Exactly 1 view should be created with 1 dimension, and one LATERAL FLATTEN expression
:return:
"""
g = Generator(column_name="data_column",
table_alias="data_table",
handle_null_values_in_sql=False,
primary_key=None)
g.collect_all_paths(
data_object={ORDERS_TABLE_NAME: [{
"id": 3
}, {
"id": 334
}]})
assert ORDERS_TABLE_NAME in g.dim_definitions
assert 1 == len(g.dim_definitions["orders"])
__test_lower_cases(g)
assert 1 == len(g.explore_joins)
def __init__(self, column_name=COLUMN_WITH_JSONS_DEFAULT, output_explore_file_name=EXPLORE_LKML_OUT_DEFAULT,
output_view_file_name=OUTPUT_VIEW_ML_OUT_DEFAULT, sql_table_name=TABLE_WITH_JSON_COLUMN_DEFAULT,
table_alias=TABLE_ALIAS_DEFAULT, handle_null_values_in_sql=HANDLE_NULL_VALUES_IN_SQL_DEFAULT,
primary_key=None):
"""
Init empty lists and ops counter.
"""
self.output_explore_file_name = output_explore_file_name or EXPLORE_LKML_OUT_DEFAULT
self.output_view_file_name = output_view_file_name or OUTPUT_VIEW_ML_OUT_DEFAULT
self.column_name = column_name or COLUMN_WITH_JSONS_DEFAULT
self.sql_table_name = sql_table_name or TABLE_WITH_JSON_COLUMN_DEFAULT
self.table_alias = get_formatted_var_name(table_alias or TABLE_ALIAS_DEFAULT)
self.handle_null_values_in_sql = handle_null_values_in_sql or HANDLE_NULL_VALUES_IN_SQL_DEFAULT
self.generator = Generator(column_name=self.column_name,
table_alias=self.table_alias,
handle_null_values_in_sql=self.handle_null_values_in_sql,
primary_key=primary_key)
self.sql_writer = SQLWriter(self.sql_table_name, self.table_alias)
self.looker_writer = LookerWriter(self.output_explore_file_name, self.output_view_file_name,
self.sql_table_name, self.table_alias)
def test_one_problematic_dim_name():
g = Generator(column_name="data_column",
table_alias="data_table",
handle_null_values_in_sql=False,
primary_key=None)
g.collect_all_paths(
data_object={
ORDERS_TABLE_NAME: [{
"aaaaId-ABC-DEF": 3
}, {
"abId-ABC--DEF": 334
}],
"zz": 5654.3
})
count = 0
dims = g.dim_definitions[ORDERS_TABLE_NAME]
for dim in dims:
if "aaaa_id_abc_def" in dim[dim.index("dimension"):dim.index("{")]:
count += 1
assert count == 1
def __init__(self,
column_name=COLUMN_WITH_JSONS_DEFAULT,
output_explore_file_name=EXPLORE_LKML_OUT_DEFAULT,
output_view_file_name=OUTPUT_VIEW_ML_OUT_DEFAULT,
sql_table_name=TABLE_WITH_JSON_COLUMN_DEFAULT,
table_alias=TABLE_ALIAS_DEFAULT,
handle_null_values_in_sql=HANDLE_NULL_VALUES_IN_SQL_DEFAULT,
primary_key=None,
sql_dialect=SQL_DIALECT_DEFAULT):
"""
Init empty lists and ops counter.
"""
self.output_explore_file_name = output_explore_file_name or EXPLORE_LKML_OUT_DEFAULT
self.output_view_file_name = output_view_file_name or OUTPUT_VIEW_ML_OUT_DEFAULT
self.column_name = column_name or COLUMN_WITH_JSONS_DEFAULT
self.sql_table_name = sql_table_name or TABLE_WITH_JSON_COLUMN_DEFAULT
self.table_alias = get_formatted_var_name(table_alias
or TABLE_ALIAS_DEFAULT)
self.handle_null_values_in_sql = handle_null_values_in_sql or HANDLE_NULL_VALUES_IN_SQL_DEFAULT
self.sql_dialect = sql_dialect or SQL_DIALECT_DEFAULT
if self.sql_dialect.lower() not in supported_dialects:
raise ValueError(
"SQL Dialect {} not supported. Dialects available: {}".format(
self.sql_dialect, ", ".join(supported_dialects)))
self.generator = Generator(
column_name=self.column_name,
table_alias=self.table_alias,
handle_null_values_in_sql=self.handle_null_values_in_sql,
sql_dialect=self.sql_dialect,
primary_key=primary_key)
self.sql_writer = SQLWriter(self.sql_table_name, self.table_alias,
self.sql_dialect)
self.looker_writer = LookerWriter(self.output_explore_file_name,
self.output_view_file_name,
self.sql_table_name,
self.table_alias)
def __init__(self,
column_name=COLUMN_WITH_JSONS_DEFAULT,
output_explore_file_name=EXPLORE_LKML_OUT_DEFAULT,
output_view_file_name=OUTPUT_VIEW_ML_OUT_DEFAULT,
sql_table_name=TABLE_WITH_JSON_COLUMN_DEFAULT,
table_alias=TABLE_ALIAS_DEFAULT):
"""
Init empty lists and ops counter.
"""
self.output_explore_file_name = output_explore_file_name if output_explore_file_name else EXPLORE_LKML_OUT_DEFAULT
self.output_view_file_name = output_view_file_name if output_view_file_name else OUTPUT_VIEW_ML_OUT_DEFAULT
self.column_name = column_name if column_name else COLUMN_WITH_JSONS_DEFAULT
self.sql_table_name = sql_table_name if sql_table_name else TABLE_WITH_JSON_COLUMN_DEFAULT
self.table_alias = table_alias if table_alias else TABLE_ALIAS_DEFAULT
self.generator = Generator(column_name=self.column_name,
table_alias=self.table_alias)
self.sql_writer = SQLWriter(self.sql_table_name, self.table_alias)
self.looker_writer = LookerWriter(self.output_explore_file_name,
self.output_view_file_name,
self.sql_table_name,
self.table_alias)
from j2v.generation.generator import Generator
from j2v.utils.config import generator_config
import argparse
import datetime
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--json_files", nargs=argparse.ONE_OR_MORE, type=str, default=[], )
parser.add_argument("--output_view", nargs=argparse.OPTIONAL, type=str,
default=generator_config['OUTPUT_VIEW_ML_OUT_DEFAULT'], )
parser.add_argument("--output_explore", nargs=argparse.OPTIONAL, type=str,
default=generator_config['EXPLORE_LKML_OUT_DEFAULT'], )
parser.add_argument("--columnn_name", nargs=argparse.OPTIONAL, type=str,
default=generator_config['COLUMN_WITH_JSONS_DEFAULT'], )
parser.add_argument("--sql_table_name", nargs=argparse.OPTIONAL, type=str,
default=generator_config['TABLE_WITH_JSON_COLUMN_DEFAULT'], )
args = parser.parse_args()
g = Generator(column_name=args.columnn_name, output_explore_file_name=args.output_explore,
output_view_file_name=args.output_view, sql_table_name=args.sql_table_name)
print("{date} Running the generator.\n\n".format(date=datetime.datetime.now()))
g.process_jsons(args.json_files)
print("\n\n{date} Finished.".format(date=datetime.datetime.now()))
