def test_array_with_multiple_elements(sql_dialect): """ Exactly 1 view should be created with 4 dimensions, and one LATERAL FLATTEN expression :return: """ g = Generator(column_name="data_column", table_alias="data_table", handle_null_values_in_sql=False, primary_key=None, sql_dialect=sql_dialect) g.collect_all_paths( data_object={ ORDERS_TABLE_NAME: [{ "id": 3 }, { "info": 334 }, { "email": "*****@*****.**" }, { "phone_number": 3344531679 }] }) assert ORDERS_TABLE_NAME in g.dim_definitions assert 4 == len(g.dim_definitions["orders"]) __test_lower_cases(g) assert 1 == len(g.explore_joins)
def test_int_key(self): """ Int key in JSON should be ignored :return: """ g = Generator() g.collect_all_paths(current_dict={1: 2}) self.assertFalse(g.views_dimensions_expr) self.assertFalse(g.explore_joins)
def test_empty(self): """ For empty JSON nothing should be created. :return: """ g = Generator() g.collect_all_paths(current_dict={}) self.assertFalse(g.views_dimensions_expr) self.assertFalse(g.explore_joins)
def test_one_array(self): """ Simple test exactly 1 view should be created with 1 dimension, and one LATERAL FLATTEN expression :return: """ g = Generator() g.collect_all_paths(current_dict={"orders": [{"id": 3}, {"id": 334}]}) self.assertIn("orders", g.views_dimensions_expr) self.assertEqual(1, len(g.views_dimensions_expr["orders"])) self.assertIn("id", list(g.views_dimensions_expr["orders"])[0]) self.assertEqual(1, len(g.explore_joins))
def test_int_key(): """ Int key in JSON should be ignored :return: """ g = Generator(column_name="data_column", table_alias="data_table", handle_null_values_in_sql=False, primary_key=None) g.collect_all_paths(data_object={1: 2}) assert count_dims(g) == 0 assert not g.explore_joins
def test_replaces_nulls_values_in_json(json_data, prefix, suffix): """" the appropriate null handling code should be added to all columns in all_non_null_fields """ g = Generator(column_name="data_column", table_alias="data_table", handle_null_values_in_sql=True, primary_key=None) g.collect_all_paths(data_object={ORDERS_TABLE_NAME: json_data}) for column_def in g.dim_sql_definitions[ORDERS_TABLE_NAME].values(): assert column_def.startswith(prefix) assert suffix in column_def
def test_empty(): """ For empty JSON nothing should be created. :return: """ g = Generator(column_name="data_column", table_alias="data_table", handle_null_values_in_sql=False, primary_key=None) g.collect_all_paths(data_object={}) assert count_dims(g) == 0 assert not g.explore_joins
def test_lower_cases_1(): g = Generator(column_name="data_column", table_alias="data_table", handle_null_values_in_sql=False, primary_key=None) g.collect_all_paths(data_object={ ORDERS_TABLE_NAME: [{ "ID": 3 }, { "ID": 334 }], "AMOUNT": 5654.3 }) __test_lower_cases(g)
def test_lower_cases_2(sql_dialect): g = Generator(column_name="data_column", table_alias="data_table", handle_null_values_in_sql=False, primary_key=None, sql_dialect=sql_dialect) g.collect_all_paths( data_object={ ORDERS_TABLE_NAME: [{ "ID": 3 }, { "ID": 334 }], "AMOUNT": { "value": 5654.3, "CURRENCY": "EUR" } }) __test_lower_cases(g)
def test_one_array(): """ Exactly 1 view should be created with 1 dimension, and one LATERAL FLATTEN expression :return: """ g = Generator(column_name="data_column", table_alias="data_table", handle_null_values_in_sql=False, primary_key=None) g.collect_all_paths( data_object={ORDERS_TABLE_NAME: [{ "id": 3 }, { "id": 334 }]}) assert ORDERS_TABLE_NAME in g.dim_definitions assert 1 == len(g.dim_definitions["orders"]) __test_lower_cases(g) assert 1 == len(g.explore_joins)
def __init__(self, column_name=COLUMN_WITH_JSONS_DEFAULT, output_explore_file_name=EXPLORE_LKML_OUT_DEFAULT, output_view_file_name=OUTPUT_VIEW_ML_OUT_DEFAULT, sql_table_name=TABLE_WITH_JSON_COLUMN_DEFAULT, table_alias=TABLE_ALIAS_DEFAULT, handle_null_values_in_sql=HANDLE_NULL_VALUES_IN_SQL_DEFAULT, primary_key=None): """ Init empty lists and ops counter. """ self.output_explore_file_name = output_explore_file_name or EXPLORE_LKML_OUT_DEFAULT self.output_view_file_name = output_view_file_name or OUTPUT_VIEW_ML_OUT_DEFAULT self.column_name = column_name or COLUMN_WITH_JSONS_DEFAULT self.sql_table_name = sql_table_name or TABLE_WITH_JSON_COLUMN_DEFAULT self.table_alias = get_formatted_var_name(table_alias or TABLE_ALIAS_DEFAULT) self.handle_null_values_in_sql = handle_null_values_in_sql or HANDLE_NULL_VALUES_IN_SQL_DEFAULT self.generator = Generator(column_name=self.column_name, table_alias=self.table_alias, handle_null_values_in_sql=self.handle_null_values_in_sql, primary_key=primary_key) self.sql_writer = SQLWriter(self.sql_table_name, self.table_alias) self.looker_writer = LookerWriter(self.output_explore_file_name, self.output_view_file_name, self.sql_table_name, self.table_alias)
def test_one_problematic_dim_name(): g = Generator(column_name="data_column", table_alias="data_table", handle_null_values_in_sql=False, primary_key=None) g.collect_all_paths( data_object={ ORDERS_TABLE_NAME: [{ "aaaaId-ABC-DEF": 3 }, { "abId-ABC--DEF": 334 }], "zz": 5654.3 }) count = 0 dims = g.dim_definitions[ORDERS_TABLE_NAME] for dim in dims: if "aaaa_id_abc_def" in dim[dim.index("dimension"):dim.index("{")]: count += 1 assert count == 1
def __init__(self, column_name=COLUMN_WITH_JSONS_DEFAULT, output_explore_file_name=EXPLORE_LKML_OUT_DEFAULT, output_view_file_name=OUTPUT_VIEW_ML_OUT_DEFAULT, sql_table_name=TABLE_WITH_JSON_COLUMN_DEFAULT, table_alias=TABLE_ALIAS_DEFAULT, handle_null_values_in_sql=HANDLE_NULL_VALUES_IN_SQL_DEFAULT, primary_key=None, sql_dialect=SQL_DIALECT_DEFAULT): """ Init empty lists and ops counter. """ self.output_explore_file_name = output_explore_file_name or EXPLORE_LKML_OUT_DEFAULT self.output_view_file_name = output_view_file_name or OUTPUT_VIEW_ML_OUT_DEFAULT self.column_name = column_name or COLUMN_WITH_JSONS_DEFAULT self.sql_table_name = sql_table_name or TABLE_WITH_JSON_COLUMN_DEFAULT self.table_alias = get_formatted_var_name(table_alias or TABLE_ALIAS_DEFAULT) self.handle_null_values_in_sql = handle_null_values_in_sql or HANDLE_NULL_VALUES_IN_SQL_DEFAULT self.sql_dialect = sql_dialect or SQL_DIALECT_DEFAULT if self.sql_dialect.lower() not in supported_dialects: raise ValueError( "SQL Dialect {} not supported. Dialects available: {}".format( self.sql_dialect, ", ".join(supported_dialects))) self.generator = Generator( column_name=self.column_name, table_alias=self.table_alias, handle_null_values_in_sql=self.handle_null_values_in_sql, sql_dialect=self.sql_dialect, primary_key=primary_key) self.sql_writer = SQLWriter(self.sql_table_name, self.table_alias, self.sql_dialect) self.looker_writer = LookerWriter(self.output_explore_file_name, self.output_view_file_name, self.sql_table_name, self.table_alias)
def __init__(self, column_name=COLUMN_WITH_JSONS_DEFAULT, output_explore_file_name=EXPLORE_LKML_OUT_DEFAULT, output_view_file_name=OUTPUT_VIEW_ML_OUT_DEFAULT, sql_table_name=TABLE_WITH_JSON_COLUMN_DEFAULT, table_alias=TABLE_ALIAS_DEFAULT): """ Init empty lists and ops counter. """ self.output_explore_file_name = output_explore_file_name if output_explore_file_name else EXPLORE_LKML_OUT_DEFAULT self.output_view_file_name = output_view_file_name if output_view_file_name else OUTPUT_VIEW_ML_OUT_DEFAULT self.column_name = column_name if column_name else COLUMN_WITH_JSONS_DEFAULT self.sql_table_name = sql_table_name if sql_table_name else TABLE_WITH_JSON_COLUMN_DEFAULT self.table_alias = table_alias if table_alias else TABLE_ALIAS_DEFAULT self.generator = Generator(column_name=self.column_name, table_alias=self.table_alias) self.sql_writer = SQLWriter(self.sql_table_name, self.table_alias) self.looker_writer = LookerWriter(self.output_explore_file_name, self.output_view_file_name, self.sql_table_name, self.table_alias)
from j2v.generation.generator import Generator from j2v.utils.config import generator_config import argparse import datetime if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("--json_files", nargs=argparse.ONE_OR_MORE, type=str, default=[], ) parser.add_argument("--output_view", nargs=argparse.OPTIONAL, type=str, default=generator_config['OUTPUT_VIEW_ML_OUT_DEFAULT'], ) parser.add_argument("--output_explore", nargs=argparse.OPTIONAL, type=str, default=generator_config['EXPLORE_LKML_OUT_DEFAULT'], ) parser.add_argument("--columnn_name", nargs=argparse.OPTIONAL, type=str, default=generator_config['COLUMN_WITH_JSONS_DEFAULT'], ) parser.add_argument("--sql_table_name", nargs=argparse.OPTIONAL, type=str, default=generator_config['TABLE_WITH_JSON_COLUMN_DEFAULT'], ) args = parser.parse_args() g = Generator(column_name=args.columnn_name, output_explore_file_name=args.output_explore, output_view_file_name=args.output_view, sql_table_name=args.sql_table_name) print("{date} Running the generator.\n\n".format(date=datetime.datetime.now())) g.process_jsons(args.json_files) print("\n\n{date} Finished.".format(date=datetime.datetime.now()))