def generate_conf(guid_batch, phase_number, load_type, tenant_code, target_schema): """ Return all needed configuration information :param guid_batch: the guid for the batch :param phase_number: the current number of the phase :param load_type: type of load. ie. assessment :param tenant_code: the tenants 2 letter code :return: A dictionary of the config details """ db_params_tuple = get_db_connection_params(udl2_conf['udl2_db_conn']['url']) conf = { # add guid_batch from msg mk.GUID_BATCH: guid_batch, # source schema mk.SOURCE_DB_SCHEMA: udl2_conf['udl2_db_conn']['db_schema'], # source database setting mk.SOURCE_DB_DRIVER: db_params_tuple[0], mk.SOURCE_DB_USER: db_params_tuple[1], mk.SOURCE_DB_PASSWORD: db_params_tuple[2], mk.SOURCE_DB_HOST: db_params_tuple[3], mk.SOURCE_DB_PORT: db_params_tuple[4], mk.SOURCE_DB_NAME: db_params_tuple[5], mk.SOURCE_DB_TABLE: Constants.UDL2_JSON_INTEGRATION_TABLE(load_type), mk.TARGET_DB_SCHEMA: target_schema, mk.REF_TABLE: Constants.UDL2_REF_MAPPING_TABLE(load_type), mk.PHASE: int(phase_number), mk.LOAD_TYPE: load_type, mk.TENANT_NAME: tenant_code, mk.PROD_DB_SCHEMA: udl2_conf['prod_db_conn'][tenant_code]['db_schema'] } return conf
def load_stored_proc(): ''' Generate and load the stored procedures to be used for transformations and validations into the database. @param udl2_conf: The configuration dictionary for ''' populate_stored_proc(Constants.UDL2_REF_MAPPING_TABLE(Constants.LOAD_TYPE_ASSESSMENT), Constants.UDL2_REF_MAPPING_TABLE(Constants.LOAD_TYPE_STUDENT_REGISTRATION))
def load_reference_data(): ''' load the reference data into the reference tables @param udl2_conf: The configuration dictionary for ''' asmt_ref_table_info = ref_table_data.ref_table_conf populate_ref_column_map(asmt_ref_table_info, Constants.UDL2_REF_MAPPING_TABLE(Constants.LOAD_TYPE_ASSESSMENT)) sr_ref_table_info = sr_ref_table_data.ref_table_conf populate_ref_column_map(sr_ref_table_info, Constants.UDL2_REF_MAPPING_TABLE(Constants.LOAD_TYPE_STUDENT_REGISTRATION))
def get_content_validator_conf(guid_batch, load_type): udl_db_conn = udl2_conf.get(Constants.UDL2_DB_CONN) conf = { mk.SOURCE_DB_SCHEMA: udl_db_conn.get(Constants.DB_SCHEMA), mk.ASMT_TABLE: Constants.UDL2_JSON_INTEGRATION_TABLE(load_type), mk.ASMT_OUTCOME_TABLE: Constants.UDL2_STAGING_TABLE(load_type), mk.GUID_BATCH: guid_batch, mk.LOAD_TYPE: load_type } return conf
def test_load_stage_to_int_student_registration(self): guid_batch = str(uuid4()) load_type = Constants.LOAD_TYPE_STUDENT_REGISTRATION conf = self.generate_conf_for_moving_from_stg_to_int(guid_batch, load_type) self.udl2_conf['guid_batch'] = guid_batch self.load_file_to_stage(os.path.join('student_registration_data', 'test_stu_reg_without_headers.csv'), os.path.join('student_registration_data', 'test_stu_reg_header.csv'), load_type, Constants.UDL2_STAGING_TABLE(load_type), guid_batch) move_data_from_staging_to_integration(conf) postloading_total = self.postloading_count(Constants.UDL2_INTEGRATION_TABLE(load_type)) self.assertEqual(10, postloading_total)
def init_udl_tenant_sequences(udl2_conf): # Create and sync sequence for each tenant on udl database if it doesn't exist with get_udl_connection() as udl_conn: all_tenants = udl2_conf.get(PRODUCTION_NAMESPACE) udl_schema_name = udl2_conf.get(UDL_NAMESPACE).get(Constants.DB_SCHEMA) # dict to keep track of tenant sequence values for each tenant defined in the ini all_tenant_sequences = {} for tenant in all_tenants: tenant_seq_name = Constants.TENANT_SEQUENCE_NAME(tenant) tenant_schema_name = all_tenants.get(tenant).get(Constants.DB_SCHEMA) # unique identifier for each tenant key = all_tenants.get(tenant).get(Constants.URL) + ':' + tenant_schema_name # check if we have already visited the tenant prod schema if not key in all_tenant_sequences: with get_prod_connection(tenant) as prod_conn: prod_seq_result = prod_conn.execute(text("select nextval(\'{schema_name}.{seq_name} \')". format(schema_name=tenant_schema_name, seq_name=Constants.SEQUENCE_NAME))) all_tenant_sequences[key] = prod_seq_result.fetchone()[0] # check if the global tenant sequence exists in udl database if not sequence_exists(udl_conn, tenant_seq_name): # create sequence if does not exist udl_conn.execute(CreateSequence(Sequence(name=tenant_seq_name, increment=1))) # update and set the current val for the tenant sequence udl_conn.execute(text("select setval(\'{schema_name}.{seq_name} \', {value}, {called})". format(schema_name=udl_schema_name, seq_name=tenant_seq_name, value=all_tenant_sequences[key], called=True)))
def import_via_fdw(conn, stg_columns, csv_table_columns, transformation_rules, apply_rules, staging_schema, staging_table, csv_schema, csv_table, start_seq, tenant_name): ''' Load data from foreign table to staging table ''' # create sequence name, use table_name and a random number combination. # This sequence is used for column src_file_rec_num seq_name = (csv_table + '_' + str(start_seq)).lower() global_tenant_seq_name = Constants.TENANT_SEQUENCE_NAME(tenant_name) # query 1 -- create query to create sequence create_sequence = queries.create_sequence_query(staging_schema, seq_name, start_seq) # query 2 -- create query to load data from fdw to staging table insert_into_staging_table = queries.create_inserting_into_staging_query( stg_columns, apply_rules, csv_table_columns, staging_schema, staging_table, csv_schema, csv_table, seq_name, global_tenant_seq_name, transformation_rules) # query 3 -- create query to drop sequence drop_sequence = queries.drop_sequence_query(staging_schema, seq_name) # logger.debug('@@@@@@@', insert_into_staging_table) # execute 3 queries in order execute_udl_queries( conn, [create_sequence, insert_into_staging_table, drop_sequence], 'Exception in loading data -- ', 'file_loader', 'import_via_fdw')
def get_source_target_column_values_from_ref_column_mapping(source_table, load_type): """ return all data for source_column from ref_column_mapping based on source_table value passed :param: source_table: source_table values to query for :return: A set containing all the columns expected in a source csv file """ with get_udl_connection() as conn: table_meta = conn.get_table(Constants.UDL2_REF_MAPPING_TABLE(load_type)) select_object = select([table_meta]).where(table_meta.c.source_table == source_table) return [(row['target_column'], row['source_column']) for row in conn.execute(select_object)]
def generate_conf_for_loading(file_to_load, start_seq, load_type, header_file_path, guid_batch, tenant_name): csv_table = extract_file_name(file_to_load) conf = { mk.FILE_TO_LOAD: file_to_load, mk.ROW_START: start_seq, mk.HEADERS: header_file_path, mk.CSV_SCHEMA: udl2_conf['udl2_db_conn']['db_schema'], mk.CSV_TABLE: csv_table, mk.FDW_SERVER: Constants.UDL2_FDW_SERVER, mk.TARGET_DB_SCHEMA: udl2_conf['udl2_db_conn']['db_schema'], mk.TARGET_DB_TABLE: Constants.UDL2_STAGING_TABLE(load_type), mk.APPLY_RULES: True, mk.REF_TABLE: Constants.UDL2_REF_MAPPING_TABLE(load_type), mk.CSV_LZ_TABLE: Constants.UDL2_CSV_LZ_TABLE, mk.GUID_BATCH: guid_batch, mk.TENANT_NAME: tenant_name } return conf
def task(msg): start_time = datetime.datetime.now() logger.info( "LOAD_SR_INTEGRATION_TO_TARGET: Migrating data from SR integration tables to target tables." ) guid_batch = msg[mk.GUID_BATCH] load_type = msg[mk.LOAD_TYPE] source_tables = [ Constants.UDL2_INTEGRATION_TABLE(load_type), Constants.UDL2_JSON_INTEGRATION_TABLE(load_type) ] target_table = Constants.SR_TARGET_TABLE target_schema = msg[ mk.TARGET_DB_SCHEMA] if mk.TARGET_DB_SCHEMA in msg else msg[ mk.GUID_BATCH] conf = generate_conf(guid_batch, msg[mk.PHASE], load_type, msg[mk.TENANT_NAME], target_schema) affected_rows = move_data_from_int_tables_to_target_table( conf, task.name, source_tables, target_table) end_time = datetime.datetime.now() # benchmark benchmark = BatchTableBenchmark(guid_batch, load_type, task.name, start_time, end_time, task_id=str(task.request.id), working_schema="", size_records=affected_rows[0], tenant=msg[mk.TENANT_NAME]) benchmark.record_benchmark() notification_data = {mk.TOTAL_ROWS_LOADED: affected_rows[0]} merge_to_udl2stat_notification(guid_batch, notification_data) outgoing_msg = {} outgoing_msg.update(msg) outgoing_msg.update(notification_data) return outgoing_msg
def load_config(self, type): if type == 'assessment': self.conf[mk.TARGET_DB_TABLE] = 'stg_sbac_asmt_outcome' self.conf[mk.REF_TABLE] = Constants.UDL2_REF_MAPPING_TABLE( 'assessment') self.conf[mk.CSV_TABLE] = 'test_csv_table' self.conf[mk.TENANT_NAME] = 'cat' self.conf[mk.FILE_TO_LOAD] = self.get_csv_file( 'test_file_realdata.csv') self.conf[mk.HEADERS] = self.get_csv_file('test_file_headers.csv') elif type == 'studentregistration': self.conf[mk.TARGET_DB_TABLE] = 'stg_sbac_stu_reg' self.conf[mk.REF_TABLE] = Constants.UDL2_REF_MAPPING_TABLE( 'studentregistration') self.conf[mk.TENANT_NAME] = 'cat' self.conf[mk.CSV_TABLE] = 'test_stu_reg_csv_table' self.conf[mk.FILE_TO_LOAD] = self.get_csv_file( 'student_registration_data/test_sample_student_reg.csv') self.conf[mk.HEADERS] = self.get_csv_file( 'student_registration_data/test_stu_reg_header.csv')
def get_assessment_type(json_file_dir): """ Get the assessment type for the UDL job from the json file @param json_file_dir: A directory that houses the json file @return: UDL job assessment type @rtype: string """ assessment_types = Constants.ASSESSMENT_TYPES() assessment_type = get_value_from_json(json_file_dir, Constants.ASSESSMENT_TYPE_KEY) if assessment_type not in assessment_types: raise ValueError('No valid load type specified in json file --') return assessment_type
def verify_json_load(self, load_type, conf, columns, guid): load_json(conf) sr_int_table = self.udl2_conn.get_table( Constants.UDL2_JSON_INTEGRATION_TABLE(load_type)) query = select(['*'], sr_int_table.c.guid_batch == guid) result = self.udl2_conn.execute(query).fetchall() for row in result: self.assertEqual(len(row), len(columns), 'Unexpected number of columns') for column in columns: self.assertTrue(row[column], 'Expected column does not have data')
def generate_config(self, load_type, file, guid): results = sfv_util.get_source_target_column_values_from_ref_column_mapping( Constants.UDL2_JSON_LZ_TABLE, load_type) conf = { mk.GUID_BATCH: guid, mk.FILE_TO_LOAD: file, mk.MAPPINGS: dict([(row[0], row[1].split('.')) for row in results]), mk.TARGET_DB_TABLE: Constants.UDL2_JSON_INTEGRATION_TABLE(load_type), mk.TARGET_DB_SCHEMA: udl2_conf['udl2_db_conn']['db_schema'], mk.TENANT_NAME: 'cat' } return conf
def generate_conf_for_moving_from_stg_to_int(self, guid_batch, load_type): db_params_tuple = get_db_connection_params(self.udl2_conf['udl2_db_conn']['url']) conf = { mk.GUID_BATCH: guid_batch, # source database setting mk.SOURCE_DB_DRIVER: db_params_tuple[0], mk.SOURCE_DB_USER: db_params_tuple[1], mk.SOURCE_DB_PASSWORD: db_params_tuple[2], mk.SOURCE_DB_HOST: db_params_tuple[3], mk.SOURCE_DB_PORT: db_params_tuple[4], mk.SOURCE_DB_NAME: db_params_tuple[5], mk.SOURCE_DB_SCHEMA: self.udl2_conf['udl2_db_conn']['db_schema'], mk.SOURCE_DB_TABLE: Constants.UDL2_STAGING_TABLE(load_type), # target database setting mk.TARGET_DB_SCHEMA: self.udl2_conf['udl2_db_conn']['db_schema'], mk.TARGET_DB_TABLE: Constants.UDL2_INTEGRATION_TABLE(load_type), mk.REF_TABLE: Constants.UDL2_REF_MAPPING_TABLE(load_type), mk.ERROR_DB_SCHEMA: self.udl2_conf['udl2_db_conn']['db_schema'], mk.ERR_LIST_TABLE: Constants.UDL2_ERR_LIST_TABLE } return conf
def generate_conf_for_loading(json_file, guid_batch, load_type, tenant_name): ''' takes the msg and pulls out the relevant parameters to pass the method that loads the json ''' results = sfv_util.get_source_target_column_values_from_ref_column_mapping( Constants.UDL2_JSON_LZ_TABLE, load_type) conf = { mk.FILE_TO_LOAD: json_file, mk.MAPPINGS: dict([(row[0], row[1].split('.')) for row in results]), mk.TARGET_DB_SCHEMA: udl2_conf['udl2_db_conn']['db_schema'], mk.TARGET_DB_TABLE: Constants.UDL2_JSON_INTEGRATION_TABLE(load_type), mk.GUID_BATCH: guid_batch, mk.TENANT_NAME: tenant_name } return conf
def load_file_to_stage(self, data_file, header_file, load_type, staging_table, guid): # file contain 30 rows conf = { mk.FILE_TO_LOAD: os.path.join(self.data_dir, data_file), mk.HEADERS: os.path.join(self.data_dir, header_file), mk.CSV_TABLE: 'csv_table_for_file_loader', mk.CSV_SCHEMA: self.udl2_conf['udl2_db_conn']['db_schema'], mk.REF_TABLE: Constants.UDL2_REF_MAPPING_TABLE(load_type), mk.CSV_LZ_TABLE: Constants.UDL2_CSV_LZ_TABLE, mk.FDW_SERVER: 'udl2_fdw_server', mk.TARGET_DB_SCHEMA: self.udl2_conf['udl2_db_conn']['db_schema'], mk.TARGET_DB_TABLE: staging_table, mk.APPLY_RULES: False, mk.ROW_START: 10, mk.GUID_BATCH: guid, mk.TENANT_NAME: 'cat' } load_file(conf) with get_udl_connection() as conn: _table = conn.get_table(staging_table) update_stmt = update(_table).values(record_sid=1000 + _table.c.src_file_rec_num - 1).\ where(_table.c.guid_batch == guid) conn.execute(update_stmt)
def setUp(self): try: config_path = dict(os.environ)['UDL2_CONF'] except Exception: config_path = UDL2_DEFAULT_CONFIG_PATH_FILE conf_tup = read_ini_file(config_path) udl2_conf = conf_tup[0] initialize_db_udl(udl2_conf) self.ref_schema = udl2_conf['udl2_db_conn']['db_schema'] self.ref_table_name = Constants.UDL2_REF_MAPPING_TABLE( Constants.LOAD_TYPE_ASSESSMENT) # Testable Rules self.rule_names = transform_rules.keys() self.rule_conf = transform_rules self.rule_list = transformation_code_generator.generate_transformations( self.rule_names, rule_conf=self.rule_conf) self.testable_rules = [] for rule in self.rule_list: self.testable_rules.append(rule[0]) test_rows = [] for rule in self.testable_rules: ins_dict = { 'phase': -999, 'source_table': 'ftest_table', 'source_column': 'ftest_column', 'target_table': 'ftest_table1', 'target_column': 'ftest_column1', 'transformation_rule': rule, } test_rows.append(ins_dict) with get_udl_connection() as conn: self.ref_table = conn.get_table(self.ref_table_name) conn.execute(self.ref_table.insert(test_rows))
def test_all_lambda_constants(self): self.assertEqual(len(Constants.LOAD_TYPES()), 2) self.assertEqual(Constants.LOAD_TYPES(), [ Constants.LOAD_TYPE_ASSESSMENT, Constants.LOAD_TYPE_STUDENT_REGISTRATION ]) self.assertEqual(len(Constants.ASSESSMENT_TYPES()), 3) self.assertEqual(Constants.ASSESSMENT_TYPES(), [ Constants.ASSESSMENT_TYPE_SUMMATIVE, Constants.ASSESSMENT_TYPE_INTERIM_COMPREHENSIVE, Constants.ASSESSMENT_TYPE_INTERIM_ASSESSMENT_BLOCKS ]) self.assertEqual( Constants.UDL2_STAGING_TABLE(Constants.LOAD_TYPE_ASSESSMENT), Constants.STG_ASMT_OUT_TABLE) self.assertEqual( Constants.UDL2_STAGING_TABLE( Constants.LOAD_TYPE_STUDENT_REGISTRATION), Constants.STG_SR_TABLE) self.assertEqual( Constants.UDL2_INTEGRATION_TABLE(Constants.LOAD_TYPE_ASSESSMENT), Constants.INT_ASMT_OUT_TABLE) self.assertEqual( Constants.UDL2_INTEGRATION_TABLE( Constants.LOAD_TYPE_STUDENT_REGISTRATION), Constants.INT_SR_TABLE) self.assertEqual( Constants.UDL2_JSON_INTEGRATION_TABLE( Constants.LOAD_TYPE_ASSESSMENT), Constants.INT_ASMT_TABLE) self.assertEqual( Constants.UDL2_JSON_INTEGRATION_TABLE( Constants.LOAD_TYPE_STUDENT_REGISTRATION), Constants.INT_SR_META_TABLE) self.assertEqual( Constants.UDL2_REF_MAPPING_TABLE(Constants.LOAD_TYPE_ASSESSMENT), Constants.ASMT_REF_TABLE) self.assertEqual( Constants.UDL2_REF_MAPPING_TABLE( Constants.LOAD_TYPE_STUDENT_REGISTRATION), Constants.SR_REF_TABLE) self.assertEqual(Constants.TENANT_SEQUENCE_NAME('cat'), Constants.SEQUENCE_NAME + '_' + 'cat') self.assertEqual(Constants.TENANT_SEQUENCE_NAME(''), None) self.assertEqual(Constants.TENANT_SEQUENCE_NAME(None), None)
def test_get_column_mapping_from_stg_to_int(self): expected_target_columns = ['name_state', 'code_state', 'guid_district', 'name_district', 'guid_school', 'name_school', 'guid_student', 'external_ssid_student', 'name_student_first', 'name_student_middle', 'name_student_last', 'birthdate_student', 'sex_student', 'grade_enrolled', 'dmg_eth_hsp', 'dmg_eth_ami', 'dmg_eth_asn', 'dmg_eth_blk', 'dmg_eth_pcf', 'dmg_eth_wht', 'dmg_multi_race', 'dmg_prg_iep', 'dmg_prg_lep', 'dmg_prg_504', 'dmg_sts_ecd', 'dmg_sts_mig', 'code_language', 'eng_prof_lvl', 'us_school_entry_date', 'lep_entry_date', 'lep_exit_date', 't3_program_type', 'prim_disability_type', 'created_date', 'guid_batch'] expected_source_columns_with_tran_rule = ['substr("A".name_state, 1, 50)', 'substr("A".code_state, 1, 2)', 'substr("A".guid_district, 1, 40)', 'substr("A".name_district, 1, 60)', 'substr("A".guid_school, 1, 40)', 'substr("A".name_school, 1, 60)', 'substr("A".guid_student, 1, 40)', 'substr("A".external_ssid_student, 1, 40)', 'substr("A".name_student_first, 1, 35)', 'substr("A".name_student_middle, 1, 35)', 'substr("A".name_student_last, 1, 35)', 'substr("A".birthdate_student, 1, 10)', 'substr("A".sex_student, 1, 10)', 'substr("A".grade_enrolled, 1, 2)', 'case "A".dmg_eth_hsp when \'\' then null else cast("A".dmg_eth_hsp as bool) end', 'case "A".dmg_eth_ami when \'\' then null else cast("A".dmg_eth_ami as bool) end', 'case "A".dmg_eth_asn when \'\' then null else cast("A".dmg_eth_asn as bool) end', 'case "A".dmg_eth_blk when \'\' then null else cast("A".dmg_eth_blk as bool) end', 'case "A".dmg_eth_pcf when \'\' then null else cast("A".dmg_eth_pcf as bool) end', 'case "A".dmg_eth_wht when \'\' then null else cast("A".dmg_eth_wht as bool) end', 'case "A".dmg_multi_race when \'\' then null else cast("A".dmg_multi_race as bool) end', 'case "A".dmg_prg_iep when \'\' then null else cast("A".dmg_prg_iep as bool) end', 'case "A".dmg_prg_lep when \'\' then null else cast("A".dmg_prg_lep as bool) end', 'case "A".dmg_prg_504 when \'\' then null else cast("A".dmg_prg_504 as bool) end', 'case "A".dmg_sts_ecd when \'\' then null else cast("A".dmg_sts_ecd as bool) end', 'case "A".dmg_sts_mig when \'\' then null else cast("A".dmg_sts_mig as bool) end', 'substr("A".code_language, 1, 3)', 'substr("A".eng_prof_lvl, 1, 20)', 'substr("A".us_school_entry_date, 1, 10)', 'substr("A".lep_entry_date, 1, 10)', 'substr("A".lep_exit_date, 1, 10)', 'substr("A".t3_program_type, 1, 27)', 'substr("A".prim_disability_type, 1, 3)', '"A".created_date', '"A".guid_batch'] with get_udl_connection() as conn: target_columns, source_columns_with_tran_rule = get_column_mapping_from_stg_to_int(conn, Constants.UDL2_REF_MAPPING_TABLE(Constants.LOAD_TYPE_STUDENT_REGISTRATION), 'stg_sbac_stu_reg', 'int_sbac_stu_reg') self.assertEqual(expected_target_columns, target_columns) self.assertEqual(expected_source_columns_with_tran_rule, source_columns_with_tran_rule)
def load_to_table(data_dict, guid_batch, int_table, tenant_name, udl_schema): ''' Load the table into the proper table @param data_dict: the dictionary containing the data to be loaded @param guid_batch: the id for the batch @param int_table: the name of the integration table @param tenant_name: name of the tenant @param udl_schema: udl schema name ''' # Create sqlalchemy connection and get table information from sqlalchemy ref_column_mapping_columns = {} with get_udl_connection() as conn: data_dict[mk.GUID_BATCH] = guid_batch data_dict = fix_empty_strings(data_dict) ref_table = conn.get_table('ref_column_mapping') s_int_table = conn.get_table(int_table) column_mapping_query = select( [ref_table.c.target_column, ref_table.c.stored_proc_name], from_obj=ref_table).where( and_(ref_table.c.source_table == 'lz_json', ref_table.c.target_table == int_table)) results = conn.get_result(column_mapping_query) for result in results: target_column = result['target_column'] stored_proc_name = result['stored_proc_name'] value = data_dict.get(target_column) if value: if stored_proc_name: if stored_proc_name.startswith('sp_'): ref_column_mapping_columns[ target_column] = stored_proc_name + '(' + QuotedString( value if type(value) is str else str(value) ).getquoted().decode('utf-8') + ')' else: format_value = dict() format_value['value'] = QuotedString( value if type(value) is str else str(value) ).getquoted().decode('utf-8') if s_int_table.c[target_column].type.python_type is str: format_value['length'] = s_int_table.c[ target_column].type.length ref_column_mapping_columns[ target_column] = stored_proc_name.format( **format_value) continue ref_column_mapping_columns[target_column] = value record_sid = 'nextval(\'{schema_name}.{tenant_sequence_name}\')'.\ format(schema_name=udl_schema, tenant_sequence_name=Constants.TENANT_SEQUENCE_NAME(tenant_name)) from_select_column_names = ['record_sid'] from_select_select_values = [record_sid] for column in s_int_table.c: value = data_dict.get(column.name) if value is not None: from_select_column_names.append(column.name) from_select_select_values.append( ref_column_mapping_columns.get( column.name, QuotedString(value if type(value) is str else str( value)).getquoted().decode('utf-8'))) insert_into_int_table = s_int_table.insert().from_select( from_select_column_names, select(from_select_select_values)) # create insert statement and execute affected_row = db_util.execute_udl_queries( conn, [insert_into_int_table], 'Exception in loading json data -- ', 'json_loader', 'load_to_table') return affected_row[0]
# Consortium as of August 1, 2014 are granted a worldwide, non-exclusive, fully # paid-up, royalty-free, perpetual license, to access, use, execute, reproduce, # display, distribute, perform and create derivative works of the software # included in the Reporting Platform, including the source code to such software. # This license includes the right to grant sublicenses by such consortium members # to third party vendors solely for the purpose of performing services on behalf # of such consortium member educational agencies. import logging from edudl2.json_util.json_util import get_value_from_json from edudl2.udl2.constants import Constants __author__ = 'tshewchuk' logger = logging.getLogger(__name__) load_types = Constants.LOAD_TYPES() def get_load_type(json_file_dir): """ Get the load type for this UDL job from the json file @param json_file_dir: A directory that houses the json file @return: UDL job load type @rtype: string """ load_type = get_value_from_json(json_file_dir, Constants.LOAD_TYPE_KEY).lower() if load_type not in load_types: raise ValueError('No valid load type specified in json file --')