def complementary_merge(run_context, bookmap_location_file_name, bookmap_sheet, mergespec_file_name, input_table_names, output_table_names): db_context = run_context.getDBContext() temp_bookmap_table_name = dbutilities.get_temp_table(db_context).table_name BookMapReader(inputfile=bookmap_location_file_name, inputsheet=bookmap_sheet, read_to_db=True, db_context=db_context, outputTable=temp_bookmap_table_name) temp_prep_table_name = dbutilities.get_temp_table(db_context).table_name _comp_prepare1(run_context=run_context, bookmaps_table_name=temp_bookmap_table_name, input_table_names=input_table_names, output_table_name=temp_prep_table_name) prepped_input_table_names = {} for key in input_table_names: if key == FLAT_TABLE_KEY_NAME: prepped_input_table_names[key] = temp_prep_table_name else: prepped_input_table_names[key] = input_table_names[key] cmpmrg = ComplementaryMerge(run_context=run_context) cmpmrg.identity_insert = False cmpmrg.complementary_merge(mergespec_file=mergespec_file_name, input_table_names=prepped_input_table_names, output_table_names=output_table_names)
def test_valid_mergespec_file_output_format(self): self._log.info( 'test_valid_mergespec_file_output_format() - testing...') valid_ds_input_file = os.path.join(self._run_context.tests_safe_dir, DS_INPUT_FILE_VALID) input_tables = {FLAT_TABLE_KEY_NAME: VALID_DS_INPUT_TABLE_NAME} dbutilities.drop_table_if_exists( db_context=self._run_context.getDBContext(), table=VALID_DS_INPUT_TABLE_NAME) input_table_reader = SafeExcelReader( run_context=self._run_context, filename=valid_ds_input_file, sheet_name=DS_INPUT_FILE_SHEET_VALID, db_context=self._run_context.getDBContext(), output_table=VALID_DS_INPUT_TABLE_NAME) input_table_reader.createTable() mergespec_file = 'C:/{}'.format(MERGESPEC_OUTPUT_FILE) if os.path.exists(mergespec_file): os.remove(mergespec_file) cmpmrg = ComplementaryMerge(run_context=self._run_context) cmpmrg.create_mergespec_file(input_table_names=input_tables, new_mergespec_file=mergespec_file) self._log.info('test_valid_mergespec_file_output_format() - tested.') filename1 = mergespec_file filename2 = os.path.join(self._run_context.tests_safe_dir, COMPARE_FILE_MERGESPEC_OUTPUT) assert filecmp.cmp( filename1, filename2), 'Python test output does not match expected output.'
def test_cmf_null_mergespec_file(self): self._log.info('test_null_mergespec_file() - testing...') cmpmrg = ComplementaryMerge(run_context=self._run_context) with self.assertRaises(StandardError): cmpmrg.create_mergespec_file(input_table_name='no_table', new_mergespec_file=None) self._log.info('test_null_mergespec_file() - tested.')
def test_cm_null_input_table_names(self): self._log.info('test_null_input_table_names() - testing...') mergespec_file = os.path.join(self._run_context.tests_safe_dir, MERGESPEC_FILE_VALID) cmpmrg = ComplementaryMerge(run_context=self._run_context) with self.assertRaises(StandardError): cmpmrg.complementary_merge(mergespec_file=mergespec_file, input_table_names=None, output_table_names=None) self._log.info('test_null_input_table_names() - tested.')
def test_mergespec_file_already_exists(self): self._log.info('test_mergespec_file_already_exists() - testing...') mergespec_file = os.path.join(self._run_context.tests_safe_dir, MERGESPEC_FILE_VALID) cmpmrg = ComplementaryMerge(run_context=self._run_context) with self.assertRaises(IOError): cmpmrg.create_mergespec_file( input_table_names={FLAT_TABLE_KEY_NAME: 'no_table'}, new_mergespec_file=mergespec_file) self._log.info('test_mergespec_file_already_exists() - tested.')
def test_merge_nonfuzzy(self): self._log.info('test_merge_nonfuzzy - testing...') mergespec_file = os.path.join(self._run_context.tests_safe_dir, MERGESPEC_FILE_VALID_NONFUZZY) valid_ds_input_file = os.path.join(self._run_context.tests_safe_dir, DS_INPUT_FILE_VALID) input_tables = {FLAT_TABLE_KEY_NAME: VALID_DS_INPUT_TABLE_NAME} output_tables = { FLAT_TABLE_KEY_NAME: VALID_DS_OUTPUT_TABLE_NAME_NONFUZZY } dbutilities.drop_table_if_exists( db_context=self._run_context.getDBContext(), table=VALID_DS_INPUT_TABLE_NAME) dbutilities.drop_table_if_exists( db_context=self._run_context.getDBContext(), table=VALID_DS_OUTPUT_TABLE_NAME_NONFUZZY) for subject in output_tables: table_name = VALID_DS_OUTPUT_TABLE_NAME_NONFUZZY + '_' + subject self._log.info('dropping output table [{}]'.format(table_name)) dbutilities.drop_table_if_exists( db_context=self._run_context.getDBContext(), table=table_name) input_table_reader = SafeExcelReader( run_context=self._run_context, filename=valid_ds_input_file, sheet_name=DS_INPUT_FILE_SHEET_VALID, db_context=self._run_context.getDBContext(), output_table=VALID_DS_INPUT_TABLE_NAME) input_table_reader.createTable() cmpmrg = ComplementaryMerge( run_context=self._run_context, flat_table_identity_field_name='[import_order]') cmpmrg.complementary_merge(mergespec_file=mergespec_file, input_table_names=input_tables, output_table_names=output_tables, force_one_only=True) columns_str = ( 'Barcode, ID, LastName, FirstName, Score1, Race, Old, Score2, Score3, Score4, variable_priority,' ' record_priority, Attempt1, Attempt2, Attempt3, Attempt4, iep') filename1 = self._dump_table(VALID_DS_OUTPUT_TABLE_NAME_NONFUZZY, columns_str=columns_str, sort_column_nbrs=[1, 6, 0]) filename2 = os.path.join(self._run_context.tests_safe_dir, COMPARE_FILE_NONFUZZY) assert filecmp.cmp( filename1, filename2), 'Python test output does not match SAS test output.' self._log.info('test_merge_nonfuzzy - tested.')
def test_ds_input_not_found(self): self._log.info('test_ds_input_not_found() - testing...') mergespec_file = os.path.join(self._run_context.tests_safe_dir, MERGESPEC_FILE_VALID) input_table_names = {FLAT_TABLE_KEY_NAME: 'no_table'} output_table_names = {FLAT_TABLE_KEY_NAME: 'no_table'} cmpmrg = ComplementaryMerge(run_context=self._run_context) with self.assertRaises(IOError): cmpmrg.complementary_merge(mergespec_file=mergespec_file, input_table_names=input_table_names, output_table_names=output_table_names) self._log.info('test_ds_input_not_found() - tested.')
def test_ds_output_already_exists(self): self._log.info('test_ds_output_already_exists() - testing...') mergespec_file = os.path.join(self._run_context.tests_safe_dir, MERGESPEC_FILE_VALID) valid_ds_input_file = os.path.join(self._run_context.tests_safe_dir, DS_INPUT_FILE_VALID) dbutilities.drop_table_if_exists( db_context=self._run_context.getDBContext(), table=VALID_DS_INPUT_TABLE_NAME) input_table_reader = SafeExcelReader( run_context=self._run_context, filename=valid_ds_input_file, sheet_name=DS_INPUT_FILE_SHEET_VALID, db_context=self._run_context.getDBContext(), output_table=VALID_DS_INPUT_TABLE_NAME) input_table_reader.createTable() input_table_names = {FLAT_TABLE_KEY_NAME: VALID_DS_INPUT_TABLE_NAME} output_table_names = {FLAT_TABLE_KEY_NAME: VALID_DS_INPUT_TABLE_NAME} cmpmrg = ComplementaryMerge(run_context=self._run_context) with self.assertRaises(IOError): cmpmrg.complementary_merge(mergespec_file=mergespec_file, input_table_names=input_table_names, output_table_names=output_table_names) self._log.info('test_ds_output_already_exists() - tested.')
def create_mergespec_file(run_context, input_table_names, new_mergespec_file): cmpmrg = ComplementaryMerge(run_context=run_context) cmpmrg.create_mergespec_file(input_table_names=input_table_names, new_mergespec_file=new_mergespec_file)
def test_null_run_context(self): self._log.info('test_null_run_context() - testing...') with self.assertRaises(StandardError): ComplementaryMerge(run_context=None) self._log.info('test_null_run_context() - tested.')
def test_mergespec_file_invalid_format(self): self._log.info('test_mergespec_file_invalid_format() - testing...') valid_ds_input_file = os.path.join(self._run_context.tests_safe_dir, DS_INPUT_FILE_VALID) dbutilities.drop_table_if_exists( db_context=self._run_context.getDBContext(), table=VALID_DS_INPUT_TABLE_NAME) dbutilities.drop_table_if_exists( db_context=self._run_context.getDBContext(), table=VALID_DS_OUTPUT_TABLE_NAME) input_table_reader = SafeExcelReader( run_context=self._run_context, filename=valid_ds_input_file, sheet_name=DS_INPUT_FILE_SHEET_VALID, db_context=self._run_context.getDBContext(), output_table=VALID_DS_INPUT_TABLE_NAME) input_table_reader.createTable() mergespec_file = os.path.join(self._run_context.tests_safe_dir, MERGESPEC_FILE_MISSING_COLUMN) input_table_names = {FLAT_TABLE_KEY_NAME: VALID_DS_INPUT_TABLE_NAME} output_table_names = {FLAT_TABLE_KEY_NAME: VALID_DS_OUTPUT_TABLE_NAME} cmpmrg1 = ComplementaryMerge(run_context=self._run_context) with self.assertRaises(EOFError): cmpmrg1.complementary_merge(mergespec_file=mergespec_file, input_table_names=input_table_names, output_table_names=output_table_names, force_one_only=True) mergespec_file = os.path.join(self._run_context.tests_safe_dir, MERGESPEC_FILE_INVALID_COLUMN) cmpmrg2 = ComplementaryMerge(run_context=self._run_context) with self.assertRaises(ValueError): cmpmrg2.complementary_merge(mergespec_file=mergespec_file, input_table_names=input_table_names, output_table_names=output_table_names, force_one_only=True) self._log.info('test_mergespec_file_invalid_format() - tested.') mergespec_file = os.path.join(self._run_context.tests_safe_dir, MERGESPEC_FILE_NO_VARIABLE_PRIORITY) cmpmrg2 = ComplementaryMerge(run_context=self._run_context) with self.assertRaises(ValueError): cmpmrg2.complementary_merge(mergespec_file=mergespec_file, input_table_names=input_table_names, output_table_names=output_table_names, force_one_only=True) self._log.info('test_mergespec_file_invalid_format() - tested.')