def complementary_merge(run_context, bookmap_location_file_name, bookmap_sheet,
mergespec_file_name, input_table_names,
output_table_names):
db_context = run_context.getDBContext()
temp_bookmap_table_name = dbutilities.get_temp_table(db_context).table_name
BookMapReader(inputfile=bookmap_location_file_name,
inputsheet=bookmap_sheet,
read_to_db=True,
db_context=db_context,
outputTable=temp_bookmap_table_name)
temp_prep_table_name = dbutilities.get_temp_table(db_context).table_name
_comp_prepare1(run_context=run_context,
bookmaps_table_name=temp_bookmap_table_name,
input_table_names=input_table_names,
output_table_name=temp_prep_table_name)
prepped_input_table_names = {}
for key in input_table_names:
if key == FLAT_TABLE_KEY_NAME:
prepped_input_table_names[key] = temp_prep_table_name
else:
prepped_input_table_names[key] = input_table_names[key]
cmpmrg = ComplementaryMerge(run_context=run_context)
cmpmrg.identity_insert = False
cmpmrg.complementary_merge(mergespec_file=mergespec_file_name,
input_table_names=prepped_input_table_names,
output_table_names=output_table_names)
def test_valid_mergespec_file_output_format(self):
self._log.info(
'test_valid_mergespec_file_output_format() - testing...')
valid_ds_input_file = os.path.join(self._run_context.tests_safe_dir,
DS_INPUT_FILE_VALID)
input_tables = {FLAT_TABLE_KEY_NAME: VALID_DS_INPUT_TABLE_NAME}
dbutilities.drop_table_if_exists(
db_context=self._run_context.getDBContext(),
table=VALID_DS_INPUT_TABLE_NAME)
input_table_reader = SafeExcelReader(
run_context=self._run_context,
filename=valid_ds_input_file,
sheet_name=DS_INPUT_FILE_SHEET_VALID,
db_context=self._run_context.getDBContext(),
output_table=VALID_DS_INPUT_TABLE_NAME)
input_table_reader.createTable()
mergespec_file = 'C:/{}'.format(MERGESPEC_OUTPUT_FILE)
if os.path.exists(mergespec_file):
os.remove(mergespec_file)
cmpmrg = ComplementaryMerge(run_context=self._run_context)
cmpmrg.create_mergespec_file(input_table_names=input_tables,
new_mergespec_file=mergespec_file)
self._log.info('test_valid_mergespec_file_output_format() - tested.')
filename1 = mergespec_file
filename2 = os.path.join(self._run_context.tests_safe_dir,
COMPARE_FILE_MERGESPEC_OUTPUT)
assert filecmp.cmp(
filename1,
filename2), 'Python test output does not match expected output.'
def test_cmf_null_mergespec_file(self):
self._log.info('test_null_mergespec_file() - testing...')
cmpmrg = ComplementaryMerge(run_context=self._run_context)
with self.assertRaises(StandardError):
cmpmrg.create_mergespec_file(input_table_name='no_table',
new_mergespec_file=None)
self._log.info('test_null_mergespec_file() - tested.')
def test_cm_null_input_table_names(self):
self._log.info('test_null_input_table_names() - testing...')
mergespec_file = os.path.join(self._run_context.tests_safe_dir,
MERGESPEC_FILE_VALID)
cmpmrg = ComplementaryMerge(run_context=self._run_context)
with self.assertRaises(StandardError):
cmpmrg.complementary_merge(mergespec_file=mergespec_file,
input_table_names=None,
output_table_names=None)
self._log.info('test_null_input_table_names() - tested.')
def test_mergespec_file_already_exists(self):
self._log.info('test_mergespec_file_already_exists() - testing...')
mergespec_file = os.path.join(self._run_context.tests_safe_dir,
MERGESPEC_FILE_VALID)
cmpmrg = ComplementaryMerge(run_context=self._run_context)
with self.assertRaises(IOError):
cmpmrg.create_mergespec_file(
input_table_names={FLAT_TABLE_KEY_NAME: 'no_table'},
new_mergespec_file=mergespec_file)
self._log.info('test_mergespec_file_already_exists() - tested.')
def test_merge_nonfuzzy(self):
self._log.info('test_merge_nonfuzzy - testing...')
mergespec_file = os.path.join(self._run_context.tests_safe_dir,
MERGESPEC_FILE_VALID_NONFUZZY)
valid_ds_input_file = os.path.join(self._run_context.tests_safe_dir,
DS_INPUT_FILE_VALID)
input_tables = {FLAT_TABLE_KEY_NAME: VALID_DS_INPUT_TABLE_NAME}
output_tables = {
FLAT_TABLE_KEY_NAME: VALID_DS_OUTPUT_TABLE_NAME_NONFUZZY
}
dbutilities.drop_table_if_exists(
db_context=self._run_context.getDBContext(),
table=VALID_DS_INPUT_TABLE_NAME)
dbutilities.drop_table_if_exists(
db_context=self._run_context.getDBContext(),
table=VALID_DS_OUTPUT_TABLE_NAME_NONFUZZY)
for subject in output_tables:
table_name = VALID_DS_OUTPUT_TABLE_NAME_NONFUZZY + '_' + subject
self._log.info('dropping output table [{}]'.format(table_name))
dbutilities.drop_table_if_exists(
db_context=self._run_context.getDBContext(), table=table_name)
input_table_reader = SafeExcelReader(
run_context=self._run_context,
filename=valid_ds_input_file,
sheet_name=DS_INPUT_FILE_SHEET_VALID,
db_context=self._run_context.getDBContext(),
output_table=VALID_DS_INPUT_TABLE_NAME)
input_table_reader.createTable()
cmpmrg = ComplementaryMerge(
run_context=self._run_context,
flat_table_identity_field_name='[import_order]')
cmpmrg.complementary_merge(mergespec_file=mergespec_file,
input_table_names=input_tables,
output_table_names=output_tables,
force_one_only=True)
columns_str = (
'Barcode, ID, LastName, FirstName, Score1, Race, Old, Score2, Score3, Score4, variable_priority,'
' record_priority, Attempt1, Attempt2, Attempt3, Attempt4, iep')
filename1 = self._dump_table(VALID_DS_OUTPUT_TABLE_NAME_NONFUZZY,
columns_str=columns_str,
sort_column_nbrs=[1, 6, 0])
filename2 = os.path.join(self._run_context.tests_safe_dir,
COMPARE_FILE_NONFUZZY)
assert filecmp.cmp(
filename1,
filename2), 'Python test output does not match SAS test output.'
self._log.info('test_merge_nonfuzzy - tested.')
def test_ds_input_not_found(self):
self._log.info('test_ds_input_not_found() - testing...')
mergespec_file = os.path.join(self._run_context.tests_safe_dir,
MERGESPEC_FILE_VALID)
input_table_names = {FLAT_TABLE_KEY_NAME: 'no_table'}
output_table_names = {FLAT_TABLE_KEY_NAME: 'no_table'}
cmpmrg = ComplementaryMerge(run_context=self._run_context)
with self.assertRaises(IOError):
cmpmrg.complementary_merge(mergespec_file=mergespec_file,
input_table_names=input_table_names,
output_table_names=output_table_names)
self._log.info('test_ds_input_not_found() - tested.')
def test_ds_output_already_exists(self):
self._log.info('test_ds_output_already_exists() - testing...')
mergespec_file = os.path.join(self._run_context.tests_safe_dir,
MERGESPEC_FILE_VALID)
valid_ds_input_file = os.path.join(self._run_context.tests_safe_dir,
DS_INPUT_FILE_VALID)
dbutilities.drop_table_if_exists(
db_context=self._run_context.getDBContext(),
table=VALID_DS_INPUT_TABLE_NAME)
input_table_reader = SafeExcelReader(
run_context=self._run_context,
filename=valid_ds_input_file,
sheet_name=DS_INPUT_FILE_SHEET_VALID,
db_context=self._run_context.getDBContext(),
output_table=VALID_DS_INPUT_TABLE_NAME)
input_table_reader.createTable()
input_table_names = {FLAT_TABLE_KEY_NAME: VALID_DS_INPUT_TABLE_NAME}
output_table_names = {FLAT_TABLE_KEY_NAME: VALID_DS_INPUT_TABLE_NAME}
cmpmrg = ComplementaryMerge(run_context=self._run_context)
with self.assertRaises(IOError):
cmpmrg.complementary_merge(mergespec_file=mergespec_file,
input_table_names=input_table_names,
output_table_names=output_table_names)
self._log.info('test_ds_output_already_exists() - tested.')
def create_mergespec_file(run_context, input_table_names, new_mergespec_file):
cmpmrg = ComplementaryMerge(run_context=run_context)
cmpmrg.create_mergespec_file(input_table_names=input_table_names,
new_mergespec_file=new_mergespec_file)
def test_null_run_context(self):
self._log.info('test_null_run_context() - testing...')
with self.assertRaises(StandardError):
ComplementaryMerge(run_context=None)
self._log.info('test_null_run_context() - tested.')
def test_mergespec_file_invalid_format(self):
self._log.info('test_mergespec_file_invalid_format() - testing...')
valid_ds_input_file = os.path.join(self._run_context.tests_safe_dir,
DS_INPUT_FILE_VALID)
dbutilities.drop_table_if_exists(
db_context=self._run_context.getDBContext(),
table=VALID_DS_INPUT_TABLE_NAME)
dbutilities.drop_table_if_exists(
db_context=self._run_context.getDBContext(),
table=VALID_DS_OUTPUT_TABLE_NAME)
input_table_reader = SafeExcelReader(
run_context=self._run_context,
filename=valid_ds_input_file,
sheet_name=DS_INPUT_FILE_SHEET_VALID,
db_context=self._run_context.getDBContext(),
output_table=VALID_DS_INPUT_TABLE_NAME)
input_table_reader.createTable()
mergespec_file = os.path.join(self._run_context.tests_safe_dir,
MERGESPEC_FILE_MISSING_COLUMN)
input_table_names = {FLAT_TABLE_KEY_NAME: VALID_DS_INPUT_TABLE_NAME}
output_table_names = {FLAT_TABLE_KEY_NAME: VALID_DS_OUTPUT_TABLE_NAME}
cmpmrg1 = ComplementaryMerge(run_context=self._run_context)
with self.assertRaises(EOFError):
cmpmrg1.complementary_merge(mergespec_file=mergespec_file,
input_table_names=input_table_names,
output_table_names=output_table_names,
force_one_only=True)
mergespec_file = os.path.join(self._run_context.tests_safe_dir,
MERGESPEC_FILE_INVALID_COLUMN)
cmpmrg2 = ComplementaryMerge(run_context=self._run_context)
with self.assertRaises(ValueError):
cmpmrg2.complementary_merge(mergespec_file=mergespec_file,
input_table_names=input_table_names,
output_table_names=output_table_names,
force_one_only=True)
self._log.info('test_mergespec_file_invalid_format() - tested.')
mergespec_file = os.path.join(self._run_context.tests_safe_dir,
MERGESPEC_FILE_NO_VARIABLE_PRIORITY)
cmpmrg2 = ComplementaryMerge(run_context=self._run_context)
with self.assertRaises(ValueError):
cmpmrg2.complementary_merge(mergespec_file=mergespec_file,
input_table_names=input_table_names,
output_table_names=output_table_names,
force_one_only=True)
self._log.info('test_mergespec_file_invalid_format() - tested.')