コード例 #1
0
def complementary_merge(run_context, bookmap_location_file_name, bookmap_sheet,
                        mergespec_file_name, input_table_names,
                        output_table_names):
    db_context = run_context.getDBContext()
    temp_bookmap_table_name = dbutilities.get_temp_table(db_context).table_name
    BookMapReader(inputfile=bookmap_location_file_name,
                  inputsheet=bookmap_sheet,
                  read_to_db=True,
                  db_context=db_context,
                  outputTable=temp_bookmap_table_name)
    temp_prep_table_name = dbutilities.get_temp_table(db_context).table_name
    _comp_prepare1(run_context=run_context,
                   bookmaps_table_name=temp_bookmap_table_name,
                   input_table_names=input_table_names,
                   output_table_name=temp_prep_table_name)

    prepped_input_table_names = {}
    for key in input_table_names:
        if key == FLAT_TABLE_KEY_NAME:
            prepped_input_table_names[key] = temp_prep_table_name
        else:
            prepped_input_table_names[key] = input_table_names[key]

    cmpmrg = ComplementaryMerge(run_context=run_context)
    cmpmrg.identity_insert = False
    cmpmrg.complementary_merge(mergespec_file=mergespec_file_name,
                               input_table_names=prepped_input_table_names,
                               output_table_names=output_table_names)
コード例 #2
0
 def test_cm_null_input_table_names(self):
     self._log.info('test_null_input_table_names() - testing...')
     mergespec_file = os.path.join(self._run_context.tests_safe_dir,
                                   MERGESPEC_FILE_VALID)
     cmpmrg = ComplementaryMerge(run_context=self._run_context)
     with self.assertRaises(StandardError):
         cmpmrg.complementary_merge(mergespec_file=mergespec_file,
                                    input_table_names=None,
                                    output_table_names=None)
     self._log.info('test_null_input_table_names() - tested.')
コード例 #3
0
    def test_merge_nonfuzzy(self):
        self._log.info('test_merge_nonfuzzy - testing...')
        mergespec_file = os.path.join(self._run_context.tests_safe_dir,
                                      MERGESPEC_FILE_VALID_NONFUZZY)
        valid_ds_input_file = os.path.join(self._run_context.tests_safe_dir,
                                           DS_INPUT_FILE_VALID)

        input_tables = {FLAT_TABLE_KEY_NAME: VALID_DS_INPUT_TABLE_NAME}
        output_tables = {
            FLAT_TABLE_KEY_NAME: VALID_DS_OUTPUT_TABLE_NAME_NONFUZZY
        }

        dbutilities.drop_table_if_exists(
            db_context=self._run_context.getDBContext(),
            table=VALID_DS_INPUT_TABLE_NAME)
        dbutilities.drop_table_if_exists(
            db_context=self._run_context.getDBContext(),
            table=VALID_DS_OUTPUT_TABLE_NAME_NONFUZZY)
        for subject in output_tables:
            table_name = VALID_DS_OUTPUT_TABLE_NAME_NONFUZZY + '_' + subject
            self._log.info('dropping output table [{}]'.format(table_name))
            dbutilities.drop_table_if_exists(
                db_context=self._run_context.getDBContext(), table=table_name)

        input_table_reader = SafeExcelReader(
            run_context=self._run_context,
            filename=valid_ds_input_file,
            sheet_name=DS_INPUT_FILE_SHEET_VALID,
            db_context=self._run_context.getDBContext(),
            output_table=VALID_DS_INPUT_TABLE_NAME)
        input_table_reader.createTable()

        cmpmrg = ComplementaryMerge(
            run_context=self._run_context,
            flat_table_identity_field_name='[import_order]')
        cmpmrg.complementary_merge(mergespec_file=mergespec_file,
                                   input_table_names=input_tables,
                                   output_table_names=output_tables,
                                   force_one_only=True)

        columns_str = (
            'Barcode, ID, LastName, FirstName, Score1, Race, Old, Score2, Score3, Score4, variable_priority,'
            ' record_priority, Attempt1, Attempt2, Attempt3, Attempt4, iep')
        filename1 = self._dump_table(VALID_DS_OUTPUT_TABLE_NAME_NONFUZZY,
                                     columns_str=columns_str,
                                     sort_column_nbrs=[1, 6, 0])
        filename2 = os.path.join(self._run_context.tests_safe_dir,
                                 COMPARE_FILE_NONFUZZY)
        assert filecmp.cmp(
            filename1,
            filename2), 'Python test output does not match SAS test output.'

        self._log.info('test_merge_nonfuzzy - tested.')
コード例 #4
0
 def test_ds_input_not_found(self):
     self._log.info('test_ds_input_not_found() - testing...')
     mergespec_file = os.path.join(self._run_context.tests_safe_dir,
                                   MERGESPEC_FILE_VALID)
     input_table_names = {FLAT_TABLE_KEY_NAME: 'no_table'}
     output_table_names = {FLAT_TABLE_KEY_NAME: 'no_table'}
     cmpmrg = ComplementaryMerge(run_context=self._run_context)
     with self.assertRaises(IOError):
         cmpmrg.complementary_merge(mergespec_file=mergespec_file,
                                    input_table_names=input_table_names,
                                    output_table_names=output_table_names)
     self._log.info('test_ds_input_not_found() - tested.')
コード例 #5
0
 def test_ds_output_already_exists(self):
     self._log.info('test_ds_output_already_exists() - testing...')
     mergespec_file = os.path.join(self._run_context.tests_safe_dir,
                                   MERGESPEC_FILE_VALID)
     valid_ds_input_file = os.path.join(self._run_context.tests_safe_dir,
                                        DS_INPUT_FILE_VALID)
     dbutilities.drop_table_if_exists(
         db_context=self._run_context.getDBContext(),
         table=VALID_DS_INPUT_TABLE_NAME)
     input_table_reader = SafeExcelReader(
         run_context=self._run_context,
         filename=valid_ds_input_file,
         sheet_name=DS_INPUT_FILE_SHEET_VALID,
         db_context=self._run_context.getDBContext(),
         output_table=VALID_DS_INPUT_TABLE_NAME)
     input_table_reader.createTable()
     input_table_names = {FLAT_TABLE_KEY_NAME: VALID_DS_INPUT_TABLE_NAME}
     output_table_names = {FLAT_TABLE_KEY_NAME: VALID_DS_INPUT_TABLE_NAME}
     cmpmrg = ComplementaryMerge(run_context=self._run_context)
     with self.assertRaises(IOError):
         cmpmrg.complementary_merge(mergespec_file=mergespec_file,
                                    input_table_names=input_table_names,
                                    output_table_names=output_table_names)
     self._log.info('test_ds_output_already_exists() - tested.')
コード例 #6
0
    def test_mergespec_file_invalid_format(self):
        self._log.info('test_mergespec_file_invalid_format() - testing...')
        valid_ds_input_file = os.path.join(self._run_context.tests_safe_dir,
                                           DS_INPUT_FILE_VALID)
        dbutilities.drop_table_if_exists(
            db_context=self._run_context.getDBContext(),
            table=VALID_DS_INPUT_TABLE_NAME)
        dbutilities.drop_table_if_exists(
            db_context=self._run_context.getDBContext(),
            table=VALID_DS_OUTPUT_TABLE_NAME)
        input_table_reader = SafeExcelReader(
            run_context=self._run_context,
            filename=valid_ds_input_file,
            sheet_name=DS_INPUT_FILE_SHEET_VALID,
            db_context=self._run_context.getDBContext(),
            output_table=VALID_DS_INPUT_TABLE_NAME)
        input_table_reader.createTable()
        mergespec_file = os.path.join(self._run_context.tests_safe_dir,
                                      MERGESPEC_FILE_MISSING_COLUMN)
        input_table_names = {FLAT_TABLE_KEY_NAME: VALID_DS_INPUT_TABLE_NAME}
        output_table_names = {FLAT_TABLE_KEY_NAME: VALID_DS_OUTPUT_TABLE_NAME}
        cmpmrg1 = ComplementaryMerge(run_context=self._run_context)
        with self.assertRaises(EOFError):
            cmpmrg1.complementary_merge(mergespec_file=mergespec_file,
                                        input_table_names=input_table_names,
                                        output_table_names=output_table_names,
                                        force_one_only=True)

        mergespec_file = os.path.join(self._run_context.tests_safe_dir,
                                      MERGESPEC_FILE_INVALID_COLUMN)
        cmpmrg2 = ComplementaryMerge(run_context=self._run_context)
        with self.assertRaises(ValueError):
            cmpmrg2.complementary_merge(mergespec_file=mergespec_file,
                                        input_table_names=input_table_names,
                                        output_table_names=output_table_names,
                                        force_one_only=True)
        self._log.info('test_mergespec_file_invalid_format() - tested.')

        mergespec_file = os.path.join(self._run_context.tests_safe_dir,
                                      MERGESPEC_FILE_NO_VARIABLE_PRIORITY)
        cmpmrg2 = ComplementaryMerge(run_context=self._run_context)
        with self.assertRaises(ValueError):
            cmpmrg2.complementary_merge(mergespec_file=mergespec_file,
                                        input_table_names=input_table_names,
                                        output_table_names=output_table_names,
                                        force_one_only=True)
        self._log.info('test_mergespec_file_invalid_format() - tested.')