def test_filter_nonexisting_obsids_and_ask_none_value_skip(self, mock_notfound):
         mock_notfound.return_value.answer = u'skip'
         mock_notfound.return_value.value = 10
         file_data = [[u'obsid', u'ae'], [u'1', u'b'], [u'2', u'c'], [u'3', u'd'], [u'10', u'e'], [u'1_g', u'f'], [u'1 a', u'g'], [None, u'h']]
         existing_obsids = [u'2', u'3', u'10', u'1_g', u'1 a']
         filtered_file_data = utils.filter_nonexisting_values_and_ask(file_data, u'obsid', existing_obsids)
         reference_list = [[u'obsid', u'ae'], [u'2', u'c'], [u'3', u'd'], [u'10', u'e'], [u'1_g', u'f'], [u'1 a', u'g']]
         assert filtered_file_data == reference_list
 def test_filter_nonexisting_obsids_and_ask_none_value_skip(self, mock_notfound, mock_iface):
         mock_notfound.return_value.answer = 'skip'
         mock_notfound.return_value.value = 10
         mock_notfound.return_value.reuse_column = 'obsid'
         file_data = [['obsid', 'ae'], ['1', 'b'], ['2', 'c'], ['3', 'd'], ['10', 'e'], ['1_g', 'f'], ['1 a', 'g'], [None, 'h']]
         existing_obsids = ['2', '3', '10', '1_g', '1 a']
         filtered_file_data = utils.filter_nonexisting_values_and_ask(file_data, 'obsid', existing_obsids)
         reference_list = [['obsid', 'ae'], ['2', 'c'], ['3', 'd'], ['10', 'e'], ['1_g', 'f'], ['1 a', 'g']]
         assert filtered_file_data == reference_list
Exemple #3
0
 def test_filter_nonexisting_obsids_and_ask_none_value_skip(self, mock_notfound, mock_iface):
         mock_notfound.return_value.answer = 'skip'
         mock_notfound.return_value.value = 10
         mock_notfound.return_value.reuse_column = 'obsid'
         
         file_data = [['obsid', 'ae'], ['1', 'b'], ['2', 'c'], ['3', 'd'], ['10', 'e'], ['1_g', 'f'], ['1 a', 'g'], [None, 'h']]
         existing_obsids = ['2', '3', '10', '1_g', '1 a']
         filtered_file_data = utils.filter_nonexisting_values_and_ask(file_data, 'obsid', existing_obsids)
         reference_list = [['obsid', 'ae'], ['2', 'c'], ['3', 'd'], ['10', 'e'], ['1_g', 'f'], ['1 a', 'g']]
         assert filtered_file_data == reference_list
 def test_filter_nonexisting_obsids_and_ask_skip_only_ask_once(self, mock_notfound):
         mock_notfound.return_value.answer = u'skip'
         mock_notfound.return_value.value = 10
         file_data = [[u'obsid', u'ae'], [u'1', u'b'], [u'2', u'c'], [u'3', u'd'], [u'10', u'e'], [u'1_g', u'f'], [u'1 a', u'g'], [u'21', u'h'], [u'1', u'i']]
         existing_obsids = [u'2', u'3', u'10', u'1_g', u'1 a']
         filtered_file_data = utils.filter_nonexisting_values_and_ask(file_data, u'obsid', existing_obsids)
         reference_list = [[u'obsid', u'ae'], [u'2', u'c'], [u'3', u'd'], [u'10', u'e'], [u'1_g', u'f'], [u'1 a', u'g']]
         assert filtered_file_data == reference_list
         #The mock should only be called twice. First for 1, then for 21, and then 1 again should use the already given answer.
         assert len(mock_notfound.mock_calls) == 2
 def test_filter_nonexisting_obsids_and_ask_header_not_found(self, mock_notfound, mock_iface):
     """If a asked for header column is not found, it's added to the end of the rows."""
     mock_notfound.return_value.answer = 'ok'
     mock_notfound.return_value.value = 10
     mock_notfound.return_value.reuse_column = 'obsid'
     file_data = [['obsid', 'ae'], ['1', 'b'], ['2', 'c'], ['3', 'd'], ['10', 'e'], ['1_g', 'f'], ['1 a', 'g'], ['21', 'h']]
     existing_obsids = ['2', '3', '10', '1_g', '1 a']
     filtered_file_data = utils.filter_nonexisting_values_and_ask(file_data, 'header_that_should_not_exist', existing_obsids)
     reference_list = [['obsid', 'ae', 'header_that_should_not_exist'], ['1', 'b', '10'], ['2', 'c', '10'], ['3', 'd', '10'], ['10', 'e', '10'], ['1_g', 'f', '10'], ['1 a', 'g', '10'], ['21', 'h', '10']]
     assert filtered_file_data == reference_list
Exemple #6
0
    def start_import(self, all_lab_results, lablitteras_to_import):
        all_lab_results = copy.deepcopy(all_lab_results)
        all_lab_results = dict([
            (lablittera, v) for lablittera, v in all_lab_results.iteritems()
            if lablittera in lablitteras_to_import
        ])

        #Allow the user to connect the metadata rows to obsids.
        meta_headers = get_metadata_headers(all_lab_results)
        ask_obsid_table = [meta_headers]
        for lablittera, v in sorted(all_lab_results.iteritems()):
            metarow = [
                v[u'metadata'].get(meta_header, u'')
                for meta_header in meta_headers
            ]
            ask_obsid_table.append(metarow)
        existing_obsids = utils.get_all_obsids()
        answer = utils.filter_nonexisting_values_and_ask(
            ask_obsid_table,
            u'obsid',
            existing_values=existing_obsids,
            try_capitalize=False,
            always_ask_user=True)
        if answer == u'cancel':
            self.status = True
            return Cancel()
        elif not answer:
            self.status = False
            utils.MessagebarAndLog.critical(
                bar_msg=u'Error, no observations remain. No import done.')
            return Cancel()
        else:
            remaining_lablitteras_obsids = dict([(x[0], x[-1])
                                                 for x in answer[1:]])
        #Filter the remaining lablitteras and add an obsid field
        _all_lab_results = {}
        for lablittera, v in all_lab_results.iteritems():
            if lablittera in remaining_lablitteras_obsids:
                v[u'metadata'][u'obsid'] = remaining_lablitteras_obsids[
                    lablittera]
                _all_lab_results[lablittera] = v
        all_lab_results = _all_lab_results

        self.wquallab_data_table = self.to_table(all_lab_results)

        importer = import_data_to_db.midv_data_importer()
        answer = importer.general_import(goal_table=u'w_qual_lab',
                                         file_data=self.wquallab_data_table)

        importer.SanityCheckVacuumDB()

        if self.close_after_import.isChecked():
            self.close()

        PyQt4.QtGui.QApplication.restoreOverrideCursor()
 def test_filter_nonexisting_obsids_and_ask_skip_only_ask_once(self, mock_notfound, mock_iface):
         mock_notfound.return_value.answer = 'skip'
         mock_notfound.return_value.value = 10
         mock_notfound.return_value.reuse_column = 'obsid'
         file_data = [['obsid', 'ae'], ['1', 'b'], ['2', 'c'], ['3', 'd'], ['10', 'e'], ['1_g', 'f'], ['1 a', 'g'], ['21', 'h'], ['1', 'i']]
         existing_obsids = ['2', '3', '10', '1_g', '1 a']
         filtered_file_data = utils.filter_nonexisting_values_and_ask(file_data, 'obsid', existing_obsids)
         reference_list = [['obsid', 'ae'], ['2', 'c'], ['3', 'd'], ['10', 'e'], ['1_g', 'f'], ['1 a', 'g']]
         assert filtered_file_data == reference_list
         #The mock should only be called twice. First for 1, then for 21, and then 1 again should use the already given answer.
         assert len(mock_notfound.mock_calls) == 2
Exemple #8
0
 def test_filter_nonexisting_obsids_and_ask_skip_only_ask_once(self, mock_notfound, mock_iface):
         mock_notfound.return_value.answer = 'skip'
         mock_notfound.return_value.value = 10
         mock_notfound.return_value.reuse_column = 'obsid'
         
         file_data = [['obsid', 'ae'], ['1', 'b'], ['2', 'c'], ['3', 'd'], ['10', 'e'], ['1_g', 'f'], ['1 a', 'g'], ['21', 'h'], ['1', 'i']]
         existing_obsids = ['2', '3', '10', '1_g', '1 a']
         filtered_file_data = utils.filter_nonexisting_values_and_ask(file_data, 'obsid', existing_obsids)
         reference_list = [['obsid', 'ae'], ['2', 'c'], ['3', 'd'], ['10', 'e'], ['1_g', 'f'], ['1 a', 'g']]
         assert filtered_file_data == reference_list
         #The mock should only be called twice. First for 1, then for 21, and then 1 again should use the already given answer.
         assert len(mock_notfound.mock_calls) == 2
Exemple #9
0
 def test_filter_nonexisting_obsids_and_ask_header_capitalize(
         self, mock_iface):
     file_data = [[u'obsid', u'ae'], [u'a', u'b'], [u'2', u'c']]
     existing_obsids = [u'A', u'2']
     filtered_file_data = utils.filter_nonexisting_values_and_ask(
         file_data=file_data,
         header_value=u'obsid',
         existing_values=existing_obsids,
         try_capitalize=True,
         always_ask_user=False)
     reference_list = [[u'obsid', u'ae'], [u'A', u'b'], [u'2', u'c']]
     assert filtered_file_data == reference_list
Exemple #10
0
 def test_filter_nonexisting_obsids_and_ask_ok(self, mock_notfound, mock_iface):
         mock_notfound.return_value.answer = 'ok'
         mock_notfound.return_value.value = 10
         mock_notfound.return_value.reuse_column = 'obsid'
         mock_checkbox = mock.Mock()
         mock_checkbox.return_value.isChecked.return_value = True
         mock_notfound.return_value.ignore_checkbox = mock_checkbox #isChecked.return_value = True
         
         file_data = [['obsid', 'ae'], ['1', 'b'], ['2', 'c'], ['3', 'd'], ['10', 'e'], ['1_g', 'f'], ['1 a', 'g'], ['21', 'h']]
         existing_obsids = ['2', '3', '10', '1_g', '1 a']
         filtered_file_data = utils.filter_nonexisting_values_and_ask(file_data, 'obsid', existing_obsids)
         reference_list = [['obsid', 'ae'], ['2', 'c'], ['3', 'd'], ['10', 'e'], ['1_g', 'f'], ['1 a', 'g'], ['10', 'b'], ['10', 'h']]
         assert filtered_file_data == reference_list
Exemple #11
0
 def test_filter_nonexisting_obsids_and_ask_header_not_found(self, mock_notfound, mock_iface):
     """If a asked for header column is not found, it's added to the end of the rows."""
     mock_notfound.return_value.answer = 'ok'
     mock_notfound.return_value.value = 10
     mock_notfound.return_value.reuse_column = 'obsid'
     mock_checkbox = mock.Mock()
     mock_checkbox.return_value.isChecked.return_value = True
     mock_notfound.return_value.ignore_checkbox = mock_checkbox  # isChecked.return_value = True
     
     file_data = [['obsid', 'ae'], ['1', 'b'], ['2', 'c'], ['3', 'd'], ['10', 'e'], ['1_g', 'f'], ['1 a', 'g'], ['21', 'h']]
     existing_obsids = ['2', '3', '10', '1_g', '1 a']
     filtered_file_data = utils.filter_nonexisting_values_and_ask(file_data, 'header_that_should_not_exist', existing_obsids)
     reference_list = [['obsid', 'ae', 'header_that_should_not_exist'], ['1', 'b', '10'], ['2', 'c', '10'], ['3', 'd', '10'], ['10', 'e', '10'], ['1_g', 'f', '10'], ['1 a', 'g', '10'], ['21', 'h', '10']]
     assert filtered_file_data == reference_list
Exemple #12
0
 def test_filter_nonexisting_obsids_and_ask_skip(self, mock_notfound,
                                                 mock_iface):
     mock_notfound.return_value.answer = u'skip'
     mock_notfound.return_value.value = 10
     mock_notfound.return_value.reuse_column = u'obsid'
     file_data = [[u'obsid', u'ae'], [u'1', u'b'], [u'2',
                                                    u'c'], [u'3', u'd'],
                  [u'10', u'e'], [u'1_g', u'f'], [u'1 a', u'g'],
                  [u'21', u'h']]
     existing_obsids = [u'2', u'3', u'10', u'1_g', u'1 a']
     filtered_file_data = utils.filter_nonexisting_values_and_ask(
         file_data, u'obsid', existing_obsids)
     reference_list = [[u'obsid', u'ae'], [u'2', u'c'], [u'3', u'd'],
                       [u'10', u'e'], [u'1_g', u'f'], [u'1 a', u'g']]
     assert filtered_file_data == reference_list
Exemple #13
0
 def test_filter_nonexisting_obsids_and_ask_header_not_found(
         self, mock_notfound, mock_iface):
     """If a asked for header column is not found, it's added to the end of the rows."""
     mock_notfound.return_value.answer = u'ok'
     mock_notfound.return_value.value = 10
     mock_notfound.return_value.reuse_column = u'obsid'
     file_data = [[u'obsid', u'ae'], [u'1', u'b'], [u'2',
                                                    u'c'], [u'3', u'd'],
                  [u'10', u'e'], [u'1_g', u'f'], [u'1 a', u'g'],
                  [u'21', u'h']]
     existing_obsids = [u'2', u'3', u'10', u'1_g', u'1 a']
     filtered_file_data = utils.filter_nonexisting_values_and_ask(
         file_data, u'header_that_should_not_exist', existing_obsids)
     reference_list = [[u'obsid', u'ae', u'header_that_should_not_exist'],
                       [u'1', u'b', u'10'], [u'2', u'c', u'10'],
                       [u'3', u'd', u'10'], [u'10', u'e', u'10'],
                       [u'1_g', u'f', u'10'], [u'1 a', u'g', u'10'],
                       [u'21', u'h', u'10']]
     assert filtered_file_data == reference_list
    def start_import(self,
                     files,
                     skip_rows_without_water_level,
                     confirm_names,
                     import_all_data,
                     from_date=None,
                     to_date=None,
                     export_csv=False,
                     import_to_db=True):
        """
        """
        utils.start_waiting_cursor(
        )  #show the user this may take a long time...
        parsed_files = []
        for selected_file in files:
            try:
                res = self.parse_func(
                    path=selected_file,
                    charset=self.charsetchoosen,
                    skip_rows_without_water_level=skip_rows_without_water_level,
                    begindate=from_date,
                    enddate=to_date)
            except:
                utils.MessagebarAndLog.critical(bar_msg=ru(
                    QCoreApplication.translate('LeveloggerImport',
                                               '''Error on file %s.''')) %
                                                selected_file)
                raise

            if res == 'cancel':
                self.status = True
                utils.stop_waiting_cursor()
                return res
            elif res in ('skip', 'ignore'):
                continue

            try:
                file_data, filename, location = res
            except Exception as e:
                utils.MessagebarAndLog.warning(
                    bar_msg=QCoreApplication.translate(
                        'DiverofficeImport',
                        'Import error, see log message panel'),
                    log_msg=ru(
                        QCoreApplication.translate(
                            'DiverofficeImport',
                            'File %s could not be parsed. Msg:\n%s')) %
                    (selected_file, str(e)))
                continue
            parsed_files.append((file_data, filename, location))

        if len(parsed_files) == 0:
            utils.MessagebarAndLog.critical(bar_msg=QCoreApplication.translate(
                'DiverofficeImport', "Import Failure: No files imported"
                ""))
            utils.stop_waiting_cursor()
            return

        #Add obsid to all parsed filedatas by asking the user for it.
        filename_location_obsid = [['filename', 'location', 'obsid']]
        filename_location_obsid.extend(
            [[parsed_file[1], parsed_file[2], parsed_file[2]]
             for parsed_file in parsed_files])

        if confirm_names:
            try_capitalize = False
        else:
            try_capitalize = True

        existing_obsids = utils.get_all_obsids()
        utils.stop_waiting_cursor()
        filename_location_obsid = utils.filter_nonexisting_values_and_ask(
            file_data=filename_location_obsid,
            header_value='obsid',
            existing_values=existing_obsids,
            try_capitalize=try_capitalize,
            always_ask_user=confirm_names)
        utils.start_waiting_cursor()

        if len(filename_location_obsid) < 2:
            utils.MessagebarAndLog.warning(bar_msg=QCoreApplication.translate(
                'DiverofficeImport',
                'Warning. All files were skipped, nothing imported!'))
            utils.stop_waiting_cursor()
            return False

        filenames_obsid = dict([(x[0], x[2])
                                for x in filename_location_obsid[1:]])

        parsed_files_with_obsid = []
        for file_data, filename, location in parsed_files:
            if filename in filenames_obsid:
                file_data = list(file_data)
                obsid = filenames_obsid[filename]
                file_data[0].append('obsid')
                [row.append(obsid) for row in file_data[1:]]
                parsed_files_with_obsid.append([file_data, filename, location])
        #Header
        file_to_import_to_db = [parsed_files_with_obsid[0][0][0]]
        file_to_import_to_db.extend([
            row for parsed_file in parsed_files_with_obsid
            for row in parsed_file[0][1:]
        ])

        # Add comment to import:
        #file_to_import_to_db[0].append('comment')
        #comment = ''
        #[row.append(comment) for row in file_to_import_to_db[1:]]

        if not import_all_data:
            file_to_import_to_db = self.filter_dates_from_filedata(
                file_to_import_to_db, utils.get_last_logger_dates())
        if len(file_to_import_to_db) < 2:
            utils.MessagebarAndLog.info(bar_msg=QCoreApplication.translate(
                'DiverofficeImport',
                'No new data existed in the files. Nothing imported.'))
            self.status = 'True'
            utils.stop_waiting_cursor()
            return True

        if import_to_db:
            importer = import_data_to_db.midv_data_importer()
            answer = importer.general_import('w_levels_logger',
                                             file_to_import_to_db)

        if export_csv:
            path = qgis.PyQt.QtWidgets.QFileDialog.getSaveFileName(
                self, 'Save File', '', 'CSV(*.csv)')
            if path:
                path = ru(path[0])
                utils.write_printlist_to_file(path, file_to_import_to_db)

        utils.stop_waiting_cursor()

        if self.close_after_import.isChecked():
            self.close()
 def test_filter_nonexisting_obsids_and_ask_header_not_found(self):
         file_data = [[u'obsid', u'ae'], [u'1', u'b'], [u'2', u'c'], [u'3', u'd'], [u'10', u'e'], [u'1_g', u'f'], [u'1 a', u'g'], [u'21', u'h']]
         existing_obsids = [u'2', u'3', u'10', u'1_g', u'1 a']
         filtered_file_data = utils.filter_nonexisting_values_and_ask(file_data, u'header_that_should_not_exist', existing_obsids)
         reference_list = [[u'obsid', u'ae'], [u'1', u'b'], [u'2', u'c'], [u'3', u'd'], [u'10', u'e'], [u'1_g', u'f'], [u'1 a', u'g'], [u'21', u'h']]
         assert filtered_file_data == reference_list
 def test_filter_nonexisting_obsids_and_ask_header_capitalize(self):
         file_data = [[u'obsid', u'ae'], [u'a', u'b'], [u'2', u'c']]
         existing_obsids = [u'A', u'2']
         filtered_file_data = utils.filter_nonexisting_values_and_ask(file_data, u'obsid', existing_obsids, True)
         reference_list = [[u'obsid', u'ae'], [u'2', u'c'], [u'A', u'b']]
         assert filtered_file_data == reference_list
Exemple #17
0
    def start_import(self):
        """
        TODO: I have NO IDEA where the dummy parameter is coming from. It gets the value False for some reason!
        :param dummy:
        :return:
        """
        if self.file_data is None:
            raise utils.UsageError(ru(QCoreApplication.translate('GeneralCsvImportGui', 'Error, must select a file first!')))

        translation_dict = self.table_chooser.get_translation_dict()

        file_data = copy.deepcopy(self.file_data)

        dest_table = self.table_chooser.import_method

        foreign_keys = db_utils.get_foreign_keys(dest_table)

        foreign_key_obsid_tables = [tname for tname, colnames in foreign_keys.items() for colname in colnames if colname[0] == 'obsid']
        if len(foreign_key_obsid_tables) == 1:
            foreign_key_obsid_table = foreign_key_obsid_tables[0]
        else:
            foreign_key_obsid_table = dest_table
        for file_column in list(translation_dict.keys()):
            alter_colnames = []
            new_value = None
            # Check if obsid should be set from selection and add an obsid-column if so.
            if isinstance(file_column, Obsids_from_selection):
                selected = utils.get_selected_features_as_tuple()
                if len(selected) != 1:
                    utils.MessagebarAndLog.critical(bar_msg=ru(QCoreApplication.translate('GeneralCsvImportGui', 'Import error, must select 1 obsid')), duration=60)
                    return 'cancel'
                alter_colnames = ['obsid']
                new_value = selected[0]
            elif isinstance(file_column, StaticValue):
                if translation_dict[file_column]:
                    alter_colnames = translation_dict[file_column]
                    new_value = file_column.value
            for alter_colname in alter_colnames:
                if alter_colnames is not None and new_value is not None:
                    try:
                        colindex = file_data[0].index(alter_colname)
                    except ValueError:
                        colindex = len(file_data[0])
                        file_data[0].append(alter_colname)

                    for row in file_data[1:]:
                        if colindex + 1 < len(file_data[0]):
                            row[colindex] = new_value
                        else:
                            row.append(new_value)

                    #[row.insert(obsidindex, selected[0]) if obsidindex + 1 < len(file_data[0]) else row.append(selected[0]) for row in file_data[1:]]
                    del translation_dict[file_column]

                    translation_dict[alter_colname] = [alter_colname]

        columns_factors = self.table_chooser.get_columns_factors_dict()

        #Translate column names and add columns that appear more than once
        file_data = self.translate_and_reorder_file_data(file_data, translation_dict)
        file_data = self.convert_comma_to_points_for_double_columns(file_data, self.tables_columns_info[dest_table])
        if columns_factors:
            file_data = self.multiply_by_factor(file_data, columns_factors)
        file_data = self.remove_preceding_trailing_spaces_tabs(file_data)
        if foreign_key_obsid_table and foreign_key_obsid_table != dest_table and 'obsid' in file_data[0]:
            file_data = utils.filter_nonexisting_values_and_ask(file_data, 'obsid', utils.get_all_obsids(foreign_key_obsid_table), try_capitalize=False)

        file_data = self.reformat_date_time(file_data)

        importer = import_data_to_db.midv_data_importer()
        answer = importer.general_import(dest_table=dest_table, file_data=file_data)
        utils.stop_waiting_cursor()

        if self.close_after_import.isChecked():
            self.close()
    def start_import(self):
        """
        TODO: I have NO IDEA where the dummy parameter is coming from. It gets the value False for some reason!
        :param dummy:
        :return:
        """
        if self.file_data is None:
            raise utils.UsageError(ru(QCoreApplication.translate('GeneralCsvImportGui', 'Error, must select a file first!')))

        translation_dict = self.table_chooser.get_translation_dict()

        file_data = copy.deepcopy(self.file_data)

        goal_table = self.table_chooser.import_method

        foreign_keys = db_utils.get_foreign_keys(goal_table)

        foreign_key_obsid_tables = [tname for tname, colnames in foreign_keys.items() for colname in colnames if colname[0] == 'obsid']
        if len(foreign_key_obsid_tables) == 1:
            foreign_key_obsid_table = foreign_key_obsid_tables[0]
        else:
            foreign_key_obsid_table = goal_table
        for file_column in list(translation_dict.keys()):
            alter_colnames = []
            new_value = None
            # Check if obsid should be set from selection and add an obsid-column if so.
            if isinstance(file_column, Obsids_from_selection):
                selected = utils.get_selected_features_as_tuple()
                if len(selected) != 1:
                    utils.MessagebarAndLog.critical(bar_msg=ru(QCoreApplication.translate('GeneralCsvImportGui', 'Import error, must select 1 obsid')), duration=60)
                    return 'cancel'
                alter_colnames = ['obsid']
                new_value = selected[0]
            elif isinstance(file_column, StaticValue):
                if translation_dict[file_column]:
                    alter_colnames = translation_dict[file_column]
                    new_value = file_column.value
            for alter_colname in alter_colnames:
                if alter_colnames is not None and new_value is not None:
                    try:
                        colindex = file_data[0].index(alter_colname)
                    except ValueError:
                        colindex = len(file_data[0])
                        file_data[0].append(alter_colname)

                    for row in file_data[1:]:
                        if colindex + 1 < len(file_data[0]):
                            row[colindex] = new_value
                        else:
                            row.append(new_value)

                    #[row.insert(obsidindex, selected[0]) if obsidindex + 1 < len(file_data[0]) else row.append(selected[0]) for row in file_data[1:]]
                    del translation_dict[file_column]

                    translation_dict[alter_colname] = [alter_colname]

        columns_factors = self.table_chooser.get_columns_factors_dict()

        #Translate column names and add columns that appear more than once
        file_data = self.translate_and_reorder_file_data(file_data, translation_dict)
        file_data = self.convert_comma_to_points_for_double_columns(file_data, self.tables_columns_info[goal_table])
        if columns_factors:
            file_data = self.multiply_by_factor(file_data, columns_factors)
        file_data = self.remove_preceding_trailing_spaces_tabs(file_data)
        if foreign_key_obsid_table and foreign_key_obsid_table != goal_table and 'obsid' in file_data[0]:
            file_data = utils.filter_nonexisting_values_and_ask(file_data, 'obsid', utils.get_all_obsids(foreign_key_obsid_table), try_capitalize=False)

        file_data = self.reformat_date_time(file_data)

        importer = import_data_to_db.midv_data_importer()
        answer = importer.general_import(goal_table=goal_table, file_data=file_data)
        utils.stop_waiting_cursor()

        if self.close_after_import.isChecked():
            self.close()
    def start_import(self, files, skip_rows_without_water_level, confirm_names, import_all_data, from_date=None, to_date=None):
        """
        """
        utils.start_waiting_cursor()  #show the user this may take a long time...
        parsed_files = []
        for selected_file in files:
            res = self.parse_func(path=selected_file, charset=self.charsetchoosen, skip_rows_without_water_level=skip_rows_without_water_level, begindate=from_date, enddate=to_date)
            if res == 'cancel':
                self.status = True
                utils.stop_waiting_cursor()
                return res
            elif res in ('skip', 'ignore'):
                continue

            try:
                file_data, filename, location = res
            except Exception as e:
                utils.MessagebarAndLog.warning(bar_msg=QCoreApplication.translate('DiverofficeImport', 'Import error, see log message panel'),
                                               log_msg=ru(QCoreApplication.translate('DiverofficeImport', 'File %s could not be parsed. Msg:\n%s'))%(selected_file, str(e)))
                continue
            parsed_files.append((file_data, filename, location))

        if len(parsed_files) == 0:
            utils.MessagebarAndLog.critical(bar_msg=QCoreApplication.translate('DiverofficeImport', "Import Failure: No files imported"""))
            utils.stop_waiting_cursor()
            return

        #Add obsid to all parsed filedatas by asking the user for it.
        filename_location_obsid = [['filename', 'location', 'obsid']]
        filename_location_obsid.extend([[parsed_file[1], parsed_file[2], parsed_file[2]] for parsed_file in parsed_files])

        if confirm_names:
            try_capitalize = False
        else:
            try_capitalize = True

        existing_obsids = utils.get_all_obsids()
        utils.stop_waiting_cursor()
        filename_location_obsid = utils.filter_nonexisting_values_and_ask(file_data=filename_location_obsid, header_value='obsid', existing_values=existing_obsids, try_capitalize=try_capitalize, always_ask_user=confirm_names)
        utils.start_waiting_cursor()

        if len(filename_location_obsid) < 2:
            utils.MessagebarAndLog.warning(bar_msg=QCoreApplication.translate('DiverofficeImport', 'Warning. All files were skipped, nothing imported!'))
            utils.stop_waiting_cursor()
            return False

        filenames_obsid = dict([(x[0], x[2]) for x in filename_location_obsid[1:]])

        parsed_files_with_obsid = []
        for file_data, filename, location in parsed_files:
            if filename in filenames_obsid:
                file_data = list(file_data)
                obsid = filenames_obsid[filename]
                file_data[0].append('obsid')
                [row.append(obsid) for row in file_data[1:]]
                parsed_files_with_obsid.append([file_data, filename, location])
        #Header
        file_to_import_to_db =  [parsed_files_with_obsid[0][0][0]]
        file_to_import_to_db.extend([row for parsed_file in parsed_files_with_obsid for row in parsed_file[0][1:]])

        if not import_all_data:
            file_to_import_to_db = self.filter_dates_from_filedata(file_to_import_to_db, utils.get_last_logger_dates())
        if len(file_to_import_to_db) < 2:
            utils.MessagebarAndLog.info(bar_msg=QCoreApplication.translate('DiverofficeImport', 'No new data existed in the files. Nothing imported.'))
            self.status = 'True'
            utils.stop_waiting_cursor()
            return True

        importer = import_data_to_db.midv_data_importer()
        answer = importer.general_import('w_levels_logger', file_to_import_to_db)

        utils.stop_waiting_cursor()

        if self.close_after_import.isChecked():
            self.close()
Exemple #20
0
    def start_import(self, files, skip_rows_without_water_level, confirm_names, import_all_data, from_date=None, to_date=None):
        """
        """
        PyQt4.QtGui.QApplication.setOverrideCursor(PyQt4.QtGui.QCursor(PyQt4.QtCore.Qt.WaitCursor))  #show the user this may take a long time...
        parsed_files = []
        for selected_file in files:
            res = self.parse_func(path=selected_file, charset=self.charsetchoosen, skip_rows_without_water_level=skip_rows_without_water_level, begindate=from_date, enddate=to_date)
            if res == u'cancel':
                self.status = True
                PyQt4.QtGui.QApplication.restoreOverrideCursor()
                return res
            elif res in (u'skip', u'ignore'):
                continue

            try:
                file_data, filename, location = res
            except Exception as e:
                utils.MessagebarAndLog.warning(bar_msg=QCoreApplication.translate('DiverofficeImport', u'Import error, see log message panel'),
                                               log_msg=ru(QCoreApplication.translate('DiverofficeImport', u'File %s could not be parsed. Msg:\n%s'))%(selected_file, str(e)))
                continue
            parsed_files.append((file_data, filename, location))

        if len(parsed_files) == 0:
            utils.MessagebarAndLog.critical(bar_msg=QCoreApplication.translate('DiverofficeImport', u"Import Failure: No files imported"""))
            PyQt4.QtGui.QApplication.restoreOverrideCursor()
            return

        #Add obsid to all parsed filedatas by asking the user for it.
        filename_location_obsid = [[u'filename', u'location', u'obsid']]
        filename_location_obsid.extend([[parsed_file[1], parsed_file[2], parsed_file[2]] for parsed_file in parsed_files])

        if confirm_names:
            try_capitalize = False
        else:
            try_capitalize = True

        existing_obsids = utils.get_all_obsids()
        filename_location_obsid = utils.filter_nonexisting_values_and_ask(file_data=filename_location_obsid, header_value=u'obsid', existing_values=existing_obsids, try_capitalize=try_capitalize, always_ask_user=confirm_names)

        if len(filename_location_obsid) < 2:
            utils.MessagebarAndLog.warning(bar_msg=QCoreApplication.translate('DiverofficeImport', u'Warning. All files were skipped, nothing imported!'))
            PyQt4.QtGui.QApplication.restoreOverrideCursor()
            return False

        filenames_obsid = dict([(x[0], x[2]) for x in filename_location_obsid[1:]])

        parsed_files_with_obsid = []
        for file_data, filename, location in parsed_files:
            if filename in filenames_obsid:
                file_data = list(file_data)
                obsid = filenames_obsid[filename]
                file_data[0].append(u'obsid')
                [row.append(obsid) for row in file_data[1:]]
                parsed_files_with_obsid.append([file_data, filename, location])
        #Header
        file_to_import_to_db =  [parsed_files_with_obsid[0][0][0]]
        file_to_import_to_db.extend([row for parsed_file in parsed_files_with_obsid for row in parsed_file[0][1:]])

        if not import_all_data:
            file_to_import_to_db = self.filter_dates_from_filedata(file_to_import_to_db, utils.get_last_logger_dates())
        if len(file_to_import_to_db) < 2:
            utils.MessagebarAndLog.info(bar_msg=QCoreApplication.translate('DiverofficeImport', u'No new data existed in the files. Nothing imported.'))
            self.status = 'True'
            PyQt4.QtGui.QApplication.restoreOverrideCursor()
            return True

        importer = import_data_to_db.midv_data_importer()
        answer = importer.general_import(u'w_levels_logger', file_to_import_to_db)

        PyQt4.QtGui.QApplication.restoreOverrideCursor()
        importer.SanityCheckVacuumDB()
        PyQt4.QtGui.QApplication.restoreOverrideCursor()

        if self.close_after_import.isChecked():
            self.close()
 def test_filter_nonexisting_obsids_and_ask_header_capitalize(self, mock_iface):
         file_data = [['obsid', 'ae'], ['a', 'b'], ['2', 'c']]
         existing_obsids = ['A', '2']
         filtered_file_data = utils.filter_nonexisting_values_and_ask(file_data=file_data, header_value='obsid', existing_values=existing_obsids, try_capitalize=True, always_ask_user=False)
         reference_list = [['obsid', 'ae'], ['A', 'b'], ['2', 'c']]
         assert filtered_file_data == reference_list