def test_read_non_unique_data_unexpected_error(self): key_col = 'tre' expected_msg = 'Unexpected non-unique columns found' expected_items = sorted(['lista', 'ett']) with self.assertRaises(MyError) as cm: csv_file_to_dict(self.test_infile.name, key_col, self.test_header, non_unique=False) # ensure error message is the same result_msg, _, result_items = cm.exception.value.partition(':') result_items = strip_list_entries(result_items.split(',')) self.assertEqual(result_msg, expected_msg) self.assertEqual(sorted(result_items), expected_items)
def test_read_list_data(self): key_col = self.test_header.split('|')[1] lists = ('lista', ) expected = { '2': { 'ett': '1', 'lista': ['1', '2', '3', '4', '5'], 'fem': '', 'tre': '3', 'två': '2', 'fyra': '4' }, 'a2': { 'lista': ['a1', 'a2', 'a3', 'a4', 'a5'], 'ett': 'a1', 'fem': 'a5', 'tre': 'a3', 'två': 'a2', 'fyra': 'a4' } } result = csv_file_to_dict(self.test_infile.name, key_col, self.test_header, lists=lists) self.assertDictEqual(result, expected)
def rename(base_dir, sub_cat, in_filename, log_file='move.log'): """ Identify any files to replace and rename them to their commons names. :param base_dir: Path to directory in which replacement image files are found. :param sub_cat: The name of the subdirectory into which processed files should be moved. :param in_filename: The photoAll.csv file filtered to only contain the files to replace. :param log_file: The name of the log file to be created (in base_dir). """ # Load indata in_filename = common.modify_path(base_dir, in_filename) header_check = u'PhoId|PhoObjId|PhoBeschreibungM|PhoAufnahmeortS|' \ u'PhoSwdS|MulId|filnamn|AdrVorNameS|AdrNameS|PhoSystematikS' data = csv.csv_file_to_dict(in_filename, "filnamn", header_check, keep=('PhoSystematikS', 'filnamn'), delimiter='|', codec='utf-16') # reformat the commons filenames url_prefix = u'https://commons.wikimedia.org/wiki/File:' for k, v in data.iteritems(): if v['PhoSystematikS'].startswith(url_prefix): data[k] = v['PhoSystematikS'][len(url_prefix):] else: pywikibot.output("error in indatafile: %s, %s" % (k, v)) # find candidate files candidates = prep.find_files(base_dir, ('.tif', ), subdir=False) # rename the files sub_cat = common.modify_path(base_dir, sub_cat) log_file = common.modify_path(base_dir, log_file) common.create_dir(sub_cat) log = [] for candidate in candidates: base_name = os.path.basename(candidate) if base_name not in data.keys(): log.append('%s not found in csv file' % base_name) continue commons_name = data.pop(base_name) commons_name = common.modify_path(sub_cat, commons_name) os.rename(candidate, commons_name) for k in data.keys(): log.append('%s not found on disk' % k) common.open_and_write_file(log_file, '\n'.join(log), codec='utf-8') pywikibot.output(u'Created %s' % log_file)
def rename(base_dir, sub_cat, in_filename, log_file='move.log'): """ Identify any files to replace and rename them to their commons names. :param base_dir: Path to directory in which replacement image files are found. :param sub_cat: The name of the subdirectory into which processed files should be moved. :param in_filename: The photoAll.csv file filtered to only contain the files to replace. :param log_file: The name of the log file to be created (in base_dir). """ # Load indata in_filename = common.modify_path(base_dir, in_filename) header_check = u'PhoId|PhoObjId|PhoBeschreibungM|PhoAufnahmeortS|' \ u'PhoSwdS|MulId|filnamn|AdrVorNameS|AdrNameS|PhoSystematikS' data = csv.csv_file_to_dict(in_filename, "filnamn", header_check, keep=('PhoSystematikS', 'filnamn'), delimiter='|', codec='utf-16') # reformat the commons filenames url_prefix = u'https://commons.wikimedia.org/wiki/File:' for k, v in data.iteritems(): if v['PhoSystematikS'].startswith(url_prefix): data[k] = v['PhoSystematikS'][len(url_prefix):] else: pywikibot.output("error in indatafile: %s, %s" % (k, v)) # find candidate files candidates = prep.find_files(base_dir, ('.tif', ), subdir=False) # rename the files sub_cat = common.modify_path(base_dir, sub_cat) log_file = common.modify_path(base_dir, log_file) common.create_dir(sub_cat) log = [] for candidate in candidates: base_name = os.path.basename(candidate) if base_name not in data.keys(): log.append('%s not found in csv file' % base_name) continue commons_name = data.pop(base_name) commons_name = common.modify_path(sub_cat, commons_name) os.rename(candidate, commons_name) for k in data.keys(): log.append('%s not found on disk' % k) common.open_and_write_file(log_file, '\n'.join(log), codec='utf-8') pywikibot.output(u'Created %s' % log_file)
def load_data(in_file): """Load csv file. @param in_data: the path to the metadata file @return: dict, list """ header = u'Folder|Original|Commons' key_col = (u'Folder', u'Original') data = csv_methods.csv_file_to_dict(in_file, key_col, header) folders = [] for val in data.values(): folders.append(val[u'Folder']) folders = list(set(folders)) # remove any duplicates return (data, folders)
def load_data(self, in_file): """ Load the provided data (in whichever format) and produce a dict with an entry per file which can be used for further processing. @param in_file: the path to the metadata file @return: dict """ key_col = u'Identifikationsnr' lists = (u'Ämnesord', u'Material', u'Motiv-ämnesord') return csv_methods.csv_file_to_dict(in_file, key_col, EXPECTED_HEADER, non_unique=True, lists=lists, list_delimiter=',')
def base_load_data(self, csv_file, metadata): """ Load and parse the provided csv file. :param csv_file: the filename to load :param metadata: the metadata for the file """ fields, list_columns, key_column = metadata expected_header = self.delimiter.join(fields.keys()) raw_dict = csv_methods.csv_file_to_dict( csv_file, key_column, expected_header, lists=list_columns, delimiter=self.delimiter, list_delimiter=self.list_delimiter) return common.relabel_inner_dicts(raw_dict, fields)
def test_read_data(self): key_col = self.test_header.split('|')[1] expected = { '2': { 'ett': '1', 'lista': '1;2;3;;4;5', 'fem': '', 'tre': '3', 'två': '2', 'fyra': '4' }, 'a2': { 'lista': 'a1;a2; a3 ;a4;a5', 'ett': 'a1', 'fem': 'a5', 'tre': 'a3', 'två': 'a2', 'fyra': 'a4' } } result = csv_file_to_dict(self.test_infile.name, key_col, self.test_header) self.assertDictEqual(result, expected)
def test_read_non_unique_data(self): key_col = 'tre' lists = ('lista', ) expected = { '3': { 'ett': ['1', '2'], 'lista': ['1', '2', '3', '4', '5'], 'tre': '3', 'fyra': '4' }, 'a3': { 'ett': ['a1', 'a2'], 'lista': ['a1', 'a2', 'a3', 'a4', 'a5', 'a5'], 'tre': 'a3', 'fyra': 'a4' } } result = csv_file_to_dict(self.test_infile.name, key_col, self.test_header, lists=lists, non_unique=True) self.assertEqual(deep_sort(result), deep_sort(expected))