def makeHitlist(filenames_file=None): """ Goes through the allowed filenames and builds up a treestructure {directory: [filenames]} as well as a look-up dictionary for filenames to phoId {MulDateiS: {phoMull, filename, ext}} :param filenamesFile: filenames data file :return: dict, dict """ # set defaults unless overridden filenames_file = filenames_file or FILENAMES # load filenames file filenames_header = 'PhoId|MulId|MulPfadS|MulDateiS|filename|ext' filenames = helpers.csvFileToDict(filenames_file, 'PhoId', filenames_header) tree = {} name_to_pho = {} for pho_id, v in filenames.iteritems(): old_name = v['MulDateiS'] path = v['MulPfadS'].replace('\\', os.sep) # windows -> current os if path not in tree.keys(): tree[path] = [] tree[path].append(old_name) name_to_pho[old_name] = {'phoMull': u'%s:%s' % (pho_id, v['MulId']), 'filename': v['filename'], 'ext': v['ext']} return (tree, name_to_pho)
def makePhotoAll(photoAllFile, photo_multi, logFile): """ @toDO: if dupes are found then prompt manual cleanup then re-run makePhotoAll(), That way crash isn't complete. Given the photoAll data file read it and drop any entries without a commons connection. Also Simplify the data :param photoAllFile: path to photoAll data file :param photo_multi: photo_multi dict :param logFile: path to logfile :return: dict """ # often requires manual fixing prior to crunch helpers.verboseInput(u"Confirm that any issues mentioned in the photoAll " u"analysis log have been corrected and the updated " u"photoAll file saved...\n" u"...by pressing enter when done") # setup flog = codecs.open(logFile, 'w', 'utf-8') # logfile output(u"Loading photoAll...") photoAllHeader = 'PhoId|PhoObjId|PhoBeschreibungM|PhoAufnahmeortS|PhoSwdS|' \ 'MulId|AdrVorNameS|AdrNameS|PhoSystematikS' photoAll = helpers.csvFileToDict(photoAllFile, ('PhoId', 'MulId'), photoAllHeader) originalSize = len(photoAll) for k, v in photoAll.items(): link = v['PhoSystematikS'] # drop any entries without files if not link: del photoAll[k] continue # simplify link if '%' in link: link = helpers.urldecode_utf8(link) link = helpers.external_2_internal_link(link, project='wikimedia') link = link[len('[[:commons:File:'):-len(']]')] v['PhoSystematikS'] = link output('PhotoAll reduced from %d to %d entries' % (originalSize, len(photoAll))) # check that none of PhoId from photo_multi occur in photo dupes = [] for phoId in photo_multi.keys(): phoMul = u'%s:%s' % (phoId, photo_multi[phoId]['MulId']) if phoMul in photoAll.keys(): dupes.append(phoMul) if dupes: output(u'Found duplicates between photoAll and photo_multi. ' u'This will most likely mess things up. Check the log at ' u'%s for details.' % logFile) flog.write(u'* duplicates found in photo and photo_all\n' u'phoId:MulId|commonsFile\n') for d in dupes: flog.write('%s|%s\n' % (d, photoAll[d]['PhoSystematikS'])) flog.close() return photoAll
def test_read_write_roundtrip(self): key_col = self.test_header.split('|')[1] read_data = helpers.csvFileToDict(self.test_infile.name, key_col, self.test_header) helpers.dictToCsvFile(self.test_outfile.name, read_data, self.test_header) self.assertEquals(self.test_outfile.read(), self.test_infile.read())
def makePhotoAll(photoAllFile, photo_multi, logFile): """ @toDO: if dupes are found then prompt manual cleanup then re-run makePhotoAll(), That way crash isn't complete. Given the photoAll data file read it and drop any entries without a commons connection. Also Simplify the data :param photoAllFile: path to photoAll data file :param photo_multi: photo_multi dict :param logFile: path to logfile :return: dict """ # often requires manual fixing prior to crunch helpers.verboseInput(u"Confirm that any issues mentioned in the photoAll " u"analysis log have been corrected and the updated " u"photoAll file saved...\n" u"...by pressing enter when done") # setup flog = codecs.open(logFile, 'w', 'utf-8') # logfile output(u"Loading photoAll...") photoAllHeader = 'PhoId|PhoObjId|PhoBeschreibungM|PhoAufnahmeortS|PhoSwdS|' \ 'MulId|AdrVorNameS|AdrNameS|PhoSystematikS' photoAll = helpers.csvFileToDict(photoAllFile, ('PhoId', 'MulId'), photoAllHeader) originalSize = len(photoAll) for k, v in photoAll.items(): link = v['PhoSystematikS'] # drop any entries without files if not link: del photoAll[k] continue # simplify link if '%' in link: link = helpers.urldecode_utf8(link) link = helpers.external_2_internal_link(link, project='wikimedia') link = link[len('[[:commons:File:'):-len(']]')] v['PhoSystematikS'] = link output('PhotoAll reduced from %d to %d entries' % (originalSize, len(photoAll))) # check that none of PhoId from photo_multi occur in photo dupes = [] for phoId in photo_multi.keys(): phoMul = u'%s:%s' % (phoId, photo_multi[phoId]['MulId']) if phoMul in photoAll.keys(): dupes.append(phoMul) if dupes: output(u'Found duplicates between photoAll and photo_multi. ' u'This will most likely mess things up. Check the log at ' u'%s for details.' % logFile) flog.write(u'* duplicates found in photo and photo_all\n' u'phoId:MulId|commonsFile\n') for d in dupes: flog.write('%s|%s\n' % (d, photoAll[d]['PhoSystematikS'])) flog.close() return photoAll
def objDaten_sam(objDatenSamFile, objDaten): """ Adds objDatenSam field to ObjDaten * adding a std_year field * combining all objIds for the same ausId * dropping AobId :param objDatenSamFile: path to ObjDaten-samhörande data file :param objDaten: objDaten dict :return: None (but updates objDaten) """ # setup output(u"Adding ObjDaten-samhörande to ObjDaten") # handle objDatenSam output('\treading ObjDaten_-_samhörande_nr into dictionary... (slow)') objDatenSamHeader = 'OobId|OobObj1ID|OobObj2ID' objDatenSam = helpers.csvFileToDict(objDatenSamFile, 'OobId', objDatenSamHeader) # map object connections output('\tmapping object connections...') objIdConnection = {} for k, v in objDatenSam.iteritems(): objId1 = v['OobObj1ID'] objId2 = v['OobObj2ID'] if objId1 not in objIdConnection.keys(): objIdConnection[objId1] = [] if objId2 not in objIdConnection.keys(): objIdConnection[objId2] = [] objIdConnection[objId1].append(objId2) objIdConnection[objId2].append(objId1) output('\tfound %d connected objIds in %d entries' % (len(objIdConnection), len(objDatenSam))) # clean up connections output('\tremoving dupes, invalids and self...') for objId, connectedIds in objIdConnection.items(): connectedIds = list(set(connectedIds)) # remove dupe if objId in connectedIds: connectedIds.remove(objId) # remove self for conId in connectedIds[:]: # slice allows changes from within loop if conId not in objDaten.keys(): connectedIds.remove(conId) # remove invalid # delete or update if not connectedIds: del objIdConnection[objId] else: objIdConnection[objId] = connectedIds # add to objDaten output('\tadding connections to objDaten...') for k, v in objDaten.iteritems(): objId = v['ObjId'] v['related'] = [] if objId in objIdConnection.keys(): v['related'] = objIdConnection.pop(objId) output(u"...done")
def objDaten_sam(objDatenSamFile, objDaten): """ Adds objDatenSam field to ObjDaten * adding a std_year field * combining all objIds for the same ausId * dropping AobId :param objDatenSamFile: path to ObjDaten-samhörande data file :param objDaten: objDaten dict :return: None (but updates objDaten) """ # setup output(u"Adding ObjDaten-samhörande to ObjDaten") # handle objDatenSam output('\treading ObjDaten_-_samhörande_nr into dictionary... (slow)') objDatenSamHeader = 'OobId|OobObj1ID|OobObj2ID' objDatenSam = helpers.csvFileToDict(objDatenSamFile, 'OobId', objDatenSamHeader) # map object connections output('\tmapping object connections...') objIdConnection = {} for k, v in objDatenSam.iteritems(): objId1 = v['OobObj1ID'] objId2 = v['OobObj2ID'] if objId1 not in objIdConnection.keys(): objIdConnection[objId1] = [] if objId2 not in objIdConnection.keys(): objIdConnection[objId2] = [] objIdConnection[objId1].append(objId2) objIdConnection[objId2].append(objId1) output('\tfound %d connected objIds in %d entries' % (len(objIdConnection), len(objDatenSam))) # clean up connections output('\tremoving dupes, invalids and self...') for objId, connectedIds in objIdConnection.items(): connectedIds = list(set(connectedIds)) # remove dupe if objId in connectedIds: connectedIds.remove(objId) # remove self for conId in connectedIds[:]: # slice allows changes from within loop if conId not in objDaten.keys(): connectedIds.remove(conId) # remove invalid # delete or update if not connectedIds: del objIdConnection[objId] else: objIdConnection[objId] = connectedIds # add to objDaten output('\tadding connections to objDaten...') for k, v in objDaten.iteritems(): objId = v['ObjId'] v['related'] = [] if objId in objIdConnection.keys(): v['related'] = objIdConnection.pop(objId) output(u"...done")
def test_read_data(self): key_col = self.test_header.split('|')[1] expected = { u'2': { u'ett': u'1', u'lista': u'1;2;3;4;5', u'fem': u'5', u'tre': u'3', u'tv\xe5': u'2', u'fyra': u'4'}, u'a2': { u'lista': u'a1;a2;a3;a4;a5', u'ett': u'a1', u'fem': u'a5', u'tre': u'a3', u'tv\xe5': u'a2', u'fyra': u'a4'}} result = helpers.csvFileToDict(self.test_infile.name, key_col, self.test_header) self.assertEquals(result, expected)
def findAllMissing(filenamesFile=FILENAME_FILE, configPath=u'config.json'): """ Goes through the filenames file and checks each name for existence. Missing files are outputted to MISSING_FILES_FILE Existing files are outputted to LSH_EXPORT_FILE :param filenamesFile: path to filenames data file :param configPath: path to config.json file :return: None """ # create targetdirectory if it doesn't exist if not os.path.isdir(POST_DIR): os.mkdir(POST_DIR) # load filenames file filenamesHeader = 'PhoId|MulId|MulPfadS|MulDateiS|filename|ext' filenames = helpers.csvFileToDict(filenamesFile, 'PhoId', filenamesHeader) # identify all Commons filenames files = {} for k, v in filenames.iteritems(): commonsFile = u'File:%s.%s' % (v['filename'], v['ext']) files[commonsFile] = v print u'Found %d filenames' % len(files) # get extra info from Commons comApi = helpers.openConnection(configPath) fileInfos = comApi.getPageInfo(files.keys()) # determine which are present and which are missing missing = {} found = {} prefix = u'https://commons.wikimedia.org/wiki/' for name, info in fileInfos.iteritems(): if name not in files.keys(): print name continue if 'missing' in info.keys(): missing[name] = files[name] else: found[name] = { 'PhoId': files[name]['PhoId'], 'MulId': files[name]['MulId'], 'CommonsFile': '%s%s' % (prefix, name.replace(' ', '_')) } # output files foundHeader = u'PhoId|MulId|CommonsFile' helpers.dictToCsvFile(LSH_EXPORT_FILE, found, foundHeader) helpers.dictToCsvFile(MISSING_FILES_FILE, missing, filenamesHeader) print u'Created %s and %s' % (LSH_EXPORT_FILE, MISSING_FILES_FILE)
def test_read_list_data(self): key_col = self.test_header.split('|')[1] lists = ('lista', ) expected = { u'2': { u'ett': u'1', u'lista': [u'1', u'2', u'3', u'4', u'5'], u'fem': u'5', u'tre': u'3', u'tv\xe5': u'2', u'fyra': u'4'}, u'a2': { u'lista': [u'a1', u'a2', u'a3', u'a4', u'a5'], u'ett': u'a1', u'fem': u'a5', u'tre': u'a3', u'tv\xe5': u'a2', u'fyra': u'a4'}} result = helpers.csvFileToDict(self.test_infile.name, key_col, self.test_header, lists=lists) self.assertEquals(result, expected)
def moveHits(path, filenamesFile=None): """ Goes through the root export directory to find any matching file and moves these to a lower case version of the directory. This flattens out the directory structure whilst making it easy to identify any non-matched files. :param path: path to directory with image file structures :param filenamesFile: filenames data file :return: None """ # set defaults unless overridden filenamesFile = filenamesFile or FILENAMES # Find and move all relevant files tree, name_to_pho = makeHitlist(filenamesFile) subdirs = [] for filename in os.listdir(path): # for LSH all files are in upper case directories filename_path = os.path.join(path, filename) if os.path.isdir(filename_path) and filename.isupper(): subdirs.append(filename_path) for subdir in subdirs: # make a subdir path where (only the) last directory is lower case tmp_path, tmp_dir = os.path.split(subdir) lower_subdir = os.path.join(tmp_path, tmp_dir.lower()) counter, file_num = moveFiles(lower_subdir, tree, name_to_pho, path=subdir) output(u'%s: %d out of %d were hits' % (subdir, counter, file_num)) # load filenames file filenames_header = 'PhoId|MulId|MulPfadS|MulDateiS|filename|ext' old_filenames = helpers.csvFileToDict(filenamesFile, 'PhoId', filenames_header) # Add found extentions to filenames file for phoId, v in old_filenames.iteritems(): old_filename = v['MulDateiS'] if old_filename in name_to_pho.keys(): v['ext'] = name_to_pho[old_filename]['ext'] # overwrite extention # output updated file helpers.dictToCsvFile(filenamesFile, old_filenames, filenames_header) # delete all emptied directories for subdir in subdirs: removeEmptyDirectories(subdir, top=False)
def stichworth_photo(stichwortFile, photo_multi): """ Given the photo-multi data and the stichwort data file add a stichwort id field to photo-multi. Also returns the stichwort data after trimming away any unused info :param stichwortFile: path to stichwort data file :param photo_multi: photo_multi dict :return: dict (and updates photo_multi) """ # setup output(u"Adding stichworth to photo") # handle stichwort output(u'\treading in stichwort...') stichwortHeader = 'PstId|PhoId|StiBezeichnungS|StiSynonymS' stichwort = helpers.csvFileToDict(stichwortFile, 'PstId', stichwortHeader) originalSize = len(stichwort) # match each phoId to several stichId # removing any entries with invalid phoIds photoStichConnection = {} for k, v in stichwort.items(): phoId = v['PhoId'] pstId = v['PstId'] if phoId in photo_multi.keys(): if phoId not in photoStichConnection.keys(): photoStichConnection[phoId] = set([]) photoStichConnection[phoId].add(pstId) else: del stichwort[k] output('\tstichwort trimmed from %d to %d, found %d phoId' % (originalSize, len(stichwort), len(photoStichConnection))) # add stichId to photo_multi for k, v in photo_multi.iteritems(): phoId = v['PhoId'] v['PstId'] = [] if phoId in photoStichConnection.keys(): v['PstId'] = list(photoStichConnection.pop(phoId)) # confirm and return output(u"...done") return stichwort
def stichworth_photo(stichwortFile, photo_multi): """ Given the photo-multi data and the stichwort data file add a stichwort id field to photo-multi. Also returns the stichwort data after trimming away any unused info :param stichwortFile: path to stichwort data file :param photo_multi: photo_multi dict :return: dict (and updates photo_multi) """ # setup output(u"Adding stichworth to photo") # handle stichwort output(u'\treading in stichwort...') stichwortHeader = 'PstId|PhoId|StiBezeichnungS|StiSynonymS' stichwort = helpers.csvFileToDict(stichwortFile, 'PstId', stichwortHeader) originalSize = len(stichwort) # match each phoId to several stichId # removing any entries with invalid phoIds photoStichConnection = {} for k, v in stichwort.items(): phoId = v['PhoId'] pstId = v['PstId'] if phoId in photo_multi.keys(): if phoId not in photoStichConnection.keys(): photoStichConnection[phoId] = set([]) photoStichConnection[phoId].add(pstId) else: del stichwort[k] output('\tstichwort trimmed from %d to %d, found %d phoId' % (originalSize, len(stichwort), len(photoStichConnection))) # add stichId to photo_multi for k, v in photo_multi.iteritems(): phoId = v['PhoId'] v['PstId'] = [] if phoId in photoStichConnection.keys(): v['PstId'] = list(photoStichConnection.pop(phoId)) # confirm and return output(u"...done") return stichwort
def photo_ObjDaten(photo_multi, photoAll, photoObjDatenFile, objDatenFile, logFile): """ Given the photo_multi data and the phoObjDaten + objDaten data files any additional relevant ObjIds are added to the PhoObjId field of the photo_multi dict, this field is also converted to a list. Also returns objDaten for later use :param photo_multi: photo_multi dict :param photoAll: photoAll dict :param photoObjDatenFile: path to phoObjDaten data file :param objDatenFile: path to objDaten data file :param logFile: path to logfile :return: dict (and updates photo_multi) """ # setup flog = codecs.open(logFile, 'w', 'utf-8') # logfile output(u"Combining all ObjId into the photo file...") # handle objDaten output(u'\treading in objDaten.. (takes a while)') objDatenHeader = 'ObjId|ObjKueId|AufId|AufAufgabeS|ObjTitelOriginalS|' \ 'ObjTitelWeitereM|ObjInventarNrS|ObjInventarNrSortiertS|' \ 'ObjReferenzNrS|ObjDatierungS|ObjJahrVonL|ObjJahrBisL|' \ 'ObjSystematikS|ObjFeld01M|ObjFeld02M|ObjFeld03M|' \ 'ObjFeld06M|ObjReserve01M' objDaten = helpers.csvFileToDict(objDatenFile, 'ObjId', objDatenHeader) # match each objInvNr to several objId objInvNr2ObjId = {} # old oDict output(u'\tfinding objInvNr connections...') for k, v in objDaten.iteritems(): objId = v['ObjId'] objInvNr = v['ObjInventarNrS'] if not objInvNr: continue if objInvNr not in objInvNr2ObjId.keys(): objInvNr2ObjId[objInvNr] = [] objInvNr2ObjId[objInvNr].append(objId) output('\tFound %d objInvNr connections in %d objects' % (len(objInvNr2ObjId), len(objDaten))) # handle photoObjDaten photoObjDatenHeader = 'PhmId|AufId|AufAufgabeS|MulId|PhoId|ObjInvNrS' photoObjDaten = helpers.csvFileToDict(photoObjDatenFile, 'PhmId', photoObjDatenHeader, keep=('PhoId', 'ObjInvNrS')) # match each phoId to several objId via the ObjInvNr output(u'\tfinding photo-object connections...') photoObjConnections = {} skipped = [] # ObjInvNr not in ObjDaten for k, v in photoObjDaten.iteritems(): objInvNr = v['ObjInvNrS'] phoId = v['PhoId'] if not objInvNr: continue if objInvNr not in objInvNr2ObjId.keys(): skipped.append(objInvNr) continue if phoId not in photoObjConnections.keys(): photoObjConnections[phoId] = [] photoObjConnections[phoId] += objInvNr2ObjId[objInvNr] output('\tFound %d connected photos in %d photoObjDaten entries' % (len(photoObjConnections), len(photoObjDaten))) # add to photo_multi and photoAll photoDicts = (photo_multi, photoAll) allBadObjId = [] for pDict in photoDicts: for k, v in pDict.iteritems(): phoId = v['PhoId'] objIds = [] if phoId not in photoObjConnections.keys(): if v['PhoObjId']: objIds.append(v['PhoObjId']) else: # combine relevant objIds objIds = photoObjConnections.pop(phoId) # new connections if v['PhoObjId']: objIds.append(v['PhoObjId']) # old connection objIds = list(set(objIds)) # remove dupes # check that all of these actually exists (old realObjOnly()) # and remove otherwise badObjId = [] for objId in objIds: if objId not in objDaten.keys(): badObjId.append(objId) if badObjId: allBadObjId += badObjId for badId in badObjId: objIds.remove(badId) # set new value v['PhoObjId'] = objIds # log any skipped ObjInvNr if skipped: skipped = list(set(skipped)) # remove dupes output(u"\tthere were %d skipped ObjInvNr, see log (%s)" % (len(skipped), logFile)) flog.write(u'*Unknown objInvs, i.e. ObjInvNrS in photoObjDaten ' u'without a match in ObjDaten\n') flog.write(u'%s\n' % ', '.join(skipped)) # log any bad objId if allBadObjId: output('\tI found some bad objIds. Check the %s' % logFile) allBadObjId = list(set(allBadObjId)) # remove dupes flog.write(u'* objIds in photo but not in objDaten\n') flog.write(u'%s\n' % ', '.join(allBadObjId)) # trim objDaten trimObjDaten(objDaten, photo_multi, photoAll) # confirm and return output(u"...done") flog.close() return objDaten
def makePhoto_multi(photoFile, multiFile, logFile, tmpFile): """ Given the photo and multimedia data this combines the two into one dict :param photoFile: path to photo data file :param multiFile: path to multimedia data file :param logFile: path to logfile :param tmpFile: path to temporary file :return: dict """ # setup flog = codecs.open(logFile, 'w', 'utf-8') # logfile output(u"Combining photo and multimedia file for unique files...") pathToTrim = u'R:\web\hires\\' tmpHeader = 'PhoId|MulId|PhoObjId|PhoBeschreibungM|PhoAufnahmeortS|' \ 'PhoSwdS|AdrVorNameS|AdrNameS|PhoSystematikS|MulPfadS|' \ 'MulDateiS|MulExtentS' # handle multimedia multiHeader = 'MulId|MulPhoId|MulPfadS|MulDateiS|MulExtentS' multi = helpers.csvFileToDict(multiFile, 'MulId', multiHeader) # check that filename is unique flog.write('* Same files used by different PhoId, format is PhoId/MulId\n') logged = False namelist = [] mulPhoIdList = [] for k, v in multi.iteritems(): name = u'%s\\%s.%s' % (v['MulPfadS'], v['MulDateiS'], v['MulExtentS']) if name in namelist: logged = True flog.write('%s/%s\n' % (v['MulPhoId'], v['MullId'])) else: mulPhoIdList.append(v['MulPhoId']) namelist.append(name) output(u'\tmultimedia: %d' % len(multi)) if not logged: flog.write(u'None =)\n') # handle photo # @toDO add duplicate check to cleanup script photoHeader = 'PhoId|PhoObjId|PhoBeschreibungM|PhoAufnahmeortS|PhoSwdS|' \ 'MulId|AdrVorNameS|AdrNameS|PhoSystematikS' photo = helpers.csvFileToDict(photoFile, 'PhoId', photoHeader) output(u'\tphoto: %d' % len(photo)) # combine combined = {} flog.write(u'* unused rows in multimedia\n') logged = False for k, v in multi.iteritems(): phoId = v['MulPhoId'] mulId = v['MulId'] v['MulPfadS'] = v['MulPfadS'].replace(pathToTrim, u'') # trim filepath v['MulExtentS'] = u'' # MulExtentS is always wrong if phoId not in photo.keys(): logged = True flog.write(u'%s\n' % v) elif not photo[phoId]['MulId'] == v['MulId']: raise MyError("phoId matched but to wrong mulId: p:%s m_found:%s, " "m_expected %s" % (phoId, photo[phoId]['MulId'], mulId)) else: del v['MulPhoId'], v['MulId'] combo = photo.pop(phoId) # move out of photo combo.update(v) # add contents from multi combined[phoId] = combo if not logged: flog.write(u'None =)\n') # log any unused rows in photo flog.write(u'* unused rows in photo\n') logged = False for k, v in photo.iteritems(): logged = True flog.write(u'%s\n' % v) if not logged: flog.write(u'None =)\n') flog.close() output(u"...done") # check if anything needs to be manually handled output(u"Read the log (%s)" % logFile) combined = helpers.promptManualUpdate(combined, tmpFile, tmpHeader, 'PhoId') return combined
def mulMass_add(objMassFile, objMultipleFile, objDaten): """ Given the objMass and the objMultiple data file and the objDaten data add an objMass and an objMultiple field to objDaten. Also returns the objMass and objMultiple data after * removing any entries where objId not in objDaten :param objMassFile: path to objMass data file :param objMultipleFile: path to objMultiple data file :param objDaten: objDaten dict :return: dict, dict (and updates objDaten) """ # setup output(u"Putting objMultiple and objMass into objDaten...") # handle objMass output('\treading ObjMass into dictionary... (yes this takes some time)') objMassHeader = 'ObmId|ObmObjId|ObmTypMasseS|ObmMasseS|' \ 'ObjAufId|AufAufgabeS' objMass = helpers.csvFileToDict(objMassFile, 'ObmId', objMassHeader) originalSize = len(objMass) # invert to get per objId connections # and remove any entries where objId not in objDaten output('\tinverting objMass...') objIdMassConnection = {} for k, v in objMass.items(): obmId = v['ObmId'] objId = v['ObmObjId'] if objId not in objDaten.keys(): del objMass[k] continue if objId not in objIdMassConnection.keys(): objIdMassConnection[objId] = set([]) objIdMassConnection[objId].add(obmId) output('\tobjMass: reduced from %d to %d' % (originalSize, len(objMass))) # handle objMultiple output('\treading ObjMultiple into dictionary... (as does this)') objMultipleHeader = 'OmuId|OmuObjId|OmuTypS|OmuBemerkungM|OmuInhalt01M|' \ 'ObjInventarNrS|ObjAufId|AufAufgabeS' objMultiple = helpers.csvFileToDict(objMultipleFile, 'OmuId', objMultipleHeader) originalSize = len(objMultiple) # invert to get per objId connections # and remove any entries where objId not in objDaten output('\tinverting objIdMultiple...') objIdMultipleConnection = {} for k, v in objMultiple.items(): omulId = v['OmuId'] objId = v['OmuObjId'] if objId not in objDaten.keys(): del objMultiple[k] continue if objId not in objIdMultipleConnection.keys(): objIdMultipleConnection[objId] = set([]) objIdMultipleConnection[objId].add(omulId) output('\tobjMultiple: reduced from %d to %d' % (originalSize, len(objMultiple))) # adding ObjMul and ObjMass id to objDaten output('\tadding ObjMul and ObjMass id to objDaten... (and this)') for k, v in objDaten.iteritems(): objId = v['ObjId'] v['massId'] = [] v['mulId'] = [] if objId in objIdMassConnection.keys(): v['massId'] = list(objIdMassConnection.pop(objId)) if objId in objIdMultipleConnection.keys(): v['mulId'] = list(objIdMultipleConnection.pop(objId)) output(u"...done") return objMass, objMultiple
def kuenstler_objDaten(kuenstlerFile, objDaten, logFile): """ Given the kuenstler data file and the objDaten data add a kuenstler id field to objDaten. Also returns the kuenstler data after * removing certain irrelevant roles and dummy entries * combining all objIds for the same kueId * standardising years * dropping a lot of unneeded fields :param kuenstlerFile: path to kuenstler data file :param objDaten: objDaten dict :param logFile: path to logfile :return: dict (and updates objDaten) """ # setup flog = codecs.open(logFile, 'w', 'utf-8') # logfile output(u"Crunching kuenstler...") dummyNames = (u'ingen uppgift', ) badRoles = (u'Leverantör', u'Auktion', u'Förmedlare', u'Givare', u'Återförsäljare', u'Konservator') badRoleCmts = (u'Förpaktare, kontrollör', u'av kopia') droppedFields = ('OkuId', 'ObjAufId', 'AufAufgabeS', 'OkuArtS', 'OkuFunktionS', 'OkuValidierungS', 'KudArtS', 'MulId', 'PhoId') # handle kuenstler kuenstlerHeader = 'OkuId|ObjId|ObjAufId|AufAufgabeS|KueId|KueVorNameS|' \ 'KueNameS|OkuArtS|OkuFunktionS|OkuValidierungS|KudArtS|' \ 'KudDatierungS|KudJahrVonL|KudJahrBisL|KudOrtS|KudLandS|' \ 'KueFunktionS|MulId|PhoId' kuenstler = helpers.csvFileToDict(kuenstlerFile, ('OkuId', 'MulId'), kuenstlerHeader) originalSize = len(kuenstler) # collect all kueId and drop any with invalid title or role # also invert to get per objId connections # @toDO: Is keeping objId in kuenstler really needed? # Otherwise populate objIdConnection here foundKueId = {} objIdConnection = {} for k, v in kuenstler.items(): # allow removing entries from within loop kueId = v['KueId'] objId = v['ObjId'] fName = v['KueVorNameS'] lName = v['KueNameS'] role = v['OkuArtS'] roleCmt = v['OkuFunktionS'] # filter out any undesired entries if role in badRoles or \ roleCmt in badRoleCmts or \ len(fName) + len(lName) == 0 or \ lName in dummyNames: del kuenstler[k] continue # send unique role/kueId combo for objid kueCombo = u'%s:%s:%s' % (role, roleCmt, kueId) if objId not in objIdConnection.keys(): objIdConnection[objId] = set([]) objIdConnection[objId].add(kueCombo) # keep only one entry per unique kueId if kueId not in foundKueId.keys(): # keep this entry foundKueId[kueId] = k kuenstler[k]['ObjId'] = set([objId, ]) else: # keep only objId part of this entry kuenstler[foundKueId[kueId]]['ObjId'].add(objId) del kuenstler[k] output('\tkueIds: reduced from %d to %d' % (originalSize, len(kuenstler))) # add to objDaten output('\tadding kueId to objDaten...') for k, v in objDaten.iteritems(): objId = v['ObjId'] v['role:roleCmt:kueId'] = [] if objId in objIdConnection.keys(): v['role:roleCmt:kueId'] = list(objIdConnection.pop(objId)) # further cleanup of kuenstler # correcting ort/land entries # stripping years from name # dropping a bunch of fields output('\tfurther cleanup of kuenstler...') for k, v in kuenstler.iteritems(): land = v['KudOrtS'] # missnamed in original database ort = v['KudLandS'] # missnamed in original database lName = v['KueNameS'] bYear = v['KudJahrVonL'] dYear = v['KudJahrBisL'] objIds = v['ObjId'] # correct missnaming in original database v['KudOrtS'] = ort v['KudLandS'] = land # convert set to list v['ObjId'] = list(objIds) # take yearinfo out of name, and store in year lName, bYear, dYear, log = extractKuenstlerYear(lName, bYear, dYear) if log: flog.write(log) v['KueNameS'] = lName v['KudJahrVonL'] = bYear v['KudJahrBisL'] = dYear for field in droppedFields: del v[field] flog.close() output(u"...done") return kuenstler
def mulMass_add(objMassFile, objMultipleFile, objDaten): """ Given the objMass and the objMultiple data file and the objDaten data add an objMass and an objMultiple field to objDaten. Also returns the objMass and objMultiple data after * removing any entries where objId not in objDaten :param objMassFile: path to objMass data file :param objMultipleFile: path to objMultiple data file :param objDaten: objDaten dict :return: dict, dict (and updates objDaten) """ # setup output(u"Putting objMultiple and objMass into objDaten...") # handle objMass output('\treading ObjMass into dictionary... (yes this takes some time)') objMassHeader = 'ObmId|ObmObjId|ObmTypMasseS|ObmMasseS|' \ 'ObjAufId|AufAufgabeS' objMass = helpers.csvFileToDict(objMassFile, 'ObmId', objMassHeader) originalSize = len(objMass) # invert to get per objId connections # and remove any entries where objId not in objDaten output('\tinverting objMass...') objIdMassConnection = {} for k, v in objMass.items(): obmId = v['ObmId'] objId = v['ObmObjId'] if objId not in objDaten.keys(): del objMass[k] continue if objId not in objIdMassConnection.keys(): objIdMassConnection[objId] = set([]) objIdMassConnection[objId].add(obmId) output('\tobjMass: reduced from %d to %d' % (originalSize, len(objMass))) # handle objMultiple output('\treading ObjMultiple into dictionary... (as does this)') objMultipleHeader = 'OmuId|OmuObjId|OmuTypS|OmuBemerkungM|OmuInhalt01M|' \ 'ObjInventarNrS|ObjAufId|AufAufgabeS' objMultiple = helpers.csvFileToDict(objMultipleFile, 'OmuId', objMultipleHeader) originalSize = len(objMultiple) # invert to get per objId connections # and remove any entries where objId not in objDaten output('\tinverting objIdMultiple...') objIdMultipleConnection = {} for k, v in objMultiple.items(): omulId = v['OmuId'] objId = v['OmuObjId'] if objId not in objDaten.keys(): del objMultiple[k] continue if objId not in objIdMultipleConnection.keys(): objIdMultipleConnection[objId] = set([]) objIdMultipleConnection[objId].add(omulId) output('\tobjMultiple: reduced from %d to %d' % (originalSize, len(objMultiple))) # adding ObjMul and ObjMass id to objDaten output('\tadding ObjMul and ObjMass id to objDaten... (and this)') for k, v in objDaten.iteritems(): objId = v['ObjId'] v['massId'] = [] v['mulId'] = [] if objId in objIdMassConnection.keys(): v['massId'] = list(objIdMassConnection.pop(objId)) if objId in objIdMultipleConnection.keys(): v['mulId'] = list(objIdMultipleConnection.pop(objId)) output(u"...done") return objMass, objMultiple
def makeDescriptions(photoFile, objDatenFile, logFile): """ Given the photo and objDaten data this uses the two generate descriptions. Also returns photo for later use :param photoFile: path to photo data file :param multiFile: path to multimedia data file :param logFile: path to logfile :return: dict, dict """ # setup flog = codecs.open(logFile, 'w', 'utf-8') # logfile # load input files photoHeader = 'PhoId|MulId|PhoObjId|PhoBeschreibungM|PhoAufnahmeortS|' \ 'PhoSwdS|AdrVorNameS|AdrNameS|PhoSystematikS|MulPfadS|' \ 'MulDateiS|MulExtentS|PstId|same_PhoId|same_object' photo = helpers.csvFileToDict(photoFile, 'PhoId', photoHeader, lists=('PhoObjId', )) objDatenHeader = 'ObjId|ObjKueId|AufId|AufAufgabeS|ObjTitelOriginalS|' \ 'ObjTitelWeitereM|ObjInventarNrS|ObjInventarNrSortiertS|' \ 'ObjReferenzNrS|ObjDatierungS|ObjJahrVonL|ObjJahrBisL|' \ 'ObjSystematikS|ObjFeld01M|ObjFeld02M|ObjFeld03M|ObjFeld06M|' \ 'ObjReserve01M|ausId|related|ergId|role:roleCmt:kueId|' \ 'mulId|massId' objDaten = helpers.csvFileToDict(objDatenFile, 'ObjId', objDatenHeader) # start process skipLog = [] # no photoDescr, no objectDescr manyLog = [] # no photoDescr, many objects noHopeLog = [] # no photoDescr, no objects descriptions = {} uniques = set([]) # unique filenames for k, v in photo.iteritems(): phoId = v['PhoId'] objIds = v['PhoObjId'] museum = v['PhoSwdS'] if not museum: museum = u'LSH' phoBes = getDescFromPhoBes(v['PhoBeschreibungM']) if not phoBes: # try to get description from object if len(objIds) == 1 and objIds[0]: # exactly one object phoBes = getDescFromObj(objDaten[objIds[0]]) if not phoBes: # failed to make a description from the object skipLog.append(phoId) elif len(objIds) > 1: # multiple objects manyLog.append(phoId) else: # no objects noHopeLog.append(phoId) if phoBes: filename = u'%s - %s - %s' % (phoBes, museum, phoId) descriptions[phoId] = {'descr': phoBes, 'filename': filename} uniques.add(filename) # check uniqueness if len(uniques) != len(descriptions): output(u'Descriptions are not unique!!!!: %d were duplicate' % (len(uniques) - len(descriptions))) # output logs if skipLog: flog.write('* No-objectDescr and No-photoDescr (phoIds)\n') flog.write('%s\n' % '\n'.join(skipLog)) if manyLog: flog.write('* Many objects and No-photoDescr (phoIds)\n') flog.write('%s\n' % '\n'.join(manyLog)) if noHopeLog: flog.write('* No-objects and No-photoDescr (phoIds)\n') flog.write('%s\n' % '\n'.join(noHopeLog)) # wrap up output(u'Processed %d images out of which %d has some type of problem. ' u'See log (%s) for more info.' % (len(photo), len(photo) - len(descriptions), logFile)) flog.close() return descriptions, photo
def ereignis_objDaten(ereignisFile, objDaten, logFile): """ Given the ereignis data file and the objDaten data add a ereignis id field to objDaten. Also returns the ereignis data after * combining all objIds for the same ergId * dropping EroId :param ereignisFile: path to eregnis data file :param objDaten: objDaten dict :param logFile: path to logfile :return: dict (and updates objDaten) """ # setup flog = codecs.open(logFile, 'w', 'utf-8') # logfile output(u"Trimming eregnis and adding eregnis to ObjDaten...") # handle eregnis ereignisHeader = 'EroId|ErgId|EroObjId|ErgKurztitelS|ErgArtS' ereignis = helpers.csvFileToDict(ereignisFile, 'EroId', ereignisHeader) originalSize = len(ereignis) # collect all ergId and drop any with invalid title # @toDO: Is keeping objId in eregnis really needed? # Otherwise populate objIdConnection here foundErgId = {} for k, v in ereignis.items(): # allow removing entries from within loop ergId = v['ErgId'] objId = v['EroObjId'] title = v['ErgKurztitelS'] if not title: # remove empty del ereignis[k] elif ergId not in foundErgId.keys(): # keep this entry foundErgId[ergId] = k ereignis[k]['EroObjId'] = set([ objId, ]) ereignis[k].pop('EroId') # drop unnecessary id else: # keep only objId part of this entry ereignis[foundErgId[ergId]]['EroObjId'].add(objId) del ereignis[k] output('\tergIds: reduced from %d to %d' % (originalSize, len(ereignis))) # handle urls in ereignis and convert set to list for k, v in ereignis.iteritems(): objIds = v['EroObjId'] url = v['ErgArtS'] # convert set to list v['EroObjId'] = list(objIds) # handle urls if u'%' in url: url = helpers.urldecode_utf8(url) # convert external links to internal if 'wikipedia' in url: url = helpers.external_2_internal_link(url) elif url: flog.write(u'weird url: %s\n' % url) v['ErgArtS'] = url # invert to get per objId connections objIdConnection = {} for k, v in ereignis.iteritems(): ergId = v['ErgId'] objIds = v['EroObjId'] for objId in objIds: if objId not in objIdConnection.keys(): objIdConnection[objId] = [] objIdConnection[objId].append(ergId) # add to objDaten output('\tadding ergId to objDaten...') for k, v in objDaten.iteritems(): objId = v['ObjId'] v['ergId'] = [] if objId in objIdConnection.keys(): v['ergId'] = objIdConnection.pop(objId) flog.close() output(u"...done") return ereignis
def photo_ObjDaten(photo_multi, photoAll, photoObjDatenFile, objDatenFile, logFile): """ Given the photo_multi data and the phoObjDaten + objDaten data files any additional relevant ObjIds are added to the PhoObjId field of the photo_multi dict, this field is also converted to a list. Also returns objDaten for later use :param photo_multi: photo_multi dict :param photoAll: photoAll dict :param photoObjDatenFile: path to phoObjDaten data file :param objDatenFile: path to objDaten data file :param logFile: path to logfile :return: dict (and updates photo_multi) """ # setup flog = codecs.open(logFile, 'w', 'utf-8') # logfile output(u"Combining all ObjId into the photo file...") # handle objDaten output(u'\treading in objDaten.. (takes a while)') objDatenHeader = 'ObjId|ObjKueId|AufId|AufAufgabeS|ObjTitelOriginalS|' \ 'ObjTitelWeitereM|ObjInventarNrS|ObjInventarNrSortiertS|' \ 'ObjReferenzNrS|ObjDatierungS|ObjJahrVonL|ObjJahrBisL|' \ 'ObjSystematikS|ObjFeld01M|ObjFeld02M|ObjFeld03M|' \ 'ObjFeld06M|ObjReserve01M' objDaten = helpers.csvFileToDict(objDatenFile, 'ObjId', objDatenHeader) # match each objInvNr to several objId objInvNr2ObjId = {} # old oDict output(u'\tfinding objInvNr connections...') for k, v in objDaten.iteritems(): objId = v['ObjId'] objInvNr = v['ObjInventarNrS'] if not objInvNr: continue if objInvNr not in objInvNr2ObjId.keys(): objInvNr2ObjId[objInvNr] = [] objInvNr2ObjId[objInvNr].append(objId) output('\tFound %d objInvNr connections in %d objects' % (len(objInvNr2ObjId), len(objDaten))) # handle photoObjDaten photoObjDatenHeader = 'PhmId|AufId|AufAufgabeS|MulId|PhoId|ObjInvNrS' photoObjDaten = helpers.csvFileToDict(photoObjDatenFile, 'PhmId', photoObjDatenHeader, keep=('PhoId', 'ObjInvNrS')) # match each phoId to several objId via the ObjInvNr output(u'\tfinding photo-object connections...') photoObjConnections = {} skipped = [] # ObjInvNr not in ObjDaten for k, v in photoObjDaten.iteritems(): objInvNr = v['ObjInvNrS'] phoId = v['PhoId'] if not objInvNr: continue if objInvNr not in objInvNr2ObjId.keys(): skipped.append(objInvNr) continue if phoId not in photoObjConnections.keys(): photoObjConnections[phoId] = [] photoObjConnections[phoId] += objInvNr2ObjId[objInvNr] output('\tFound %d connected photos in %d photoObjDaten entries' % (len(photoObjConnections), len(photoObjDaten))) # add to photo_multi and photoAll photoDicts = (photo_multi, photoAll) allBadObjId = [] for pDict in photoDicts: for k, v in pDict.iteritems(): phoId = v['PhoId'] objIds = [] if phoId not in photoObjConnections.keys(): if v['PhoObjId']: objIds.append(v['PhoObjId']) else: # combine relevant objIds objIds = photoObjConnections.pop(phoId) # new connections if v['PhoObjId']: objIds.append(v['PhoObjId']) # old connection objIds = list(set(objIds)) # remove dupes # check that all of these actually exists (old realObjOnly()) # and remove otherwise badObjId = [] for objId in objIds: if objId not in objDaten.keys(): badObjId.append(objId) if badObjId: allBadObjId += badObjId for badId in badObjId: objIds.remove(badId) # set new value v['PhoObjId'] = objIds # log any skipped ObjInvNr if skipped: skipped = list(set(skipped)) # remove dupes output(u"\tthere were %d skipped ObjInvNr, see log (%s)" % (len(skipped), logFile)) flog.write(u'*Unknown objInvs, i.e. ObjInvNrS in photoObjDaten ' u'without a match in ObjDaten\n') flog.write(u'%s\n' % ', '.join(skipped)) # log any bad objId if allBadObjId: output('\tI found some bad objIds. Check the %s' % logFile) allBadObjId = list(set(allBadObjId)) # remove dupes flog.write(u'* objIds in photo but not in objDaten\n') flog.write(u'%s\n' % ', '.join(allBadObjId)) # trim objDaten trimObjDaten(objDaten, photo_multi, photoAll) # confirm and return output(u"...done") flog.close() return objDaten
def makePhoto_multi(photoFile, multiFile, logFile, tmpFile): """ Given the photo and multimedia data this combines the two into one dict :param photoFile: path to photo data file :param multiFile: path to multimedia data file :param logFile: path to logfile :param tmpFile: path to temporary file :return: dict """ # setup flog = codecs.open(logFile, 'w', 'utf-8') # logfile output(u"Combining photo and multimedia file for unique files...") pathToTrim = u'R:\web\hires\\' tmpHeader = 'PhoId|MulId|PhoObjId|PhoBeschreibungM|PhoAufnahmeortS|' \ 'PhoSwdS|AdrVorNameS|AdrNameS|PhoSystematikS|MulPfadS|' \ 'MulDateiS|MulExtentS' # handle multimedia multiHeader = 'MulId|MulPhoId|MulPfadS|MulDateiS|MulExtentS' multi = helpers.csvFileToDict(multiFile, 'MulId', multiHeader) # check that filename is unique flog.write('* Same files used by different PhoId, format is PhoId/MulId\n') logged = False namelist = [] mulPhoIdList = [] for k, v in multi.iteritems(): name = u'%s\\%s.%s' % (v['MulPfadS'], v['MulDateiS'], v['MulExtentS']) if name in namelist: logged = True flog.write('%s/%s\n' % (v['MulPhoId'], v['MullId'])) else: mulPhoIdList.append(v['MulPhoId']) namelist.append(name) output(u'\tmultimedia: %d' % len(multi)) if not logged: flog.write(u'None =)\n') # handle photo # @toDO add duplicate check to cleanup script photoHeader = 'PhoId|PhoObjId|PhoBeschreibungM|PhoAufnahmeortS|PhoSwdS|' \ 'MulId|AdrVorNameS|AdrNameS|PhoSystematikS' photo = helpers.csvFileToDict(photoFile, 'PhoId', photoHeader) output(u'\tphoto: %d' % len(photo)) # combine combined = {} flog.write(u'* unused rows in multimedia\n') logged = False for k, v in multi.iteritems(): phoId = v['MulPhoId'] mulId = v['MulId'] v['MulPfadS'] = v['MulPfadS'].replace(pathToTrim, u'') # trim filepath v['MulExtentS'] = u'' # MulExtentS is always wrong if phoId not in photo.keys(): logged = True flog.write(u'%s\n' % v) elif not photo[phoId]['MulId'] == v['MulId']: raise MyError("phoId matched but to wrong mulId: p:%s m_found:%s, " "m_expected %s" % (phoId, photo[phoId]['MulId'], mulId)) else: del v['MulPhoId'], v['MulId'] combo = photo.pop(phoId) # move out of photo combo.update(v) # add contents from multi combined[phoId] = combo if not logged: flog.write(u'None =)\n') # log any unused rows in photo flog.write(u'* unused rows in photo\n') logged = False for k, v in photo.iteritems(): logged = True flog.write(u'%s\n' % v) if not logged: flog.write(u'None =)\n') flog.close() output(u"...done") # check if anything needs to be manually handled output(u"Read the log (%s)" % logFile) combined = helpers.promptManualUpdate(combined, tmpFile, tmpHeader, 'PhoId') return combined
def ausstellung_objDaten(austellungFile, objDaten): """ Given the austellung data file and the objDaten data add a austellung id field to objDaten. Also returns the austellung data after * adding a std_year field * combining all objIds for the same ausId * dropping AobId :param austellungFile: path to austellung data file :param objDaten: objDaten dict :return: dict (and updates objDaten) """ # often requires manual fixing prior to crunch helpers.verboseInput(u"Confirm that any year formatting issues mentioned " u"in the analysis log have been corrected and the " u"updated ausstellung file saved...\n" u"...by pressing enter when done") # setup dummyTitles = ( u'reparation', u'utställning', u'lån för undersökning', u'OBS! Testpost för admin - utställning, export wikimedia commons', u'lån till Frankrike 1947', u'test karin 20100520', u'test 20100629 (en post skapad för administrativa tester)', u'tennföremål 8 st till Strömsholm', u'utlån f justering av urverk') output(u"Trimming ausstellung and adding ausstellung to ObjDaten...") # handle ausstellung austellungHeader = 'AobId|AusId|AusTitelS|AusOrtS|AusJahrS|AusDatumVonD|' \ 'AusDatumBisD|AobObjId|AufAufgabeS' austellung = helpers.csvFileToDict(austellungFile, 'AobId', austellungHeader) originalSize = len(austellung) # collect all ausId and drop any with invalid title # @toDO: Is keeping objId in austellung really needed? # Otherwise populate objIdConnection here foundAusId = {} for k, v in austellung.items(): # allow removing entries from within loop ausId = v['AusId'] objId = v['AobObjId'] title = v['AusTitelS'] if not title or title in dummyTitles: # remove empty/dummy del austellung[k] elif ausId not in foundAusId: # keep this entry foundAusId[ausId] = k austellung[k]['AobObjId'] = set([ objId, ]) austellung[k].pop('AobId') # drop unnecessary id else: # keep only objId part of this entry austellung[foundAusId[ausId]]['AobObjId'].add(objId) del austellung[k] output('\taustellung reduced from %d to %d entries' % (originalSize, len(austellung))) # populate std_year output('\tstandardising years...') for k, v in austellung.iteritems(): year = v['AusJahrS'] yfrom = v['AusDatumVonD'].replace(u' 00:00:00', u'').strip() ytil = v['AusDatumBisD'].replace(u' 00:00:00', u'').strip() v['std_year'] = stdAustellungYear(year, yfrom, ytil) # to match with pre-redux results. Could possibly be dropped instead? v['AusDatumVonD'] = yfrom v['AusDatumBisD'] = ytil # invert to get per objId connections # and convert set to list objIdConnection = {} for k, v in austellung.iteritems(): ausId = v['AusId'] objIds = v['AobObjId'] v['AobObjId'] = list(objIds) for objId in objIds: if objId not in objIdConnection.keys(): objIdConnection[objId] = [] objIdConnection[objId].append(ausId) output('\tadding ausId to objDaten...') for k, v in objDaten.iteritems(): objId = v['ObjId'] v['ausId'] = [] if objId in objIdConnection.keys(): v['ausId'] = objIdConnection.pop(objId) output(u"...done") return austellung
def ereignis_objDaten(ereignisFile, objDaten, logFile): """ Given the ereignis data file and the objDaten data add a ereignis id field to objDaten. Also returns the ereignis data after * combining all objIds for the same ergId * dropping EroId :param ereignisFile: path to eregnis data file :param objDaten: objDaten dict :param logFile: path to logfile :return: dict (and updates objDaten) """ # setup flog = codecs.open(logFile, 'w', 'utf-8') # logfile output(u"Trimming eregnis and adding eregnis to ObjDaten...") # handle eregnis ereignisHeader = 'EroId|ErgId|EroObjId|ErgKurztitelS|ErgArtS' ereignis = helpers.csvFileToDict(ereignisFile, 'EroId', ereignisHeader) originalSize = len(ereignis) # collect all ergId and drop any with invalid title # @toDO: Is keeping objId in eregnis really needed? # Otherwise populate objIdConnection here foundErgId = {} for k, v in ereignis.items(): # allow removing entries from within loop ergId = v['ErgId'] objId = v['EroObjId'] title = v['ErgKurztitelS'] if not title: # remove empty del ereignis[k] elif ergId not in foundErgId.keys(): # keep this entry foundErgId[ergId] = k ereignis[k]['EroObjId'] = set([objId, ]) ereignis[k].pop('EroId') # drop unnecessary id else: # keep only objId part of this entry ereignis[foundErgId[ergId]]['EroObjId'].add(objId) del ereignis[k] output('\tergIds: reduced from %d to %d' % (originalSize, len(ereignis))) # handle urls in ereignis and convert set to list for k, v in ereignis.iteritems(): objIds = v['EroObjId'] url = v['ErgArtS'] # convert set to list v['EroObjId'] = list(objIds) # handle urls if u'%' in url: url = helpers.urldecode_utf8(url) # convert external links to internal if 'wikipedia' in url: url = helpers.external_2_internal_link(url) elif url: flog.write(u'weird url: %s\n' % url) v['ErgArtS'] = url # invert to get per objId connections objIdConnection = {} for k, v in ereignis.iteritems(): ergId = v['ErgId'] objIds = v['EroObjId'] for objId in objIds: if objId not in objIdConnection.keys(): objIdConnection[objId] = [] objIdConnection[objId].append(ergId) # add to objDaten output('\tadding ergId to objDaten...') for k, v in objDaten.iteritems(): objId = v['ObjId'] v['ergId'] = [] if objId in objIdConnection.keys(): v['ergId'] = objIdConnection.pop(objId) flog.close() output(u"...done") return ereignis
def run(outPath=None, dataPath=None, mappingsPath=None, commonsPrefix=None, configPath=u'config.json'): """ Define a list of pages and output files where page has the format Commons:Batch uploading/LSH/* and outputfile the format: commons-*.csv """ # set defaults unless overridden outPath = outPath or OUT_PATH dataPath = dataPath or DATA_PATH mappingsPath = mappingsPath or MAPPING_FOLDER commonsPrefix = commonsPrefix or COMMONS_PREFIX pages = {u'People': u'People', u'Events': u'Events', u'ObjKeywords': u'ObjKeywords', u'Keywords': u'Keywords', # stichwort u'Materials': u'Materials', u'Places': u'Places', u'Photographers': u'Photographers' } # create out_path if it doesn't exist if not os.path.isdir(outPath): os.mkdir(outPath) # fetch, parse and save each page comApi = helpers.openConnection(configPath) for k, v in pages.iteritems(): comPage = u'%s/%s' % (commonsPrefix, k) contents = comApi.getPage(comPage) units = parseEntries(contents[comPage]) outdata = formatOutput(units, k) outFile = os.path.join(outPath, u'commons-%s.csv' % v) out = codecs.open(outFile, 'w', 'utf8') out.write(outdata) out.close() output(u'Created %s' % outFile) # need to do filenames differently mappingFile = os.path.join(mappingsPath, u'Filenames.txt') comPage = u'%s/Filenames' % commonsPrefix contents = comApi.getPage(comPage) # identify changes units, allEntries = parseFilenameEntries(contents[comPage]) if units: # load old filenames filenamesHeader = 'PhoId|MulId|MulPfadS|MulDateiS|filename|ext' filenamesFile = os.path.join(dataPath, u'filenames.csv') oldFilenames = helpers.csvFileToDict(filenamesFile, 'PhoId', filenamesHeader) for unit in units: pho_id = unit[u'phoId'] if pho_id not in oldFilenames.keys(): print u'could not find id in old: %s, %s' % \ (pho_id, unit[u'generated']) exit(1) old_desc = oldFilenames[pho_id][u'filename'] # newDesc = oldDesc.replace(unit[u'generated'], unit[u'improved']) # a safer implementation where new description is appended to # old ending. I.e. "- Museum - idNo" new_desc = u'%s %s' % (unit[u'improved'], splitFilename(old_desc)[1]) if old_desc == new_desc: # indicator that commons file may not having been updated which # may cause more complex problems which are hard to test for print u'did you run the updater a second time without ' \ u'first updating the filenames table on Commons?' exit(1) oldFilenames[pho_id][u'filename'] = new_desc # overwrite old filenames and old mapping # new filename.csv file w. header helpers.dictToCsvFile(filenamesFile, oldFilenames, filenamesHeader) # new Commons mapping file needs a dict with all descriptions mapping_dict = {} for phoId, v in oldFilenames.iteritems(): descr = splitFilename(v[u'filename'])[0] mapping_dict[phoId] = {'descr': descr} Filenames.commonsOutput(mapping_dict, mappingFile, allEntries) output(u'Updated %s and produced a new mappingfile %s. Please upload ' u'the new one to Commons.' % (filenamesFile, mappingFile))
def kuenstler_objDaten(kuenstlerFile, objDaten, logFile): """ Given the kuenstler data file and the objDaten data add a kuenstler id field to objDaten. Also returns the kuenstler data after * removing certain irrelevant roles and dummy entries * combining all objIds for the same kueId * standardising years * dropping a lot of unneeded fields :param kuenstlerFile: path to kuenstler data file :param objDaten: objDaten dict :param logFile: path to logfile :return: dict (and updates objDaten) """ # setup flog = codecs.open(logFile, 'w', 'utf-8') # logfile output(u"Crunching kuenstler...") dummyNames = (u'ingen uppgift', ) badRoles = (u'Leverantör', u'Auktion', u'Förmedlare', u'Givare', u'Återförsäljare', u'Konservator') badRoleCmts = (u'Förpaktare, kontrollör', u'av kopia') droppedFields = ('OkuId', 'ObjAufId', 'AufAufgabeS', 'OkuArtS', 'OkuFunktionS', 'OkuValidierungS', 'KudArtS', 'MulId', 'PhoId') # handle kuenstler kuenstlerHeader = 'OkuId|ObjId|ObjAufId|AufAufgabeS|KueId|KueVorNameS|' \ 'KueNameS|OkuArtS|OkuFunktionS|OkuValidierungS|KudArtS|' \ 'KudDatierungS|KudJahrVonL|KudJahrBisL|KudOrtS|KudLandS|' \ 'KueFunktionS|MulId|PhoId' kuenstler = helpers.csvFileToDict(kuenstlerFile, ('OkuId', 'MulId'), kuenstlerHeader) originalSize = len(kuenstler) # collect all kueId and drop any with invalid title or role # also invert to get per objId connections # @toDO: Is keeping objId in kuenstler really needed? # Otherwise populate objIdConnection here foundKueId = {} objIdConnection = {} for k, v in kuenstler.items(): # allow removing entries from within loop kueId = v['KueId'] objId = v['ObjId'] fName = v['KueVorNameS'] lName = v['KueNameS'] role = v['OkuArtS'] roleCmt = v['OkuFunktionS'] # filter out any undesired entries if role in badRoles or \ roleCmt in badRoleCmts or \ len(fName) + len(lName) == 0 or \ lName in dummyNames: del kuenstler[k] continue # send unique role/kueId combo for objid kueCombo = u'%s:%s:%s' % (role, roleCmt, kueId) if objId not in objIdConnection.keys(): objIdConnection[objId] = set([]) objIdConnection[objId].add(kueCombo) # keep only one entry per unique kueId if kueId not in foundKueId.keys(): # keep this entry foundKueId[kueId] = k kuenstler[k]['ObjId'] = set([ objId, ]) else: # keep only objId part of this entry kuenstler[foundKueId[kueId]]['ObjId'].add(objId) del kuenstler[k] output('\tkueIds: reduced from %d to %d' % (originalSize, len(kuenstler))) # add to objDaten output('\tadding kueId to objDaten...') for k, v in objDaten.iteritems(): objId = v['ObjId'] v['role:roleCmt:kueId'] = [] if objId in objIdConnection.keys(): v['role:roleCmt:kueId'] = list(objIdConnection.pop(objId)) # further cleanup of kuenstler # correcting ort/land entries # stripping years from name # dropping a bunch of fields output('\tfurther cleanup of kuenstler...') for k, v in kuenstler.iteritems(): land = v['KudOrtS'] # missnamed in original database ort = v['KudLandS'] # missnamed in original database lName = v['KueNameS'] bYear = v['KudJahrVonL'] dYear = v['KudJahrBisL'] objIds = v['ObjId'] # correct missnaming in original database v['KudOrtS'] = ort v['KudLandS'] = land # convert set to list v['ObjId'] = list(objIds) # take yearinfo out of name, and store in year lName, bYear, dYear, log = extractKuenstlerYear(lName, bYear, dYear) if log: flog.write(log) v['KueNameS'] = lName v['KudJahrVonL'] = bYear v['KudJahrBisL'] = dYear for field in droppedFields: del v[field] flog.close() output(u"...done") return kuenstler
def ausstellung_objDaten(austellungFile, objDaten): """ Given the austellung data file and the objDaten data add a austellung id field to objDaten. Also returns the austellung data after * adding a std_year field * combining all objIds for the same ausId * dropping AobId :param austellungFile: path to austellung data file :param objDaten: objDaten dict :return: dict (and updates objDaten) """ # often requires manual fixing prior to crunch helpers.verboseInput(u"Confirm that any year formatting issues mentioned " u"in the analysis log have been corrected and the " u"updated ausstellung file saved...\n" u"...by pressing enter when done") # setup dummyTitles = ( u'reparation', u'utställning', u'lån för undersökning', u'OBS! Testpost för admin - utställning, export wikimedia commons', u'lån till Frankrike 1947', u'test karin 20100520', u'test 20100629 (en post skapad för administrativa tester)', u'tennföremål 8 st till Strömsholm', u'utlån f justering av urverk') output(u"Trimming ausstellung and adding ausstellung to ObjDaten...") # handle ausstellung austellungHeader = 'AobId|AusId|AusTitelS|AusOrtS|AusJahrS|AusDatumVonD|' \ 'AusDatumBisD|AobObjId|AufAufgabeS' austellung = helpers.csvFileToDict(austellungFile, 'AobId', austellungHeader) originalSize = len(austellung) # collect all ausId and drop any with invalid title # @toDO: Is keeping objId in austellung really needed? # Otherwise populate objIdConnection here foundAusId = {} for k, v in austellung.items(): # allow removing entries from within loop ausId = v['AusId'] objId = v['AobObjId'] title = v['AusTitelS'] if not title or title in dummyTitles: # remove empty/dummy del austellung[k] elif ausId not in foundAusId: # keep this entry foundAusId[ausId] = k austellung[k]['AobObjId'] = set([objId, ]) austellung[k].pop('AobId') # drop unnecessary id else: # keep only objId part of this entry austellung[foundAusId[ausId]]['AobObjId'].add(objId) del austellung[k] output('\taustellung reduced from %d to %d entries' % (originalSize, len(austellung))) # populate std_year output('\tstandardising years...') for k, v in austellung.iteritems(): year = v['AusJahrS'] yfrom = v['AusDatumVonD'].replace(u' 00:00:00', u'').strip() ytil = v['AusDatumBisD'].replace(u' 00:00:00', u'').strip() v['std_year'] = stdAustellungYear(year, yfrom, ytil) # to match with pre-redux results. Could possibly be dropped instead? v['AusDatumVonD'] = yfrom v['AusDatumBisD'] = ytil # invert to get per objId connections # and convert set to list objIdConnection = {} for k, v in austellung.iteritems(): ausId = v['AusId'] objIds = v['AobObjId'] v['AobObjId'] = list(objIds) for objId in objIds: if objId not in objIdConnection.keys(): objIdConnection[objId] = [] objIdConnection[objId].append(ausId) output('\tadding ausId to objDaten...') for k, v in objDaten.iteritems(): objId = v['ObjId'] v['ausId'] = [] if objId in objIdConnection.keys(): v['ausId'] = objIdConnection.pop(objId) output(u"...done") return austellung