def index(personDB, familyDB, relationDB): #config = IndexWriterConfig(Version.LUCENE_CURRENT, analyzer) config = IndexWriterConfig(analyzer) config.setOpenMode(IndexWriterConfig.OpenMode.CREATE) writer = IndexWriter(indexDir, config) #?#indexWriter.setRAMBufferSizeMB(50); KOLLA 256 mt = matchtext() for p in personDB.find({}, no_cursor_timeout=True): matchtxt = mt.matchtextPerson(p, personDB, familyDB, relationDB) doc = Document() doc.add(Field('uid',str(p['_id']), StringField.TYPE_STORED)) doc.add(Field('sex',str(p['sex']), StringField.TYPE_STORED)) doc.add(Field("text", matchtxt, TextField.TYPE_NOT_STORED)) writer.addDocument(doc) #Family matchtext for f in familyDB.find(): matchtxt = mt.matchtextFamily(f, familyDB, personDB, relationDB) doc = Document() doc.add(Field('uid',str(f['_id']), StringField.TYPE_STORED)) doc.add(Field('sex','FAM', StringField.TYPE_STORED)) doc.add(Field("text", matchtxt, TextField.TYPE_NOT_STORED)) writer.addDocument(doc) writer.commit() writer.close() return
def viewNoRelErr(personIds, familyIds, config): #from luceneUtils import setupDir, search #setupDir(config['workDB']) searchDB = luceneDB(config['workDB']) from matchtext import matchtext mt_tmp = matchtext() person = config['persons'].find_one({'_id': personIds}) matchtxt = mt_tmp.matchtextPerson(person, config['persons'], config['families'], config['relations']) candidates = searchDB.search(matchtxt, person['sex'], 5) #Lucene search tab = [] tab.append(['Score',u'Namn/refId', u'Född', u'Död', '', u'Namn/refId', u'Född', u'Död']) for (kid,score) in candidates: if kid == personIds: continue cand = config['persons'].find_one({'_id': kid}) if not cand: continue try: if abs(int(person['birth']['date'][0:4]) - int(cand['birth']['date'][0:4])) > 10: continue except:pass t = [] t.append("%3.0f" % (score)) t.extend(persTab(personIds, config['persons'])) args = {'where': 'visa', 'what': '/actions/mergePers', 'id1': str(personIds), 'id2': str(kid)} button = '<br><button onclick="doAction('+str(args)+')">Samma person</button>' t.append(button) t.extend(persTab(kid, config['persons'])) tab.append(t) return (tab,'')
def updateDeleteRec(self, pid1, pid2, personDB, familyDB, relationDB): config = IndexWriterConfig(self.analyzer) config.setOpenMode(IndexWriterConfig.OpenMode.APPEND) writer = IndexWriter(self.indexDir, config) mt = matchtext() writer.deleteDocuments(Term('uid', pid1)) writer.deleteDocuments(Term('uid', pid2)) p = personDB.find_one({'_id': pid1}) matchtxt = mt.matchtextPerson(p, personDB, familyDB, relationDB) doc = Document() doc.add(Field('uid',str(pid1), StringField.TYPE_STORED)) doc.add(Field('sex',str(p['sex']), StringField.TYPE_STORED)) doc.add(Field("match", matchtxt, TextField.TYPE_NOT_STORED)) doc.add(Field("text", mt.luceneFix(self.personText(p)), TextField.TYPE_NOT_STORED)) writer.addDocument(doc) writer.commit() writer.close() self.searcher = IndexSearcher(DirectoryReader.open(self.indexDir)) return
def updateDeleteRec(self, pid1, pid2, personDB, familyDB, relationDB): config = IndexWriterConfig(self.analyzer) config.setOpenMode(IndexWriterConfig.OpenMode.APPEND) writer = IndexWriter(self.indexDir, config) mt = matchtext() writer.deleteDocuments(Term('uid', pid1)) writer.deleteDocuments(Term('uid', pid2)) p = personDB.find_one({'_id': pid1}) matchtxt = mt.matchtextPerson(p, personDB, familyDB, relationDB) doc = Document() doc.add(Field('uid', str(pid1), StringField.TYPE_STORED)) doc.add(Field('sex', str(p['sex']), StringField.TYPE_STORED)) doc.add(Field("match", matchtxt, TextField.TYPE_NOT_STORED)) doc.add( Field("text", mt.luceneFix(self.personText(p)), TextField.TYPE_NOT_STORED)) writer.addDocument(doc) writer.commit() writer.close() self.searcher = IndexSearcher(DirectoryReader.open(self.indexDir)) return
def index(self, personDB, familyDB, relationDB): """ indexes a database Field match includes information about parents and is used to find matches Field text has Ids, names, places, and dates and is used to find a person/family """ config = IndexWriterConfig(self.analyzer) config.setOpenMode(IndexWriterConfig.OpenMode.CREATE) writer = IndexWriter(self.indexDir, config) #indexWriter.setRAMBufferSizeMB(256) #? mt = matchtext() for p in personDB.find({}, no_cursor_timeout=True): matchtxt = mt.matchtextPerson(p, personDB, familyDB, relationDB) doc = Document() doc.add(Field('uid', str(p['_id']), StringField.TYPE_STORED)) doc.add(Field('sex', str(p['sex']), StringField.TYPE_STORED)) doc.add(Field("match", matchtxt, TextField.TYPE_NOT_STORED)) doc.add( Field("text", mt.luceneFix(self.personText(p)), TextField.TYPE_NOT_STORED)) writer.addDocument(doc) #Family matchtext for f in familyDB.find(): #matchtxt = mt.matchtextFamily(f, familyDB, personDB, relationDB) doc = Document() doc.add(Field('uid', str(f['_id']), StringField.TYPE_STORED)) #doc.add(Field('sex','FAM', StringField.TYPE_STORED)) #doc.add(Field("match", matchtxt, TextField.TYPE_NOT_STORED)) txt = f['_id'] if 'refId' in f: txt += ' ' + f['refId'] doc.add(Field("text", txt, TextField.TYPE_NOT_STORED)) writer.addDocument(doc) writer.commit() writer.close() return
def index(self, personDB, familyDB, relationDB): """ indexes a database Field match includes information about parents and is used to find matches Field text has Ids, names, places, and dates and is used to find a person/family """ config = IndexWriterConfig(self.analyzer) config.setOpenMode(IndexWriterConfig.OpenMode.CREATE) writer = IndexWriter(self.indexDir, config) #indexWriter.setRAMBufferSizeMB(256) #? mt = matchtext() for p in personDB.find({}, no_cursor_timeout=True): matchtxt = mt.matchtextPerson(p, personDB, familyDB, relationDB) doc = Document() doc.add(Field('uid',str(p['_id']), StringField.TYPE_STORED)) doc.add(Field('sex',str(p['sex']), StringField.TYPE_STORED)) doc.add(Field("match", matchtxt, TextField.TYPE_NOT_STORED)) doc.add(Field("text", mt.luceneFix(self.personText(p)), TextField.TYPE_NOT_STORED)) writer.addDocument(doc) #Family matchtext for f in familyDB.find(): #matchtxt = mt.matchtextFamily(f, familyDB, personDB, relationDB) doc = Document() doc.add(Field('uid',str(f['_id']), StringField.TYPE_STORED)) #doc.add(Field('sex','FAM', StringField.TYPE_STORED)) #doc.add(Field("match", matchtxt, TextField.TYPE_NOT_STORED)) txt = f['_id'] if 'refId' in f: txt += ' ' + f['refId'] doc.add(Field("text", txt, TextField.TYPE_NOT_STORED)) writer.addDocument(doc) writer.commit() writer.close() return
# -*- coding: utf-8 -*- # This Python file uses the following encoding: utf-8 import common from matchUtils import * import os, sys, logging from collections import defaultdict from matchtext import matchtext from luceneUtils import search, setupDir, index mt_tmp = matchtext() from dbUtils import getFamilyFromChild #FIX! Normalization factor Lucene score norm = {'kalle_testp1': 8.7, 'kalle_testp2': 5.1, 'kalle_testp3': 8.6, 'kalle_testp4': 8.4, 'kalle_testp5': 10.2, 'default': 9.0} def SVMfeatures(work, match, conf, score): return svmbaseline(work, match, conf, score) # alg = os.environ['SVMalgoritm'] # if alg == 'all42': return svmall42(work, match, conf, score) # elif alg == 'baseline': return svmbaseline(work, match, conf, score) # elif alg == 'oldPerson': return svmbaseline(work, match, conf, score) # else: # print 'ERR unkown person algorithm:', alg # sys.exit() def famSVMfeatures(work, match, conf, score): return svmFamily(work, match, conf) # alg = os.environ['famSVMalgoritm'] # if alg == 'family': return svmFamily(work, match, conf) # else: # print 'ERR unkown family algorithm:', alg
# -*- coding: utf-8 -*- # This Python file uses the following encoding: utf-8 import re, sys, math from datetime import date from svmutil import svm_load_model, svm_predict import importlib from matchtext import matchtext from dbUtils import getFamilyFromChild mt_tmp = matchtext() svmModel = False SVMfeatures = None _cache = {} def cos(l1, l2): """ Similarity between two vectors = cosine for the angle between the vectors: cosine = ( V1 * V2 ) / ||V1|| x ||V2|| Vectors expressed as strings, split on blankspace, assume boolean weights """ v1 = l1.split() v2 = l2.split() s = 0 for w1 in v1: if w1 in v2: s += 1 return s / (math.sqrt(len(v1)) * math.sqrt(len(v2))) def compName(n1, n2): """ Compare names: n1 n2 strings, blankspace separated names return value between -1 (mismatch) and 1 (match)
def editList(config, typ): global dubblList tit = 'Verktyg databas: ' + config['workDB'] doTyp = [] if typ in ('child', 'family', 'relation', 'dubblett', 'dubblettFind'): doTyp.append(typ) else: dubblList = [] doTyp.append('child') doTyp.append('family') childErrs = [['Typ', 'Person', 'Familjer', 'action']] famErrs = [['Typ', 'Personer', 'Familj', 'action']] relErrs = [['Typ', 'Person', '', 'action']] dubbls = [] (cErr, fErr, rErr) = sanity(config['persons'], config['families'], config['relations'], do=doTyp) cErr = repairChild(cErr, config['persons'], config['families'], config['relations'], config['originalData']) for (pers, chFams) in cErr: child = "%s %s" % (pers['_id'], pers['name']) args = {'where': 'visa', 'what': '/view/relErr', 'typ': 'child', 'person': str(pers['_id']), 'family': str(':'.join(chFams))} visa = '<button onclick="doAction('+str(args)+')">Visa</button>' childErrs.append(['Child', child, '; '.join(chFams), visa]) #1 husb/wife per family fErr = repairFam(fErr, config['persons'], config['families'], config['relations'], config['originalData']) for (famId, persList) in fErr: person = [] pids = [] for pers in persList: person.append("%s %s" % (pers['_id'], pers['name'])) pids.append(pers['_id']) args = {'where': 'visa', 'what': '/view/relErr', 'typ': 'partner', 'person': str(':'.join(pids)), 'family': str(famId)} visa = '<button onclick="doAction('+str(args)+')">Visa</button>' famErrs.append(['Partner', '<br>'.join(person), famId, visa]) rErr = repairRel(rErr, config['persons'], config['families'], config['relations'], config['originalData']) for pid in rErr: pers = config['persons'].find_one({'_id': pid}) person = "%s %s" % (pers['_id'], pers['name']) args = {'where': 'visa', 'what': '/view/relErr', 'typ': 'noRel', 'person': str(pers['_id']), 'family': ''} visa = '<button onclick="doAction('+str(args)+')">Visa</button>' relErrs.append(['Inga relationer', person, '', visa]) if typ in ('dubblett', 'dubblettFind'): tit = 'Dubblett editor databas: ' + config['workDB'] if len(dubblList) == 0 or typ=='dubblettFind': searchDB = luceneDB(config['workDB']) from matchtext import matchtext mt_tmp = matchtext() tab = [] done = [] tab.append(['Score',u'Namn/Id', 'Kandidat Id']) for person in config['persons'].find(): matchtxt = mt_tmp.matchtextPerson(person, config['persons'], config['families'], config['relations']) txt = [] for term in matchtxt.split(): if term.startswith('Father') or term.startswith('Mother'): continue txt.append(term) #candidates = search(' '.join(txt), person['sex'], 4) #Lucene search candidates = searchDB.search(' '.join(txt), person['sex'], 4) #Lucene search pstr = "%s %s" % (person['_id'], person['name']) for (kid,score) in candidates: if kid == person['_id']: continue if ';'.join([person['_id'],kid]) in done: continue cand = config['persons'].find_one({'_id': kid}) if not cand: continue try: if abs(int(person['birth']['date'][0:4]) - int(cand['birth']['date'][0:4])) > 10: continue except:pass args = {'where': 'visa', 'what': '/view/relErr', 'typ': 'dubblett', 'person': str(person['_id']), 'family': str(kid)} visa = '<button onclick="doAction('+str(args)+')">Visa</button>' tab.append(["%3.0f" % (score), pstr, kid, visa]) done.append(';'.join([kid,person['_id']])) dubblList = sorted(tab, key=itemgetter(0), reverse=True)[0:25] findAndMergeDuplFams(config['persons'], config['families'], config['relations'], config['originalData']) #OBS does not update luceneDB - OK? dubbls = dubblList return (tit, childErrs, famErrs, relErrs, dubbls)