def pythonvectors(so: SearchObject) -> JSON_STR: """ this is the matching function to golangvectors() [0] test to see what will happen: [a] scope problems? [jump away if so...] [b] already a model on file? ... [jump down to #5 if so] [1] generate a searchlist [2] do a searchlistintosqldict() [3] acquire and bag the words [a] grab db lines that are relevant to the search [b] turn them into a unified text block [c] do some preliminary cleanups [d] break the text into sentences and assemble []SentenceWithLocus (NB: these are "unlemmatized bags of words") [e] figure out all of the words used in the passage [f] find all of the parsing info relative to these words [g] figure out which headwords to associate with the collection of words [h] build the lemmatized bags of words ('unlemmatized' can skip [f] and [g]...) [4] hand the bags over to Word2Vec(), etc. [*] [5] run queries against the model and return the JSON results """ # debugmessage('pythonvectors()') assert so.vectorquerytype in [ 'analogies', 'nearestneighborsquery', 'topicmodel' ] # [0] is this really going to happen? so.poll.statusis('Checking for valid search') # [i] do we bail out before even getting started? # note that this can / will return independently and break here abortjson = checkneedtoabort(so) if abortjson: del so.poll return abortjson # [ii] do we actually have a model stored already? so.poll.statusis('Checking for stored search') # calculatewholeauthorsearches() + configurewhereclausedata() so = updatesearchlistandsearchobject(so) so.setsearchlistthumbprint() so.poll.allworkis(-1) # this turns off the % completed notice in the JS so.poll.sethits(0) themodel = checkforstoredvector(so) if not themodel: # [1] generate a searchlist: use executesearch() as the template so.usecolumn = 'marked_up_line' so.cap = 199999999 # [2] do a searchlistintosqldict() [this is killing lda...] so.searchsqldict = searchlistintosqldict(so, str(), vectors=True) bagsofwords = acquireandbagthewords(so) # [4] hand the bags over to Word2Vec(), etc. so.poll.statusis('Building the model') if so.vectorquerytype == 'nearestneighborsquery': themodel = buildgensimmodel(so, bagsofwords) elif so.vectorquerytype == 'analogies': # the same gensim model can serve both analogies and neighbors themodel = buildgensimmodel(so, bagsofwords) elif so.vectorquerytype == 'topicmodel': stops = list(mostcommonwordsviaheadwords()) bagsofsentences = [' '.join(b) for b in bagsofwords] bagsofsentences = [ removestopwords(s, stops) for s in bagsofsentences ] themodel = buildsklearnselectedworks(so, bagsofsentences) else: pass elif so.iamarobot: # there is a model and the bot is attempting to build something that has already been build return '<!-- MODEL EXISTS -->' # so we have a model one way or the other by now... # [5] run queries against the model if so.iamarobot: return '<!-- MODEL BUILT -->' if so.vectorquerytype == 'nearestneighborsquery': jsonoutput = generatenearestneighbordata(None, len(so.searchlist), so, themodel) elif so.vectorquerytype == 'analogies': jsonoutput = gensimgenerateanalogies(themodel, so) elif so.vectorquerytype == 'topicmodel': # def ldatopicsgenerateoutput(ldavishtmlandjs: str, workssearched: int, settings: dict, searchobject: SearchObject): jsonoutput = ldatopicsgenerateoutput(themodel, so) else: jsonoutput = json.dumps( 'golang cannot execute {s} queries'.format(s=so.vectorquerytype)) return jsonoutput
def buildfakesearchobject(qtype='nearestneighborsquery') -> SearchObject: """ do what it takes to build a hollow searchobject :return: """ frozensession = dict() frozensession['vdim'] = hipparchia.config['VECTORDIMENSIONS'] frozensession['vwindow'] = hipparchia.config['VECTORWINDOW'] frozensession['viterat'] = hipparchia.config['VECTORTRAININGITERATIONS'] frozensession['vminpres'] = hipparchia.config['VECTORMINIMALPRESENCE'] frozensession['vdsamp'] = hipparchia.config['VECTORDOWNSAMPLE'] frozensession['vcutloc'] = hipparchia.config['VECTORDISTANCECUTOFFLOCAL'] frozensession['vcutneighb'] = hipparchia.config[ 'VECTORDISTANCECUTOFFNEARESTNEIGHBOR'] frozensession['vcutlem'] = hipparchia.config[ 'VECTORDISTANCECUTOFFLEMMAPAIR'] frozensession['vnncap'] = hipparchia.config['NEARESTNEIGHBORSCAP'] frozensession['vsentperdoc'] = hipparchia.config['SENTENCESPERDOCUMENT'] frozensession['ldamaxfeatures'] = hipparchia.config['LDAMAXFEATURES'] frozensession['ldacomponents'] = hipparchia.config['LDACOMPONENTS'] frozensession['ldamaxfreq'] = hipparchia.config['LDAMAXFREQ'] frozensession['ldaminfreq'] = hipparchia.config['LDAMINFREQ'] frozensession['ldaiterations'] = hipparchia.config['LDAITERATIONS'] frozensession['ldamustbelongerthan'] = hipparchia.config[ 'LDAMUSTBELONGERTHAN'] frozensession['baggingmethod'] = hipparchia.config['DEFAULTBAGGINGMETHOD'] blanks = ['searchscope', 'nearornot', 'onehit'] for b in blanks: frozensession[b] = None nulls = ['psgselections', 'psgexclusions'] for n in nulls: frozensession[n] = list() zeroes = ['proximity', 'maxresults', 'linesofcontext'] for z in zeroes: frozensession[z] = 0 trueorfalse = [ 'onehit', 'icandodates', 'nearestneighborsquery', 'searchinsidemarkup' ] for x in trueorfalse: frozensession[x] = False for x in [ 'agnexclusions', 'agnselections', 'alocexclusions', 'alocselections', 'analogyfinder', 'auexclusions', 'auselections' ]: frozensession[x] = list() for s in [ 'wkexclusions', 'wkgnexclusions', 'wkgnselections', 'wkselections', 'wlocexclusions', 'wlocselections' ]: frozensession[s] = list() for p in ['psgexclusions', 'psgselections']: frozensession[p] = list() for c in [ 'christiancorpus', 'latincorpus', 'greekcorpus', 'inscriptioncorpus' ]: frozensession[c] = True frozensession['latestdate'] = 1500 frozensession['earliestdate'] = -850 so = SearchObject('vectorbot', str(), str(), None, None, frozensession) # parsevectorsentences() needs the following: so.vectorquerytype = qtype so.usecolumn = 'marked_up_line' so.sortorder = 'shortname' so.iamarobot = True return so