def _indexObject(objectToIndex: ObjectToIndexTuple) -> List[SearchIndex]: """ Index Object This method creates the "SearchIndex" objects to insert into the DB. Because our data is not news articles, we can skip some of the advanced natural language processing (NLP) We're going to be indexing things like unique IDs, job titles, and equipment names. We may add exclusions for nuisance words later on. """ searchIndexes = [] for propKey, text in objectToIndex.fullKwProps.items(): for token in splitFullKeywords(text): searchIndexes.append( SearchIndex(chunkKey=makeSearchIndexChunkKey(token), keyword=token, propertyName=propKey, objectId=objectToIndex.id)) for propKey, text in objectToIndex.partialKwProps.items(): for token in splitPartialKeywords(text): searchIndexes.append( SearchIndex(chunkKey=makeSearchIndexChunkKey(token), keyword=token, propertyName=propKey, objectId=objectToIndex.id)) return searchIndexes
def _filterObjectsForSearchString(self, results: List[SearchResultObjectTuple], searchString: str, propertyName: Optional[str]) -> Deferred: """ Filter Objects For Search String STAGE 2 of the search. This method filters the loaded objects to ensure we have full matches. :param results: :param searchString: :param propertyName: :return: """ noFulls = lambda t: not t.endswith('$') # Get the partial tokens, and match them tokens = set(filter(noFulls, splitPartialKeywords(searchString))) def filterResult(result: SearchResultObjectTuple) -> bool: props = result.properties if propertyName: props = {propertyName: props[propertyName]} \ if propertyName in props else {} allPropVals = ' '.join(props.values()) theseTokens = set(filter(noFulls, splitPartialKeywords(allPropVals))) return bool(tokens & theseTokens) return list(filter(filterResult, results))
def test_twoCharTokens(self): self.assertEqual(set(), twoCharTokens("smith smith")) self.assertEqual({'^to$'}, twoCharTokens(splitFullKeywords("two to"))) self.assertEqual({'^to'}, twoCharTokens(splitPartialKeywords("two to")))
def _getObjectIdsForSearchString(self, searchString: str, propertyName: Optional[str]) -> Deferred: """ Get ObjectIds For Search String STAGE 1 of the search. This method loads all of the search objects that match the search strings. This will load in some false matches, they are filtered out in _filterObjectsForSearchString Searching is complex because we don't know if we're looking for a full or partial tokenizing. :rtype List[int] """ logger.debug("Started search with string |%s|", searchString) # --------------- # Search for fulls fullTokens = splitFullKeywords(searchString) logger.debug("Searching for full tokens |%s|", fullTokens) # Now lookup any remaining keywords, if any resultsByFullKw = self._getObjectIdsForTokensBlocking(fullTokens, propertyName) resultsByFullKw = {k: v for k, v in resultsByFullKw.items() if v} logger.debug("Found results for full tokens |%s|", set(resultsByFullKw)) # --------------- # Search for partials partialTokens = splitPartialKeywords(searchString) logger.debug("Searching for partial tokens |%s|", partialTokens) # Now lookup any remaining keywords, if any resultsByPartialKw = self._getObjectIdsForTokensBlocking(partialTokens, propertyName) resultsByPartialKw = {k: v for k, v in resultsByPartialKw.items() if v} logger.debug("Found results for partial tokens |%s|", set(resultsByPartialKw)) # --------------- # Process the results # Merge partial kw results with full kw results. resultsByKw = self._mergePartialAndFullMatches(searchString, resultsByFullKw, resultsByPartialKw) logger.debug("Merged tokens |%s|", set(resultsByKw)) # Now, return the ObjectIDs that exist in all keyword lookups objectIdsUnion = self._setIntersectFilterIndexResults(resultsByKw) # Limit to 50 and return return list(objectIdsUnion)[:50]
def filterResult(result: SearchResultObjectTuple) -> bool: props = result.properties if propertyName: props = {propertyName: props[propertyName]} \ if propertyName in props else {} allPropVals = ' '.join(props.values()) theseTokens = set(filter(noFulls, splitPartialKeywords(allPropVals))) return bool(tokens & theseTokens)
def test_mergePartialAndFullMatches_3(self): searchString = 'tatu west fus' fullByKw = {} partialByKw = {t: [7, 6] for t in splitPartialKeywords(searchString)} inst = FastKeywordController(None, None) resultByKw = inst._mergePartialAndFullMatches(searchString, fullByKw, partialByKw) self.assertEqual(set(resultByKw), {'west', 'fus', 'tatu'}) self.assertEqual(set(resultByKw['west']), {6, 7}) self.assertEqual(set(resultByKw['fus']), {6, 7}) self.assertEqual(set(resultByKw['tatu']), {6, 7})
def _mergePartialAndFullMatches(self, searchString: str, resultsByFullKw: Dict[str, List[int]], resultsByPartialKw: Dict[str, List[int]] ) -> Dict[str, List[int]]: """ Merge Partial """ # Copy this, because we want to modify it and don't want to affect other logic resultsByPartialKw = resultsByPartialKw.copy() resultsByPartialKwSet = set(resultsByPartialKw) mergedResultsByKw = {} for fullKw, fullObjectIds in resultsByFullKw.items(): # Merge in full fullKw = fullKw.strip('^$') existing = mergedResultsByKw.get(fullKw.strip('^$'), list()) # Include the fulls existing.extend(fullObjectIds) mergedResultsByKw[fullKw] = existing tokens = _splitFullTokens(searchString) for token in tokens: token = token.strip('^$') existing = mergedResultsByKw.get(token.strip('^$'), list()) partialKws = splitPartialKeywords(token) if not partialKws <= resultsByPartialKwSet: continue # Union all objectIdsForToken = set(resultsByPartialKw[partialKws.pop()]) while partialKws: objectIdsForToken &= set(resultsByPartialKw[partialKws.pop()]) existing.extend(list(objectIdsForToken)) mergedResultsByKw[token] = existing return mergedResultsByKw
def testPartialKeywordSplit(self): self.assertEqual({'^smi', 'mit', 'ith'}, splitPartialKeywords("smith")) self.assertEqual( {'^zor', 'orr', 'rro', 'ror', 'ore', 'rey', 'eyn', 'yne', 'ner'}, splitPartialKeywords("ZORRO-REYNER")) self.assertEqual({'^345', '535', '453', '534', '345'}, splitPartialKeywords("34534535")) self.assertEqual({'^and'}, splitPartialKeywords("and")) self.assertEqual({"^to"}, splitPartialKeywords("to")) self.assertEqual({'a55', '^ha5'}, splitPartialKeywords("ha55")) self.assertEqual({'^mil', 'ilt', 'lto', 'ton', '^uni', 'nit', '^22'}, splitPartialKeywords("Milton Unit 22")) self.assertEqual({'^mil', 'ill', 'lls', '^un', '^no'}, splitPartialKeywords("mills un no")) self.assertEqual({'^uni', 'nit', "^22"}, splitPartialKeywords("Unit 22")) self.assertEqual({'^uni', 'nit'}, splitPartialKeywords("Unit 1")) self.assertEqual({'^uni', 'nit'}, splitPartialKeywords("A Unit")) self.assertEqual({'^uni', 'nit'}, splitPartialKeywords("2 Unit")) self.assertEqual( {'^ats', "^cb", "^b3", '^66k', '6kv', '^tra', 'ran', 'ans'}, splitPartialKeywords("ATS B3 TRANS 66KV CB")) self.assertEqual({'^col', '^lin', 'ins'}, splitPartialKeywords("COL LINS")) self.assertNotEqual(splitPartialKeywords("COLLINS"), splitPartialKeywords("COL LINS"))