def testLoadEmptyFile(self): tempfile = join(self.tempdir, 'json.json') open(tempfile, 'w').close() self.assertRaises(JSONDecodeError, lambda: JsonDict.load(tempfile)) self.assertEquals({}, JsonDict.load(tempfile, emptyOnError=True)) self.assertRaises(JSONDecodeError, lambda: JsonList.load(tempfile)) self.assertEquals([], JsonList.load(tempfile, emptyOnError=True))
def getRepositoryIds(self, domainId, repositoryGroupId=None): result = JsonList() allIds = self.getRepositoryGroupIds(domainId) if repositoryGroupId is None else [repositoryGroupId] for repositoryGroupId in allIds: jsonData = JsonDict.load(open(join(self._dataPath, '%s.%s.repositoryGroup' % (domainId, repositoryGroupId)))) result.extend(jsonData.get('repositoryIds', [])) return result
def getRepositoryIds(self, domainId, repositoryGroupId=None): result = JsonList() allIds = self.getRepositoryGroupIds( domainId) if repositoryGroupId is None else [repositoryGroupId] for repositoryGroupId in allIds: jsonData = self.getRepositoryGroup(repositoryGroupId, domainId) result.extend(jsonData.get('repositoryIds', [])) return result
def testLoadList(self): jd = JsonList(['hello', 'world']) tempfile = join(self.tempdir, 'json.json') with open(tempfile, 'w') as fp: fp.write(str(jd)) with open(tempfile) as fp: jd2 = JsonList.load(fp) self.assertEqual(jd, jd2)
def testLoadEmptyFile(self): tempfile = join(self.tempdir, 'json.json') with open(tempfile, 'w') as fp: pass self.assertRaises(json.JSONDecodeError, lambda: JsonDict.load(tempfile)) self.assertEqual({}, JsonDict.load(tempfile, emptyOnError=True)) self.assertRaises(json.JSONDecodeError, lambda: JsonList.load(tempfile)) self.assertEqual([], JsonList.load(tempfile, emptyOnError=True))
def testPrefixSearch(self): self.response = JsonList([["value0", 1], ["value1", 2]]).dumps() response = retval(self._lucene.prefixSearch(fieldname='field1', prefix='valu')) self.assertEquals(['value1', 'value0'], response.hits) response = retval(self._lucene.prefixSearch(fieldname='field1', prefix='valu', showCount=True)) self.assertEquals([('value1', 2), ('value0', 1)], response.hits)
def getRepositories(self, domainId, repositoryGroupId=None): try: repositoryIds = self.getRepositoryIds( domainId=domainId, repositoryGroupId=repositoryGroupId) except IOError: raise ValueError("idDoesNotExist") return JsonList([ self.getRepository(repositoryId, domainId) for repositoryId in repositoryIds ])
def testPrefixSearch(self): data = JsonList([ {"type": "TextField", "name": "prefixField", "value": "value0"}, {"type": "TextField", "name": "prefixField", "value": "value1"}, {"type": "TextField", "name": "prefixField", "value": "value2"}, ]).dumps() header, body = postRequest(self.luceneServerPort, self._path + '/update/?identifier=id1', data=data) self.assertTrue("200 OK" in header.upper(), header) header, body = postRequest(self.luceneServerPort, self._path + '/prefixSearch/?fieldname=prefixField&prefix=val', parse=False) self.assertEqual([['value0', 1], ['value1', 1], ['value2', 1]], loads(body))
def testSuggestionRequest(self): data = JsonList([ {"type": "TextField", "name": "field", "value": "value"}, ]).dumps() header, body = postRequest(self.luceneServerPort, self._path + '/update/?identifier=id1', data=data) self.assertTrue("200 OK" in header.upper(), header) header, body = postRequest(self.luceneServerPort, self._path + '/query/', parse=False, data=JsonDict(query=dict(type="MatchAllDocsQuery"), suggestionRequest=dict(field="field", count=1, suggests=['valeu'])).dumps()) jsonResponse = loads(body) self.assertEqual({'valeu': ['value']}, jsonResponse["suggestions"]) self.assertTrue("suggestionTime" in jsonResponse["times"])
def parseHeaderAndBody(h, b=None, parseBody=True): if b is None: h, b = h header, body = parseResponse(h + CRLF * 2 + b) if body and parseBody and 'Content-Type' in header['Headers']: contentType = header['Headers']['Content-Type'] if 'xml' in contentType: return header, XML(body) if 'json' in contentType: try: return header, JsonDict.loads(body) if body[0] == '{' else JsonList.loads(body) except JSONDecodeError: return header, 'JSONDecodeError in: ' + body return header, body
def testFacets(self): data = JsonList([ {"type": "TextField", "name": "fieldname", "value": "value"}, {"type": "FacetField", "name": "fieldname", "path": ["value"]} ]).dumps() header, body = postRequest(self.luceneServerPort, self._path + '/update/?identifier=id1', data=data) self.assertTrue("200 OK" in header.upper(), header) header, body = postRequest(self.luceneServerPort, self._path + '/query/', data=JsonDict(query=dict(type="MatchAllDocsQuery"), facets=[{"fieldname": "fieldname", "maxTerms": 10}]).dumps(), parse=False) self.assertTrue("200 OK" in header.upper(), header) jsonResponse = loads(body) self.assertEqual(1, jsonResponse['total']) self.assertEqual([{'path': [], 'fieldname': 'fieldname', 'terms': [{'count': 1, 'term': 'value'}]}], jsonResponse['drilldownData']) self.assertTrue("facetTime" in jsonResponse["times"])
def _suggest(self, arguments, outside): prefix = arguments['prefix'][0] fieldname = arguments.get('field', [self._defaultField])[0] limit = int(arguments.get('limit', [self._defaultLimit])[0]) yield HttpOk yield ContentTypeHeader + CONTENT_TYPE_JSON_SUGGESTIONS + CRLF yield "Access-Control-Allow-Origin: *" + CRLF yield "Access-Control-Allow-Headers: X-Requested-With" + CRLF yield CRLF hits = [] if len(prefix) >= self._minimumLength: response = yield outside.prefixSearch(fieldname=fieldname, prefix=prefix.lower(), limit=limit) hits = response.hits yield JsonList([prefix, hits]).dumps()
def startServer(port, dataDir=None, jsonConfig=None, batchSize=None): batchSize = batchSize or DEFAULT_BATCH_SIZE setSignalHandlers() tempDir = mkdtemp(prefix='mockoai-') config = JsonList.loads(jsonConfig or '[]') if dataDir: config.append({'dirs': dataDir, 'path': '/'}) try: reactor = Reactor() server = be(dna(reactor, port, config=config, tempDir=tempDir, batchSize=batchSize)) print 'Ready to rumble the mock plein server at', port list(compose(server.once.observer_init())) registerShutdownHandler(statePath=tempDir, server=server, reactor=reactor) stdout.flush() reactor.loop() finally: rmtree(tempDir)
def testAddAndQueryDocument(self): data = JsonList([ {"type": "TextField", "name": "fieldname", "value": "value"} ]).dumps() header, body = postRequest(self.luceneServerPort, self._path + '/update/?identifier=id1', data=data) self.assertTrue("200 OK" in header.upper(), header) header, body = postRequest(self.luceneServerPort, self._path + '/query/', data=JsonDict(query=dict(type="MatchAllDocsQuery")).dumps(), parse=False) self.assertTrue("200 OK" in header.upper(), header) response = loads(body) self.assertEqual(1, response['total']) self.assertEqual([{'id': 'id1', 'score': 1.0}], response['hits']) header, body = postRequest(self.luceneServerPort, self._path + '/query/', data=JsonDict(query=dict(type="TermQuery", term=dict(field="fieldname", value="value"))).dumps(), parse=False) self.assertTrue("200 OK" in header.upper(), header) response = loads(body) self.assertEqual(1, response['total']) self.assertTrue("queryTime" in response) self.assertTrue("times" in response) self.assertEqual([{'id': 'id1', 'score': 0.28768208622932434}], response['hits'])
def testCommit(self): header, body = postRequest(self.luceneServerPort, self._path + '/settings/', data=JsonDict(commitCount=10).dumps()) self.assertTrue("200 OK" in header.upper(), header) try: data = JsonList([ {"type": "TextField", "name": "fieldname", "value": "value"} ]).dumps() header, body = postRequest(self.luceneServerPort, self._path + '/update/?identifier=idCommit', data=data) self.assertTrue("200 OK" in header.upper(), header) header, body = postRequest(self.luceneServerPort, self._path + '/query/', parse=False, data=JsonDict(query=dict(type="TermQuery", term=dict(field="__id__", value="idCommit"))).dumps()) response = loads(body) self.assertEqual(0, response['total']) header, body = postRequest(self.luceneServerPort, '/commit/', parse=False) self.assertTrue("200 OK" in header.upper(), header) header, body = postRequest(self.luceneServerPort, self._path + '/query/', parse=False, data=JsonDict(query=dict(type="TermQuery", term=dict(field="__id__", value="idCommit"))).dumps()) response = loads(body) self.assertEqual(1, response['total']) finally: header, body = postRequest(self.luceneServerPort, self._path + '/delete/?identifier=idCommit', data=data) self.assertTrue("200 OK" in header.upper(), header)
def testLoadList(self): jd = JsonList(['hello', 'world']) tempfile = join(self.tempdir, 'json.json') open(tempfile, 'w').write(str(jd)) jd2 = JsonList.load(open(tempfile)) self.assertEquals(jd, jd2)
def testPrettyPrintList(self): jd = JsonList(['hello', 'world']) printedList = jd.pretty_print(indent=5) self.assertTrue('\n "hello"' in printedList) self.assertTrue('\n "world"' in printedList)
def testLoadsList(self): jd = JsonList(['hello', 'world']) jd2 = JsonList.loads(str(jd)) self.assertEquals(jd, jd2)
def testStrList(self): jl = JsonList(['hello', 'world']) self.assertEquals('["hello", "world"]', str(jl))
class SuggestionIndexComponent(Observable): def __init__(self, host, port, **kwargs): super(SuggestionIndexComponent, self).__init__(**kwargs) self._connect = _Connect(host, port, observable=self) def addSuggestions(self, identifier, key, values): titles = [v.get('title') for v in values] types = [v.get('type') for v in values] creators = [v.get('creator') for v in values] yield self._connect.send( "/add?{}".format(urlencode(dict(identifier=identifier))), JsonDict(key=key, values=titles, types=types, creators=creators)) def deleteSuggestions(self, identifier): yield self._connect.send("/delete?{}".format( urlencode(dict(identifier=identifier)))) def registerFilterKeySet(self, name, keySet): yield self._connect.send("/registerFilterKeySet?{}".format( urlencode(dict(name=name))), data=keySet) def createSuggestionNGramIndex(self): yield self._connect.send("/createSuggestionNGramIndex") def suggest(self, value, trigram=False, filters=None, keySetName=None, limit=None): suggestions = yield self._connect.send( "/suggest", JsonDict(value=value, trigram=trigram, filters=filters or [], keySetName=keySetName, limit=limit)) raise StopIteration([Suggestion(s) for s in suggestions]) def indexingState(self): indexingState = yield self._connect.read("/indexingState") raise StopIteration(indexingState if indexingState else None) def totalShingleRecords(self): total = yield self._connect.read("/totalRecords", parse=False) raise StopIteration(int(total)) def totalSuggestions(self): total = yield self._connect.read("/totalSuggestions", parse=False) raise StopIteration(int(total)) def ngramIndexTimestamp(self): timestamp = yield self._connect.read("/ngramIndexTimestamp", parse=False) raise StopIteration(int(timestamp) / 1000.0) def handleRequest(self, arguments, path, **kwargs): value = arguments.get("value", [None])[0] debug = arguments.get("x-debug", ["False"])[0] != 'False' trigram = arguments.get("trigram", ["False"])[0] != 'False' showConcepts = arguments.get("concepts", ["False"])[0] != 'False' filters = arguments.get("filter", None) minScore = float(arguments.get("minScore", ["0"])[0]) apikey = arguments.get("apikey", [None])[0] apikeyFilter = arguments.get("x-apikey-filter", [''])[0] if apikeyFilter: apikey += "-" + apikeyFilter suggest = None if value: t0 = time() try: suggest = yield self.suggest(value, trigram=trigram, filters=filters, keySetName=apikey) except Exception, e: yield serverErrorPlainText yield str(e) return tTotal = time() - t0 yield Ok yield ContentTypeHeader + "application/x-suggestions+json" + CRLF yield "Access-Control-Allow-Origin: *" + CRLF yield "Access-Control-Allow-Headers: X-Requested-With" + CRLF yield 'Access-Control-Allow-Methods: GET, POST, OPTIONS' + CRLF yield 'Access-Control-Max-Age: 86400' + CRLF yield CRLF result = [] if value: suggestions = [] for s in suggest: suggestion = str(s.suggestion) recordType = str(s.type) if s.type else None creator = str(s.creator) if s.creator else None distanceScore = max( 0, -log(distance(value.lower(), suggestion.lower()) + 1) / 4 + 1) matchScore = match(value.lower(), suggestion.lower()) score = float(s.score) sortScore = distanceScore * score**2 * (matchScore * 2) scores = dict(distanceScore=distanceScore, score=score, sortScore=sortScore, matchScore=matchScore) if sortScore > minScore: suggestions.append( (suggestion, recordType, creator, scores)) suggestions = sorted( suggestions, reverse=True, key=lambda (suggestion, recordType, creator, scores): scores['sortScore']) if debug: concepts = [(s, t, c) for s, t, c, _ in suggestions if t] yield JsonDict( dict(value=value, suggestions=suggestions, concepts=concepts, time=tTotal)).dumps() return concepts = [(s, t, c) for s, t, c, _ in suggestions if t][:10] dedupSuggestions = [] for s in suggestions: if s[0] not in dedupSuggestions: dedupSuggestions.append(s[0]) suggestions = dedupSuggestions[:10] result = [value, suggestions] if showConcepts: result.append(concepts) yield JsonList(result).dumps()
def handleRequest(self, *args, **kwargs): yield "HTTP/1.0 200 OK\r\n" yield "\r\n" yield JsonList(args).pretty_print() yield JsonDict(kwargs).pretty_print()
def getDomainIds(self): return JsonList(self._store.listForDatatype('domain'))
def testConvertFromBytes(self): converter = BytesToJson(fromKwarg='kwarg') self.assertEqual(JsonDict({'aap': 3}), converter._convert(b'{\n "aap": 3\n}')) self.assertEqual(JsonList(['aap', 3]), converter._convert(b'[\n "aap", 3\n]'))
def addDocument(self, fields, identifier=None): if self._readonly: raise RuntimeError('Adding documents not allowed for readonly Lucene connection.') args = urlencode(dict(identifier=identifier)) if identifier else '' yield self._connect().send(jsonDict=JsonList(fields), path='/update/?{}'.format(args))
def getStatus(self, **kwargs): return JsonList([x for x in asList(self._getStatus(**kwargs))])
def _load(self): if not isfile(self._jsonFilepath): return {} data = open(self._jsonFilepath).read().strip() result = {} if '[' != data[0]: for identifier, serviceDict in JsonDict.loads(data).items(): service = Service(domainname=self._domainname, timeout=self._timeout, identifier=identifier, ultimateTimeout=self._ultimateTimeout, **serviceDict) service.validate() result[service.identifier] = service return result for service in (Service(domainname=self._domainname, timeout=self._timeout, ultimateTimeout=self._ultimateTimeout, **item) for item in JsonList.loads(data)): service.validate() result[service.identifier] = service return result