def testLoadEmptyFile(self):
tempfile = join(self.tempdir, 'json.json')
open(tempfile, 'w').close()
self.assertRaises(JSONDecodeError, lambda: JsonDict.load(tempfile))
self.assertEquals({}, JsonDict.load(tempfile, emptyOnError=True))
self.assertRaises(JSONDecodeError, lambda: JsonList.load(tempfile))
self.assertEquals([], JsonList.load(tempfile, emptyOnError=True))
def testLoadEmptyFile(self):
tempfile = join(self.tempdir, 'json.json')
open(tempfile, 'w').close()
self.assertRaises(JSONDecodeError, lambda: JsonDict.load(tempfile))
self.assertEquals({}, JsonDict.load(tempfile, emptyOnError=True))
self.assertRaises(JSONDecodeError, lambda: JsonList.load(tempfile))
self.assertEquals([], JsonList.load(tempfile, emptyOnError=True))
def getRepositoryIds(self, domainId, repositoryGroupId=None):
result = JsonList()
allIds = self.getRepositoryGroupIds(domainId) if repositoryGroupId is None else [repositoryGroupId]
for repositoryGroupId in allIds:
jsonData = JsonDict.load(open(join(self._dataPath, '%s.%s.repositoryGroup' % (domainId, repositoryGroupId))))
result.extend(jsonData.get('repositoryIds', []))
return result
def getRepositoryIds(self, domainId, repositoryGroupId=None):
result = JsonList()
allIds = self.getRepositoryGroupIds(
domainId) if repositoryGroupId is None else [repositoryGroupId]
for repositoryGroupId in allIds:
jsonData = self.getRepositoryGroup(repositoryGroupId, domainId)
result.extend(jsonData.get('repositoryIds', []))
return result
def testLoadList(self):
jd = JsonList(['hello', 'world'])
tempfile = join(self.tempdir, 'json.json')
with open(tempfile, 'w') as fp:
fp.write(str(jd))
with open(tempfile) as fp:
jd2 = JsonList.load(fp)
self.assertEqual(jd, jd2)
def testLoadEmptyFile(self):
tempfile = join(self.tempdir, 'json.json')
with open(tempfile, 'w') as fp:
pass
self.assertRaises(json.JSONDecodeError,
lambda: JsonDict.load(tempfile))
self.assertEqual({}, JsonDict.load(tempfile, emptyOnError=True))
self.assertRaises(json.JSONDecodeError,
lambda: JsonList.load(tempfile))
self.assertEqual([], JsonList.load(tempfile, emptyOnError=True))
def testPrefixSearch(self):
self.response = JsonList([["value0", 1], ["value1", 2]]).dumps()
response = retval(self._lucene.prefixSearch(fieldname='field1', prefix='valu'))
self.assertEquals(['value1', 'value0'], response.hits)
response = retval(self._lucene.prefixSearch(fieldname='field1', prefix='valu', showCount=True))
self.assertEquals([('value1', 2), ('value0', 1)], response.hits)
def getRepositories(self, domainId, repositoryGroupId=None):
try:
repositoryIds = self.getRepositoryIds(
domainId=domainId, repositoryGroupId=repositoryGroupId)
except IOError:
raise ValueError("idDoesNotExist")
return JsonList([
self.getRepository(repositoryId, domainId)
for repositoryId in repositoryIds
])
def testPrefixSearch(self):
data = JsonList([
{"type": "TextField", "name": "prefixField", "value": "value0"},
{"type": "TextField", "name": "prefixField", "value": "value1"},
{"type": "TextField", "name": "prefixField", "value": "value2"},
]).dumps()
header, body = postRequest(self.luceneServerPort, self._path + '/update/?identifier=id1', data=data)
self.assertTrue("200 OK" in header.upper(), header)
header, body = postRequest(self.luceneServerPort, self._path + '/prefixSearch/?fieldname=prefixField&prefix=val', parse=False)
self.assertEqual([['value0', 1], ['value1', 1], ['value2', 1]], loads(body))
def testSuggestionRequest(self):
data = JsonList([
{"type": "TextField", "name": "field", "value": "value"},
]).dumps()
header, body = postRequest(self.luceneServerPort, self._path + '/update/?identifier=id1', data=data)
self.assertTrue("200 OK" in header.upper(), header)
header, body = postRequest(self.luceneServerPort, self._path + '/query/', parse=False, data=JsonDict(query=dict(type="MatchAllDocsQuery"), suggestionRequest=dict(field="field", count=1, suggests=['valeu'])).dumps())
jsonResponse = loads(body)
self.assertEqual({'valeu': ['value']}, jsonResponse["suggestions"])
self.assertTrue("suggestionTime" in jsonResponse["times"])
def parseHeaderAndBody(h, b=None, parseBody=True):
if b is None:
h, b = h
header, body = parseResponse(h + CRLF * 2 + b)
if body and parseBody and 'Content-Type' in header['Headers']:
contentType = header['Headers']['Content-Type']
if 'xml' in contentType:
return header, XML(body)
if 'json' in contentType:
try:
return header, JsonDict.loads(body) if body[0] == '{' else JsonList.loads(body)
except JSONDecodeError:
return header, 'JSONDecodeError in: ' + body
return header, body
def testFacets(self):
data = JsonList([
{"type": "TextField", "name": "fieldname", "value": "value"},
{"type": "FacetField", "name": "fieldname", "path": ["value"]}
]).dumps()
header, body = postRequest(self.luceneServerPort, self._path + '/update/?identifier=id1', data=data)
self.assertTrue("200 OK" in header.upper(), header)
header, body = postRequest(self.luceneServerPort, self._path + '/query/', data=JsonDict(query=dict(type="MatchAllDocsQuery"), facets=[{"fieldname": "fieldname", "maxTerms": 10}]).dumps(), parse=False)
self.assertTrue("200 OK" in header.upper(), header)
jsonResponse = loads(body)
self.assertEqual(1, jsonResponse['total'])
self.assertEqual([{'path': [], 'fieldname': 'fieldname', 'terms': [{'count': 1, 'term': 'value'}]}], jsonResponse['drilldownData'])
self.assertTrue("facetTime" in jsonResponse["times"])
def _suggest(self, arguments, outside):
prefix = arguments['prefix'][0]
fieldname = arguments.get('field', [self._defaultField])[0]
limit = int(arguments.get('limit', [self._defaultLimit])[0])
yield HttpOk
yield ContentTypeHeader + CONTENT_TYPE_JSON_SUGGESTIONS + CRLF
yield "Access-Control-Allow-Origin: *" + CRLF
yield "Access-Control-Allow-Headers: X-Requested-With" + CRLF
yield CRLF
hits = []
if len(prefix) >= self._minimumLength:
response = yield outside.prefixSearch(fieldname=fieldname,
prefix=prefix.lower(),
limit=limit)
hits = response.hits
yield JsonList([prefix, hits]).dumps()
def startServer(port, dataDir=None, jsonConfig=None, batchSize=None):
batchSize = batchSize or DEFAULT_BATCH_SIZE
setSignalHandlers()
tempDir = mkdtemp(prefix='mockoai-')
config = JsonList.loads(jsonConfig or '[]')
if dataDir:
config.append({'dirs': dataDir, 'path': '/'})
try:
reactor = Reactor()
server = be(dna(reactor, port, config=config, tempDir=tempDir, batchSize=batchSize))
print 'Ready to rumble the mock plein server at', port
list(compose(server.once.observer_init()))
registerShutdownHandler(statePath=tempDir, server=server, reactor=reactor)
stdout.flush()
reactor.loop()
finally:
rmtree(tempDir)
def testAddAndQueryDocument(self):
data = JsonList([
{"type": "TextField", "name": "fieldname", "value": "value"}
]).dumps()
header, body = postRequest(self.luceneServerPort, self._path + '/update/?identifier=id1', data=data)
self.assertTrue("200 OK" in header.upper(), header)
header, body = postRequest(self.luceneServerPort, self._path + '/query/', data=JsonDict(query=dict(type="MatchAllDocsQuery")).dumps(), parse=False)
self.assertTrue("200 OK" in header.upper(), header)
response = loads(body)
self.assertEqual(1, response['total'])
self.assertEqual([{'id': 'id1', 'score': 1.0}], response['hits'])
header, body = postRequest(self.luceneServerPort, self._path + '/query/', data=JsonDict(query=dict(type="TermQuery", term=dict(field="fieldname", value="value"))).dumps(), parse=False)
self.assertTrue("200 OK" in header.upper(), header)
response = loads(body)
self.assertEqual(1, response['total'])
self.assertTrue("queryTime" in response)
self.assertTrue("times" in response)
self.assertEqual([{'id': 'id1', 'score': 0.28768208622932434}], response['hits'])
def testCommit(self):
header, body = postRequest(self.luceneServerPort, self._path + '/settings/', data=JsonDict(commitCount=10).dumps())
self.assertTrue("200 OK" in header.upper(), header)
try:
data = JsonList([
{"type": "TextField", "name": "fieldname", "value": "value"}
]).dumps()
header, body = postRequest(self.luceneServerPort, self._path + '/update/?identifier=idCommit', data=data)
self.assertTrue("200 OK" in header.upper(), header)
header, body = postRequest(self.luceneServerPort, self._path + '/query/', parse=False, data=JsonDict(query=dict(type="TermQuery", term=dict(field="__id__", value="idCommit"))).dumps())
response = loads(body)
self.assertEqual(0, response['total'])
header, body = postRequest(self.luceneServerPort, '/commit/', parse=False)
self.assertTrue("200 OK" in header.upper(), header)
header, body = postRequest(self.luceneServerPort, self._path + '/query/', parse=False, data=JsonDict(query=dict(type="TermQuery", term=dict(field="__id__", value="idCommit"))).dumps())
response = loads(body)
self.assertEqual(1, response['total'])
finally:
header, body = postRequest(self.luceneServerPort, self._path + '/delete/?identifier=idCommit', data=data)
self.assertTrue("200 OK" in header.upper(), header)
def testLoadList(self):
jd = JsonList(['hello', 'world'])
tempfile = join(self.tempdir, 'json.json')
open(tempfile, 'w').write(str(jd))
jd2 = JsonList.load(open(tempfile))
self.assertEquals(jd, jd2)
def testPrettyPrintList(self):
jd = JsonList(['hello', 'world'])
printedList = jd.pretty_print(indent=5)
self.assertTrue('\n "hello"' in printedList)
self.assertTrue('\n "world"' in printedList)
def testLoadsList(self):
jd = JsonList(['hello', 'world'])
jd2 = JsonList.loads(str(jd))
self.assertEquals(jd, jd2)
def testPrettyPrintList(self):
jd = JsonList(['hello', 'world'])
printedList = jd.pretty_print(indent=5)
self.assertTrue('\n "hello"' in printedList)
self.assertTrue('\n "world"' in printedList)
def testStrList(self):
jl = JsonList(['hello', 'world'])
self.assertEquals('["hello", "world"]', str(jl))
class SuggestionIndexComponent(Observable):
def __init__(self, host, port, **kwargs):
super(SuggestionIndexComponent, self).__init__(**kwargs)
self._connect = _Connect(host, port, observable=self)
def addSuggestions(self, identifier, key, values):
titles = [v.get('title') for v in values]
types = [v.get('type') for v in values]
creators = [v.get('creator') for v in values]
yield self._connect.send(
"/add?{}".format(urlencode(dict(identifier=identifier))),
JsonDict(key=key, values=titles, types=types, creators=creators))
def deleteSuggestions(self, identifier):
yield self._connect.send("/delete?{}".format(
urlencode(dict(identifier=identifier))))
def registerFilterKeySet(self, name, keySet):
yield self._connect.send("/registerFilterKeySet?{}".format(
urlencode(dict(name=name))),
data=keySet)
def createSuggestionNGramIndex(self):
yield self._connect.send("/createSuggestionNGramIndex")
def suggest(self,
value,
trigram=False,
filters=None,
keySetName=None,
limit=None):
suggestions = yield self._connect.send(
"/suggest",
JsonDict(value=value,
trigram=trigram,
filters=filters or [],
keySetName=keySetName,
limit=limit))
raise StopIteration([Suggestion(s) for s in suggestions])
def indexingState(self):
indexingState = yield self._connect.read("/indexingState")
raise StopIteration(indexingState if indexingState else None)
def totalShingleRecords(self):
total = yield self._connect.read("/totalRecords", parse=False)
raise StopIteration(int(total))
def totalSuggestions(self):
total = yield self._connect.read("/totalSuggestions", parse=False)
raise StopIteration(int(total))
def ngramIndexTimestamp(self):
timestamp = yield self._connect.read("/ngramIndexTimestamp",
parse=False)
raise StopIteration(int(timestamp) / 1000.0)
def handleRequest(self, arguments, path, **kwargs):
value = arguments.get("value", [None])[0]
debug = arguments.get("x-debug", ["False"])[0] != 'False'
trigram = arguments.get("trigram", ["False"])[0] != 'False'
showConcepts = arguments.get("concepts", ["False"])[0] != 'False'
filters = arguments.get("filter", None)
minScore = float(arguments.get("minScore", ["0"])[0])
apikey = arguments.get("apikey", [None])[0]
apikeyFilter = arguments.get("x-apikey-filter", [''])[0]
if apikeyFilter:
apikey += "-" + apikeyFilter
suggest = None
if value:
t0 = time()
try:
suggest = yield self.suggest(value,
trigram=trigram,
filters=filters,
keySetName=apikey)
except Exception, e:
yield serverErrorPlainText
yield str(e)
return
tTotal = time() - t0
yield Ok
yield ContentTypeHeader + "application/x-suggestions+json" + CRLF
yield "Access-Control-Allow-Origin: *" + CRLF
yield "Access-Control-Allow-Headers: X-Requested-With" + CRLF
yield 'Access-Control-Allow-Methods: GET, POST, OPTIONS' + CRLF
yield 'Access-Control-Max-Age: 86400' + CRLF
yield CRLF
result = []
if value:
suggestions = []
for s in suggest:
suggestion = str(s.suggestion)
recordType = str(s.type) if s.type else None
creator = str(s.creator) if s.creator else None
distanceScore = max(
0,
-log(distance(value.lower(), suggestion.lower()) + 1) / 4 +
1)
matchScore = match(value.lower(), suggestion.lower())
score = float(s.score)
sortScore = distanceScore * score**2 * (matchScore * 2)
scores = dict(distanceScore=distanceScore,
score=score,
sortScore=sortScore,
matchScore=matchScore)
if sortScore > minScore:
suggestions.append(
(suggestion, recordType, creator, scores))
suggestions = sorted(
suggestions,
reverse=True,
key=lambda
(suggestion, recordType, creator, scores): scores['sortScore'])
if debug:
concepts = [(s, t, c) for s, t, c, _ in suggestions if t]
yield JsonDict(
dict(value=value,
suggestions=suggestions,
concepts=concepts,
time=tTotal)).dumps()
return
concepts = [(s, t, c) for s, t, c, _ in suggestions if t][:10]
dedupSuggestions = []
for s in suggestions:
if s[0] not in dedupSuggestions:
dedupSuggestions.append(s[0])
suggestions = dedupSuggestions[:10]
result = [value, suggestions]
if showConcepts:
result.append(concepts)
yield JsonList(result).dumps()
def handleRequest(self, *args, **kwargs):
yield "HTTP/1.0 200 OK\r\n"
yield "\r\n"
yield JsonList(args).pretty_print()
yield JsonDict(kwargs).pretty_print()
def getDomainIds(self):
return JsonList(self._store.listForDatatype('domain'))
def testConvertFromBytes(self):
converter = BytesToJson(fromKwarg='kwarg')
self.assertEqual(JsonDict({'aap': 3}),
converter._convert(b'{\n "aap": 3\n}'))
self.assertEqual(JsonList(['aap', 3]),
converter._convert(b'[\n "aap", 3\n]'))
def addDocument(self, fields, identifier=None):
if self._readonly:
raise RuntimeError('Adding documents not allowed for readonly Lucene connection.')
args = urlencode(dict(identifier=identifier)) if identifier else ''
yield self._connect().send(jsonDict=JsonList(fields), path='/update/?{}'.format(args))
def getStatus(self, **kwargs):
return JsonList([x for x in asList(self._getStatus(**kwargs))])
def testLoadList(self):
jd = JsonList(['hello', 'world'])
tempfile = join(self.tempdir, 'json.json')
open(tempfile, 'w').write(str(jd))
jd2 = JsonList.load(open(tempfile))
self.assertEquals(jd, jd2)
def testLoadsList(self):
jd = JsonList(['hello', 'world'])
jd2 = JsonList.loads(str(jd))
self.assertEquals(jd, jd2)
def _load(self):
if not isfile(self._jsonFilepath):
return {}
data = open(self._jsonFilepath).read().strip()
result = {}
if '[' != data[0]:
for identifier, serviceDict in JsonDict.loads(data).items():
service = Service(domainname=self._domainname, timeout=self._timeout, identifier=identifier, ultimateTimeout=self._ultimateTimeout, **serviceDict)
service.validate()
result[service.identifier] = service
return result
for service in (Service(domainname=self._domainname, timeout=self._timeout, ultimateTimeout=self._ultimateTimeout, **item) for item in JsonList.loads(data)):
service.validate()
result[service.identifier] = service
return result
