def merge_ranked(msg):
msg = cStringIO.StringIO(msg)
r = cStringIO.StringIO()
num_hits = 0
while True:
try:
iblock_ranked = decode_netstring_fd(msg)
r.write(iblock_ranked['ranked'])
num_hits += int(iblock_ranked['num_hits'])
except EOFError:
break
top = ainodex.merge_ranked(r.getvalue())
erlay.report("Top keys: %s" % array.array("I", top)[:20:2])
return encode_netstring_fd({'merged': top, 'num_hits': str(num_hits)})
erlay.report("--- merger starts ---")
ainodex.open()
if "TESTDEX" in os.environ:
globals()[sys.argv[1]](file(sys.argv[2]).read())
else:
erlay.register_str(sys.argv[1], 'dex/merger')
erlay.serve_forever(globals())
return encode_netstring_fd(ret)
erlay.report("--- dex [%s/%d] starts ---" % (NAME, IBLOCK))
ainodex.open()
ainopy.open()
try:
seg_hashes = map(int, file("%s.%d.seghash" % (NAME, IBLOCK)).readlines())
erlay.report("Segment hashes read. Duplicate checking enabled.")
except:
erlay.report("Could not open segment hashes. Duplicate checking disabled.")
seg_hashes = None
try:
sites = read_sites()
erlay.report("Site hashes read. Site checking enabled.")
except:
erlay.report("Could not open site hashes. Site checking disabled.")
sites = None
if "TESTDEX" in os.environ:
globals()[sys.argv[1]](file(sys.argv[2]).read())
else:
erlay.register_str(sys.argv[1], 'dex/iblock:%d' % IBLOCK)
erlay.serve_forever(globals())
(query_msg['keys'], query_msg['cues'],
(time.time() - t) * 1000.0))
return encode_netstring_fd(ret)
erlay.report("--- dex [%s/%d] starts ---" % (NAME, IBLOCK))
ainodex.open()
ainopy.open()
try:
seg_hashes = map(int, file("%s.%d.seghash" % (NAME, IBLOCK)).readlines())
erlay.report("Segment hashes read. Duplicate checking enabled.")
except:
erlay.report("Could not open segment hashes. Duplicate checking disabled.")
seg_hashes = None
try:
sites = read_sites()
erlay.report("Site hashes read. Site checking enabled.")
except:
erlay.report("Could not open site hashes. Site checking disabled.")
sites = None
if "TESTDEX" in os.environ:
globals()[sys.argv[1]](file(sys.argv[2]).read())
else:
erlay.register_str(sys.argv[1], 'dex/iblock:%d' % IBLOCK)
erlay.serve_forever(globals())
#erlay.report("CUES <%s>" % cues)
#erlay.report("KEYS <%s>" % keys)
if site_check:
base_q = " ".join(keys) + " ".join(cues)
else:
base_q = ""
#erlay.report("BASE <%s>" % base_q)
if not cues:
cues = keys
return results_tmpl(int(rank_msg['num_hits']), query_q, base_q,\
nice_re.sub("", " ".join(cues)),
nice_re.sub("", " ".join(keys)), results, offs)
erlay.report("--- renderer starts ---")
ainodex.open()
rixicon = ainodex.ixicon(0)
t = time.time()
read_docs()
erlay.report("Doc index read in %dms" % ((time.time() - t) * 1000.0))
if "TESTDEX" in os.environ:
globals()[sys.argv[1]](file(sys.argv[2]).read())
else:
erlay.register_str(sys.argv[1], 'render')
erlay.serve_forever(globals())
if site_check:
base_q = " ".join(keys) + " ".join(cues)
else:
base_q = ""
#erlay.report("BASE <%s>" % base_q)
if not cues:
cues = keys
return results_tmpl(int(rank_msg['num_hits']), query_q, base_q,\
nice_re.sub("", " ".join(cues)),
nice_re.sub("", " ".join(keys)), results, offs)
erlay.report("--- renderer starts ---")
ainodex.open()
rixicon = ainodex.ixicon(0)
t = time.time()
read_docs()
erlay.report("Doc index read in %dms" % ((time.time() - t) * 1000.0))
if "TESTDEX" in os.environ:
globals()[sys.argv[1]](file(sys.argv[2]).read())
else:
erlay.register_str(sys.argv[1], 'render')
erlay.serve_forever(globals())