def printstats(stats):
if not isinstance(stats, list):
stats = [stats]
a = [
'headerline', 'Dim', 'Size', 'K', 'Var', 'NClus', 'IdxName', 'Cost',
'Average QueryTime', 'Average Calcs', 'Precision', 'Recall'
]
final = [a]
for s in stats:
if 'synthetic' in s.data.cfg:
c = s.data.cfg
a = [
'statline', c.D, c.S, c.K, c['variance'], c['nclus'], s.name,
s.cost, s.querytime, s.average, s.precision, s.recall
]
final.append(a)
else:
pass ### TODO
for s in final:
printl('\t'.join([str(x) for x in s]))
# for s in final:
# print("\t".join([str(s) for s in s]))
return final
def wrapper(*args, **kw):
ts = time.time()
result = func(*args, **kw)
te = time.time()
if len(args) > 0 and 'Data' == args[0].__class__.__name__:
printl('func<%r %s %s> took: %2.4f sec' %
(func.__name__, args[0].cfg.S, args[0].cfg.D, te - ts))
else:
printl('func<%r> took: %2.4f sec' % (func.__name__, te - ts))
return result
def process(data, overwriteindex=False, overwritebench=False):
indextype = data.cfg['mstype']
indextype = MSTypeEnum.prog(indextype)
print("---------------------------------------", data.cfg['mstype'],
indextype)
bprog = "build-%s-vectors" % indextype #specify the prog to use
qprog = "query-%s-vectors" % indextype #specify the query prog to use
if overwriteindex or not os.path.exists(data.msindexfilepath):
### annoyingly it seems the build has a filename limit
### make a temp file then rename
__, of = tempfile.mkstemp(dir=data.datadir,
prefix='tt{}_{}_'.format(
data.cfg.S, data.cfg.D))
cmd = [
bprog,
data.msbinfilepath,
"0",
of,
'3', # bucket size
'3',
'3'
]
runordel(cmd,
data.msbuildbenchfilepath,
outtofile=data.msbuildbenchfilepath,
overwrite=overwriteindex,
printcmd=True)
printl('renaming idx\n$> mv {} {}'.format(of, data.msindexfilepath))
os.rename(of, data.msindexfilepath)
cmd = [qprog, data.msindexfilepath]
if not os.path.exists(data.msbenchfilepath) or overwritebench:
if data.cfg['mstype'] == MSTypeEnum.prunedmvp:
cmdstr = "(time %s %s %d %f) < %s 1<&- 2> %s " % (
qprog, data.msindexfilepath, data.cfg.D,
data.cfg['prune_threshold'], data.qmsvecfilepath,
data.msbenchfilepath)
pst = PMSStatter(data.getFoldedFiles('msbenchfilepath'), data)
mean = pst.meanofall
var = pst.varofall
dev = pst.devofall
print("FFFF", mean, var, dev)
cmdstr = "(time %s %s %d %f %f %f %f) < %s 1<&- 2> %s " % (
qprog, data.msindexfilepath, data.cfg.D,
data.cfg['prune_threshold'], mean, var, dev,
data.qmsvecfilepath, data.msbenchfilepath)
else:
cmdstr = "(time %s %s) < %s 1<&- 2> %s " % (
qprog, data.msindexfilepath, data.qmsvecfilepath,
data.msbenchfilepath)
print(cmdstr)
call(cmdstr, shell=True)
def run(cmd, stdout=sys.stdout, stdin=None, stderr=None, printcmd=False):
cmd = [str(x) for x in cmd]
if printcmd:
printl('\n$>', " ".join(cmd), " > ", stdout.name, '\n')
oargs = {}
if stdin is not None: oargs = {'stdin': stdin}
if stderr is not None: oargs = {'stderr': stderr}
processcomplete = subprocess.run(cmd, stdout=stdout, **oargs)
try:
processcomplete.check_returncode()
except subprocess.CalledProcessError as e:
printe("Error running process", " ".join(cmd), " > ", stdout.name)
raise e
import runlsh
import runkd
import genGauss
from logger import printl, addLogFile
overwrite = True
if len(sys.argv) == 1:
sys.argv = sysarg.args(__file__)
args, unknown = sysarg.getParsed(sys.argv, True)
cfg = config.Config(vars(args))
data = dh.Data(cfg)
addLogFile(data.logfile)
printl('-#--------------------------------------#-')
printl(cfg)
genGauss.process(data)
printl("@@@@ Running LSH @@@@")
runlsh.process(data, overwrite)
printl("@@@@ Running KD @@@@")
runkd.fullprocess(data, overwrite)
# N0692-ZY350-78988-0XAK6-992NN
ls = lyz.FileStatter(data.lshbenchfilepath)
lsh = lyz.FileStatter(data.lshrfilepath)
kd = lyz.FileStatter(data.kdbenchfilepath)
ls.print()
printl("############")
lsh.print()
printl("############")