def epeStat(self):
if len(self._methods) > 1:
raise Exception('cannot make epe stat for more than one method')
method = self._methods[0]
entries = []
for ds in self._datasets:
for ent in ds.bents():
entries.append(ent.bind(method).flowStatParams())
cmd = 'FlowStat --epe-type=%s %s %s %s %s' % (
self._args.type, '--make-epe' if self._args.make_epe else '',
'--make-stat' if self._args.make_stat else '',
'--refresh' if self._args.refresh else '', '--stat')
print tb.run(cmd, '\n'.join(entries)),
def epeStat(self):
if len(self._methods) > 1:
raise Exception("cannot make epe stat for more than one method")
method = self._methods[0]
entries = []
for ds in self._datasets:
for ent in ds.bents():
entries.append(ent.bind(method).flowStatParams())
cmd = "FlowStat --epe-type=%s %s %s %s %s" % (
self._args.type,
"--make-epe" if self._args.make_epe else "",
"--make-stat" if self._args.make_stat else "",
"--refresh" if self._args.refresh else "",
"--stat",
)
print tb.run(cmd, "\n".join(entries)),
def epeLists(methods,datasets,type='all'):
epe = {}
for m in methods:
entries = []
for ds in datasets:
for ent in ds.bents():
entries.append(ent.bind(m).flowStatParams())
cmd = 'FlowStat --epe-type=%s' % (
type,
)
epe[str(m)] = [line for line in tb.run(cmd, '\n'.join(entries)).split('\n') if line.strip() != '']
return epe
def preparePythonBackend():
os.system("mkdir -p training")
folder = os.path.dirname(caffe.__file__)
print "copying %s to training" % folder
os.system("cp %s training -r" % folder)
ldd = tb.run("ldd %s" % caffe._caffe.__file__)
caffeLib = None
for line in ldd.split("\n"):
match = re.match("\\s*libcaffe.so => (.*\.so)", line)
if match:
caffeLib = match.group(1)
break
if caffeLib is None:
raise Exception("cannot find libcaffe.so dependency")
print "copying %s to training" % caffeLib
os.system("cp %s %s" % (caffeLib, env.trainDir()))
def preparePythonBackend():
os.system('mkdir -p training')
folder = os.path.dirname(caffe.__file__)
print 'copying %s to training' % folder
os.system('cp %s training -r' % folder)
ldd = tb.run('ldd %s' % caffe._caffe.__file__)
caffeLib = None
for line in ldd.split('\n'):
match = re.match('\\s*libcaffe.so => (.*\.so)', line)
if match:
caffeLib = match.group(1)
break
if caffeLib is None:
raise Exception('cannot find libcaffe.so dependency')
print 'copying %s to training' % caffeLib
os.system('cp %s %s' % (caffeLib, env.trainDir()))
def _callCopiedBin(self, cmd):
bin = './' + os.path.basename(caffeBin())
tb.notice('making a local copy of %s' % caffeBin())
os.system('cp %s .' % caffeBin())
ldd = tb.run('ldd %s' % caffeBin())
caffeLib = None
for line in ldd.split('\n'):
match = re.match('\\s*libcaffe.so => (.*\.so)', line)
if match:
caffeLib = match.group(1)
break
if caffeLib is None:
raise Exception('cannot find libcaffe.so dependency')
tb.notice('making a local copy of %s' % caffeLib)
os.system('cp %s .' % caffeLib)
cmd = 'GLOG_logtostderr=%d LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH %s %s' % (not self._quiet, bin, cmd)
if not self._silent:
tb.notice('running "%s"' % cmd, 'run')
tb.system(cmd)
def _callCopiedBin(self, cmd):
bin = './' + os.path.basename(caffeBin())
tb.notice('making a local copy of %s' % caffeBin())
os.system('cp %s .' % caffeBin())
ldd = tb.run('ldd %s' % caffeBin())
caffeLib = None
for line in ldd.split('\n'):
match = re.match('\\s*libcaffe.so => (.*\.so)', line)
if match:
caffeLib = match.group(1)
break
if caffeLib is None:
raise Exception('cannot find libcaffe.so dependency')
tb.notice('making a local copy of %s' % caffeLib)
os.system('cp %s .' % caffeLib)
cmd = 'GLOG_logtostderr=%d LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH %s %s' % (
not self._quiet, bin, cmd)
if not self._silent:
tb.notice('running "%s"' % cmd, 'run')
tb.system(cmd)
if args.gpu_id is not None:
gpuIds = "%d" % args.gpu_id
else:
for i in range(0, args.gpus):
gpuIds += ",%d" % i
gpuIds = gpuIds[1:]
if args.backend == "binary":
backend = BinaryBackend(gpuIds, args.quiet, args.silent)
else:
backend = PythonBackend(gpuIds, args.quiet, args.silent)
if args.execute:
print "using caffe module from: %s" % caffe.__file__
print "using caffe._caffe module from: %s" % caffe._caffe.__file__
ldd = tb.run("ldd %s" % caffe._caffe.__file__)
caffeLib = None
for line in ldd.split("\n"):
match = re.match("\\s*libcaffe.so => (.*\.so)", line)
if match:
caffeLib = match.group(1)
break
if caffeLib is None:
raise Exception("cannot find libcaffe.so dependency")
print "using caffe from %s" % caffeLib
env = Environment(args.path, backend, args.unattended, args.silent)
if args.command != "copy" and args.command != "compare":
env.init()
def epe(self):
epe = {}
for m in self._methods:
entries = []
for ds in self._datasets:
for ent in ds.bents():
entries.append(ent.bind(m).flowStatParams())
cmd = 'FlowStat --epe-type=%s %s %s %s' % (
self._args.type, '--make-epe' if self._args.make_epe else '',
'--make-stat' if self._args.make_stat else '',
'--refresh' if self._args.refresh else '')
epe[str(m)] = [
line for line in tb.run(cmd, '\n'.join(entries)).split('\n')
if line.strip() != ''
]
if self._args.no_output:
return
fw = [0]
for m in epe:
fw.append(max(len(m), 10))
for ds in self._datasets:
for ent in ds.bents():
l = len(ent.detailedName())
if l > fw[0]:
fw[0] = l
s = 0
for n in fw:
s += n
s += len(fw) - 1
if self._args.list:
if self._args.plain:
i = 0
for ds in self._datasets:
for ent in ds.bents():
for list in epe.itervalues():
print list[i],
print
i += 1
for list in epe.itervalues():
print list[-1],
print
else:
print ' ' * fw[0],
j = 1
for m in epe.iterkeys():
print '{0:>{1}s}'.format(m, fw[j]),
j += 1
print
i = 0
print '-' * s
for ds in self._datasets:
for ent in ds.bents():
print '{0:{1}s}'.format(ent.detailedName(), fw[0]),
j = 1
for list in epe.itervalues():
print '{0:{1}s}'.format(list[i], fw[j]),
j += 1
print
i += 1
print '-' * s
print '{0:{1}s}'.format('average', fw[0]),
j = 1
for list in epe.itervalues():
print '{0:>{1}s}'.format(
list[-1].replace('avg=', '').strip(), fw[j]),
j += 1
print
else:
if len(entries) > 1:
maxMLen = 0
for i in range(1, len(fw)):
if fw[i] > maxMLen: maxMLen = fw[i]
for m, list in epe.iteritems():
print '{0:{1}s}'.format(m, maxMLen),
if not len(list):
print
continue
print '{0:>{1}s}'.format(
list[-1].replace('avg=', '').strip(), 10),
print
else:
for list in epe.itervalues():
print list[-1].replace('avg=', '').strip()
def epe(self):
epe = {}
for m in self._methods:
entries = []
for ds in self._datasets:
for ent in ds.bents():
entries.append(ent.bind(m).flowStatParams())
cmd = "FlowStat --epe-type=%s %s %s %s" % (
self._args.type,
"--make-epe" if self._args.make_epe else "",
"--make-stat" if self._args.make_stat else "",
"--refresh" if self._args.refresh else "",
)
epe[str(m)] = [line for line in tb.run(cmd, "\n".join(entries)).split("\n") if line.strip() != ""]
if self._args.no_output:
return
fw = [0]
for m in epe:
fw.append(max(len(m), 10))
for ds in self._datasets:
for ent in ds.bents():
l = len(ent.detailedName())
if l > fw[0]:
fw[0] = l
s = 0
for n in fw:
s += n
s += len(fw) - 1
if self._args.list:
if self._args.plain:
i = 0
for ds in self._datasets:
for ent in ds.bents():
for list in epe.itervalues():
print list[i],
print
i += 1
for list in epe.itervalues():
print list[-1],
print
else:
print " " * fw[0],
j = 1
for m in epe.iterkeys():
print "{0:>{1}s}".format(m, fw[j]),
j += 1
print
i = 0
print "-" * s
for ds in self._datasets:
for ent in ds.bents():
print "{0:{1}s}".format(ent.detailedName(), fw[0]),
j = 1
for list in epe.itervalues():
print "{0:{1}s}".format(list[i], fw[j]),
j += 1
print
i += 1
print "-" * s
print "{0:{1}s}".format("average", fw[0]),
j = 1
for list in epe.itervalues():
print "{0:>{1}s}".format(list[-1].replace("avg=", "").strip(), fw[j]),
j += 1
print
else:
if len(entries) > 1:
maxMLen = 0
for i in range(1, len(fw)):
if fw[i] > maxMLen:
maxMLen = fw[i]
for m, list in epe.iteritems():
print "{0:{1}s}".format(m, maxMLen),
if not len(list):
print
continue
print "{0:>{1}s}".format(list[-1].replace("avg=", "").strip(), 10),
print
else:
for list in epe.itervalues():
print list[-1].replace("avg=", "").strip()
if args.gpu_id is not None:
gpuIds = '%d' % args.gpu_id
else:
for i in range(0, args.gpus):
gpuIds += ',%d' % i
gpuIds = gpuIds[1:]
if args.backend == 'binary':
backend = BinaryBackend(gpuIds, args.quiet, args.silent)
else:
backend = PythonBackend(gpuIds, args.quiet, args.silent)
if args.execute:
print 'using caffe module from: %s' % caffe.__file__
print 'using caffe._caffe module from: %s' % caffe._caffe.__file__
ldd = tb.run('ldd %s' % caffe._caffe.__file__)
caffeLib = None
for line in ldd.split('\n'):
match = re.match('\\s*libcaffe.so => (.*\.so)', line)
if match:
caffeLib = match.group(1)
break
if caffeLib is None:
raise Exception('cannot find libcaffe.so dependency')
print 'using caffe from %s' % caffeLib
env = Environment(args.path, backend, args.unattended, args.silent)
if args.command != 'copy' and args.command != 'compare': env.init()
def checkJob():
