def profile(self, repo):
# for gprof we can run each test in the application
# however, we must move and/or rename gman.out each time
# and then we should go ahead and read hatchet profile
profiles = {}
for test in repo.itertests():
vprint(self.verbose,
'Profiling test \'{}\'...'.format(test['name']))
exec_path = os.path.join(test['prefix'], test['executable'])
cmd = Command(exec_path)
try:
cmd(test['args'])
except:
vprint(self.verbose,
'Running test \'{}\' failed...'.format(test['name']))
continue
if not os.path.isfile('gmon.out'):
vprint(self.verbose, 'Unable to read profile...')
continue
# read in profile with gprof and gprof2dot
dotfile_name = 'profile-dot-graph.dot'
gprof = Command('gprof')
gprof2dot = Command('gprof2dot')
with open(dotfile_name, 'w+') as outFile:
gprof2dot(gprof(exec_path), '-n0', '-e0', _out=outFile)
# finally read this into hatchet
gf = ht.GraphFrame.from_gprof_dot(dotfile_name)
profiles[test['name']] = gf
return profiles
def generate_pkgspec(pkgoutdir, spectemplate, pkgname):
obsservicedir = '/usr/lib/obs/service/'
outdir = ('--outdir', pkgoutdir)
olddir = os.getcwd()
try:
os.chdir(pkgoutdir)
renderspec = Command(os.path.join(obsservicedir, 'renderspec'))
renderspec('--input-template', os.path.join(olddir, spectemplate),
'--output-name', pkgname + '.spec', *outdir)
format_spec_file = Command(
os.path.join(obsservicedir, 'format_spec_file'))
format_spec_file(*outdir)
# configure a download cache to avoid downloading the same files
download_env = os.environ.copy()
download_env["CACHEDIRECTORY"] = os.path.join(os.path.expanduser("~"),
".cache",
"download_files")
download_files = Command(os.path.join(obsservicedir, 'download_files'))
download_files(_env=download_env, *outdir)
finally:
os.chdir(olddir)
def task(dataset, _job):
startTime = time.time()
vtp.setJob(_job)
vtp.gotoTmp()
rm('-rf', 'vehicleTracking')
vtp.getVT()
vtp.cmakeVT()
vtp.makeVT('basicDetector')
basicDetector = Command("util/basicDetector")
vtp.makeVT('detectionAccuracy')
detectionAccuracy = Command("util/detectionAccuracy")
cp('-r', vtp.srcDir + '/data/labels/skycomp1', '.')
results = dict()
for threshold in np.arange(5, 251, 5):
basicDetector('-r', x, y, w, h, '-s', sz, '-n', n, '-g', vtp.gpuDev,
'-t', threshold, "-d", "diff", dataset)
out = vtp.detectionAccuracy(l=dataset,
d='detections.pb',
t=' '.join(str(t) for t in trainFrames),
T=' '.join(str(t) for t in testFrames))
results[threshold] = out
return results
def run(cls, root_path, test, base_url):
logging.debug("run test")
module, class_name, test_name = (test.module, test.class_name,
test.test_name)
temp_path = mkdtemp()
try:
logging.debug("creating venv")
venv_path = "%s/venv" % temp_path.rstrip('/')
virtualenv.create_environment(venv_path,
site_packages=False,
unzip_setuptools=True,
use_distribute=True)
logging.debug("installing funkload")
pip = Command("%s/bin/pip" % venv_path)
pip.install(FUNKLOAD_GIT)
for dep in test.deps:
logging.debug("install deps")
pip.install(dep)
logging.debug("creating command")
fl_run_test = Command("%s/bin/fl-run-test" % venv_path)
logging.debug("command created")
logging.debug("run command: %s" %
("%s/bin/fl-run-test %s %s -u %s" %
(venv_path, module,
("%s.%s" % (class_name, test_name)), base_url)))
result = fl_run_test(module,
"%s.%s" % (class_name, test_name),
u=base_url,
_env={
"PYTHONPATH":
'$PYTHONPATH:%s' %
join(root_path.rstrip('/'), "bench")
},
simple_fetch=True,
_cwd=temp_path)
logging.debug("command run")
logging.debug("get result")
exit_code = result.exit_code
text = result.stdout + result.stderr
with open(join(temp_path, 'funkload.log')) as fl_log:
logging.debug("write log")
log = fl_log.read()
except ErrorReturnCode:
err = sys.exc_info()[1]
text = err.stderr
exit_code = 1
log = err.stderr + err.stdout
logging.error(log)
logging.debug("test run")
return FunkLoadTestRunResult(exit_code, text, log)
def __init__(self, profiler_settings, verbose=False):
super().__init__(profiler_settings, verbose)
self.hpcrun_cmd = Command('hpcrun')
self.hpcrun_cmd = self.hpcrun_cmd.bake('-e', 'WALLCLOCK@5000')
self.hpcstruct_cmd = Command('hpcstruct')
self.hpcprof_cmd = Command('mpirun')
self.hpcprof_cmd = self.hpcprof_cmd.bake('-np', '1', 'hpcprof-mpi',
'--metric-db', 'yes')
def test_command_wrapper(self):
from sh import Command, which
ls = Command(which("ls"))
wc = Command(which("wc"))
c1 = int(wc(ls("-A1"), l=True))
c2 = len(os.listdir("."))
self.assertEqual(c1, c2)
def task(dataset, modelLocation, frameDiff, detectorSize, netParams, _job):
for k in netParams.keys():
netParams[k] = int(netParams[k])
startTime = time.time()
vtp.setJob(_job)
vtp.gotoTmp()
rm('-rf', 'vehicleTracking')
vtp.getVT()
vtp.cmakeParams.append('-DTRAIN_ITERATIONS=' + str(caffeIterations))
vtp.cmakeParams.append('-DDETECTOR_WIDTH=' + str(detectorSize))
vtp.cmakeParams.append('-DDETECTOR_HEIGHT=' + str(detectorSize))
vtp.changeNetParams(**netParams)
vtp.cmakeVT()
vtp.makeVT('basicDetector')
basicDetector = Command("util/basicDetector")
vtp.makeVT('detectionAccuracy')
detectionAccuracy = Command("util/detectionAccuracy")
cp('-r', vtp.srcDir + '/data/labels/skycomp1', '.')
cp(vtp.srcDir + '/../negatives.yml', '.')
mkdir('-p', 'src/caffe')
cp(
'{0}/vehicle_detector_train_iter_{1}.caffemodel'.format(
modelLocation, caffeIterations), 'src/caffe/')
cp('{0}/mean.cvs'.format(modelLocation), 'src/caffe/')
vtp.makeVT('buildNet')
results = dict()
for threshold in np.arange(-2.0, 4.0, 0.2):
print "running with threshold = {0}".format(threshold)
bdArgs = [
'-r', x, y, w, h, '-s', sz, '-n', n, '-g', vtp.gpuDev, '-t',
threshold, dataset
]
if frameDiff != 0:
bdArgs.append('-f')
bdArgs.append(frameDiff)
vtp.basicDetector(*bdArgs)
out = vtp.detectionAccuracy(l=dataset,
d='detections.pb',
t=' '.join(str(t) for t in trainFrames),
T=' '.join(str(t) for t in testFrames))
results[threshold] = out
return results
def task(index):
hmw = Command('hyperopt-mongo-worker')
hmw('--mongo={0}:{1}/{2}'.format(hostname, 1234, dbname),
_out=_print_line,
_err=_print_line)
return 'done'
def exe(ctx, context, target_url, pending, success, failure, error, command):
def _status_set(state, description):
log.info('%s -> %s', context, state)
ctx.invoke(status_set,
context=context,
target_url=target_url,
state=state,
description=description)
try:
_status_set('pending', pending)
cmd = Command(command[0])
cmd = cmd.bake(command[1:]) if len(command) > 1 else cmd
cmd(_fg=True)
_status_set('success', success)
exit(0)
except ErrorReturnCode as e:
log.error('Command failed with exit code: %d', e.exit_code)
_status_set('failure', failure)
except CommandNotFound as e:
log.error('Command not found: %s', e)
_status_set('error', error)
except Exception as e:
log.error(e, exc_info=True)
_status_set('error', error)
exit(1)
def jenkins_job_trigger(repo, github_opts, cloudsource, ptfdir):
print("triggering jenkins job with " + htdocs_url + ptfdir)
jenkins = Command(
os.path.abspath(
os.path.join(os.path.dirname(sys.argv[0]),
'jenkins/jenkins-job-trigger')))
job_parameters = (
'nodenumber=2', 'networkingplugin=openvswitch')
if repo in JOB_PARAMETERS:
job_parameters = JOB_PARAMETERS[repo]
job_parameters += ('all_noreboot',)
print(jenkins(
'openstack-mkcloud',
'-p',
"github_pr=crowbar/%s:%s" % (repo, github_opts),
"cloudsource=" + cloudsource,
# trailing slash required
# to prevent wget from traversing all test-updates
'UPDATEREPOS=' + htdocs_url + ptfdir + "/",
'mkcloudtarget=all_noreboot',
*job_parameters))
def mount_nas():
try:
mountnas = Command("dims.nas.mount")
mountnas()
except Exception as exception_:
print u"[!] Exception: {0}".format(exception_)
cleanexit(1)
def detectionAccuracy(self, **kwargs):
da = Command("util/detectionAccuracy")
out = da(**kwargs)
print "Got stdout from detectionAccuracy:"
print out.stdout
mode = ['TRAIN', 'TEST']
value = ['TP', 'FP', 'DP', 'FN']
pattern = ''.join('{0} {1}:\s+(?P<{0}_{1}>\d+).*'.format(m,v) for m in mode for v in value)
match = re.search(pattern, out.stdout, re.DOTALL)
if not match:
return None
d = match.groupdict()
d = dict([(k, int(v)) for k,v in d.iteritems() ])
tp = float(d['TEST_TP'])
fn = float(d['TEST_FN'])
dp = float(d['TEST_DP'])
fp = dp + float(d['TEST_FP'])
if tp > 0:
p = tp/(tp+fp)
r = tp/(tp+fn)
d['TEST_P'] = p
d['TEST_R'] = r
d['TEST_F2'] = 5.0*p*r/(4*p+r)
d['TEST_MR'] = fn/(fn+tp)
if 'T' in kwargs:
n = len(kwargs['T'])
d['TEST_FPPI'] = fp/float(n)
return d
def doSim(err, fileName, weights, _job):
vtp.setJob(_job)
vtp.gotoTmp()
rm('-rf', 'vehicleTracking')
vtp.getVT()
vtp.cmakeVT()
vtp.makeVT('tracker')
tracker = Command("javaTracker/runTracker.sh")
ret = {}
for w in weights:
rm('-rf', 'output.csv')
for line in tracker(f=fileName,
p=err,
n=err,
g=False,
a=5,
w=w,
_iter=True):
print(line)
with open('output.csv', 'r') as f:
out = f.read().split(',')
out = [o.strip() for o in out]
print out
ret[w] = out
vtp.export('frame_output.csv')
return ret
def get_hosts(inventory_path, group_name):
ansible_inventory = Command('ansible-inventory')
json_inventory = json.loads(
ansible_inventory('-i', inventory_path, '--list').stdout)
if group_name not in json_inventory:
raise AssertionError('Group %r not found.' % group_name)
hosts = []
if 'hosts' in json_inventory[group_name]:
return json_inventory[group_name]['hosts']
else:
children = json_inventory[group_name]['children']
for child in children:
if 'hosts' in json_inventory[child]:
for host in json_inventory[child]['hosts']:
if host not in hosts:
hosts.append(host)
else:
grandchildren = json_inventory[child]['children']
for grandchild in grandchildren:
if 'hosts' not in json_inventory[grandchild]:
raise AssertionError('Group nesting cap exceeded.')
for host in json_inventory[grandchild]['hosts']:
if host not in hosts:
hosts.append(host)
return hosts
def call(name, *args, _show=True, _capture=False, _ignore=False):
"""Call a shell program with arguments."""
msg = CMD_PREFIX + ' '.join([name] + list(args))
if _show:
common.show(msg)
else:
log.debug(msg)
if name == 'cd' and len(args) == 1:
return os.chdir(args[0])
try:
program = Command(name)
if _capture:
line = program(*args).strip()
log.debug(OUT_PREFIX + line)
return line
else:
for line in program(*args, _iter='err'):
log.debug(OUT_PREFIX + line.strip())
except ErrorReturnCode as exc:
msg = "\n IN: '{}'{}".format(os.getcwd(), exc)
if _ignore:
log.debug("Ignored error from call to '%s'", name)
else:
raise ShellError(msg)
def _run_alembic_upgrade():
import sh
from sh import Command
try:
flask_cmd = Command('flask')
flask_cmd('db', 'upgrade')
except sh.ErrorReturnCode as e:
print(e.stderr)
def ghs_set_status(repo, head_sha1, status):
ghs = Command(
os.path.abspath(
os.path.join(os.path.dirname(sys.argv[0]),
'github-status/github-status.rb')))
ghs('-r', 'crowbar/' + repo,
'-c', head_sha1, '-a', 'set-status', '-s', status)
def ghs_set_status(org, repo, head_sha1, status):
ghs = Command(
os.path.abspath(
os.path.join(os.path.dirname(sys.argv[0]),
'github-status/github-status.rb')))
ghs('-r', org + '/' + repo,
'--context', 'suse/mkcloud/testbuild',
'-c', head_sha1, '-a', 'set-status', '-s', status)
def run(self, cache, args=[]):
if self.url.fragment is None:
raise Invalid('Arx can not execute tarball URLs that have no '
'fragment.')
program = cache.join('program', self.url.fragment.split('/')[-1])
self.place(cache, program)
chmod('a+rx', str(program))
cmd = Command(str(program))
cmd(*args)
def runFile(self, blendFile):
imageDir = self.imageDirectory + blendFile.split("/")[-1].replace(
".blend", "") + "/"
try:
cmd = Command(self.blenderplayer)
upbgeargs = ("-p", self.pythonMainScript, blendFile, "-", imageDir)
cmd.run(upbgeargs, _out=debugSh)
except ErrorReturnCode:
return False
return True
def add_pr_to_checkout(repo, pr_id, head_sha1, pr_branch, spec):
sh.curl(
'-s', '-k', '-L',
"https://github.com/crowbar/%s/compare/%s...%s.patch" %
(repo, pr_branch, head_sha1), '-o', 'prtest.patch')
sh.sed('-i', '-e', 's,Url:.*,%define _default_patch_fuzz 2,', '-e',
's,%patch[0-36-9].*,,', spec)
Command('/usr/lib/build/spec_add_patch')(spec, 'prtest.patch')
iosc('vc', '-m',
"added PR test patch from %s#%s (%s)" % (repo, pr_id, head_sha1))
def runFile(self, blendFile):
try:
cmd = Command(self.blenderplayer)
upbgeargs = ("-p", self.pythonMainScript, blendFile)
if len(self.args) > 0:
upbgeargs += ("-", self.args)
cmd.run(upbgeargs, _out=debugSh)
except ErrorReturnCode:
return False
return True
def check_llvm():
try:
llvm_path = './llvm/Release+Asserts'
assert isfile(llvm_path+'/lib/ocaml/llvm.cma') # is llvm.cma there?
llvm_config = Command(llvm_path+'/bin/llvm-config')
ver = llvm_config('--version') # try calling llvm-config
print ver
assert '3.2' in ver
return True
except:
return False
def task(dataset, threshold, frameDiff, iteration, _job):
startTime = time.time()
vtp.setJob(_job)
vtp.gotoTmp()
rm('-rf', 'vehicleTracking')
vtp.getVT()
vtp.cmakeParams.append('-DTRAIN_ITERATIONS='+str(caffeIterations))
vtp.cmakeVT()
vtp.makeVT('labeledDataToDB')
labeledDataToDB = Command("util/labeledDataToDB")
vtp.makeVT('basicDetector')
#basicDetector = Command("util/basicDetector")
vtp.makeVT('detectionAccuracy')
#detectionAccuracy = Command("util/detectionAccuracy")
cp('-r', vtp.srcDir+'/data/labels/skycomp1', '.')
cp(vtp.srcDir+'/../negatives.yml', '.')
results = []
i = -1
#for i in range(trainIterations):
bestF2 = 0.0
while time.time() < (startTime + 60*60*runHours):
i+=1
rm('-rf', 'src/caffe/train.leveldb', 'src/caffe/test.leveldb')
labeledDataParams = dict(l=dataset, n='negatives.yml',
t=' '.join(str(t) for t in trainFrames),
T=' '.join(str(t) for t in testFrames))
if i != 0:
labeledDataParams['d']='detections.pb'
if frameDiff:
labeledDataParams['f']=frameDiff
labeledDataToDB(**labeledDataParams)
vtp.makeVT('trainNet')
vtp.makeVT('buildNet')
bdArgs = ['-r', x, y, w, h, '-s', sz, '-n', n, '-g', vtp.gpuDev, '-t', threshold, dataset ]
if frameDiff != 0:
bdArgs.append('-f')
bdArgs.append(frameDiff)
vtp.basicDetector(*bdArgs)
out = vtp.detectionAccuracy(l=dataset, d='detections.pb',
t=' '.join(str(t) for t in trainFrames),
T=' '.join(str(t) for t in testFrames))
results.append(out)
if out['TEST_F2'] > bestF2:
bestF2 = out['TEST_F2']
vtp.export('src/caffe/mean.cvs')
vtp.export('src/caffe/vehicle_detector_train_iter_'+str(caffeIterations)+'.caffemodel')
vtp.export('negatives.yml')
return results
def run(self, cache, args=[]):
if not True:
raise Invalid('Directories can not be run as commands.')
if not os.access(str(self.resolved), os.X_OK):
if self.dot:
chmod('a+rx', str(self.resolved))
else:
log.error('Not able to mark `%s` as executable :/' %
self.resolved)
cmd = Command(str(self.resolved))
cmd(*args)
def _run_query(self, input_string):
# Trying to make the buffering process working.
try:
self.output = StringIO()
def grab_output(line):
self.output.write(line)
self.gnash = Command("./Gnash.exe")
self.gnash(_in=input_string, _out=grab_output).wait()
except:
print "Error, cannot run the query on Gnash"
def system(self, progname, *args):
dprint("EXEC", progname, *["'{}'".format(arg) for arg in args])
env = {k: v for k, v in self.variables.items() if k in self.exports}
command = Command(progname)
try:
return command(*args,
_in=self.stdin,
_out=self.stdout,
_err=self.stderr,
_cwd=self.cwd,
_env=env).exit_code
except ErrorReturnCode as e:
return e.exit_code
def get_command(self):
""" Returns a reusable sh.Command object that can execute multiple different SFTP commands.
"""
# A list of arguments that will be added to the base command
args = []
# Buffer size is always available
args.append('-B')
args.append(self.buffer_size)
# Bandwidth limit is always available
args.append('-l')
args.append(self.bandwidth_limit)
# Preserving file and directory metadata is optional
if self.should_preserve_meta:
args.append('-p')
# Immediate flushing is optional
if self.should_flush:
args.append('-f')
# Compression is optional
if self.is_compression_enabled:
args.append('-C')
# Forcing a particular IP version is optional
if self.force_ip_type:
args.append(ip_type_map[self.force_ip_type])
# Port is optional
if self.port:
args.append('-P')
args.append(self.port)
# Identity file is optional
if self.identity_file:
args.append('-i')
args.append(self.identity_file)
# SSH config file is optional
if self.ssh_config_file:
args.append('-F')
args.append(self.ssh_config_file)
# Base command to build additional arguments into
command = Command(self.sftp_command)
command = command.bake(*args)
return command
def __init__(self,
scylla_data_dir,
db_path,
storage_obj,
nodetool_path='/usr/bin/nodetool',
cqlsh_path='/usr/bin/cqlsh',
cqlsh_host='127.0.0.1',
cqlsh_port='9042',
prefix='scyllabackup',
max_workers=4):
self.scylla_data_dir = scylla_data_dir
self.db = DB(db_path)
self.db_path = db_path
self.nodetool = Command(nodetool_path)
self.cqlsh = Command(cqlsh_path).bake(cqlsh_host, cqlsh_port)
self._upload_queue = gevent.queue.JoinableQueue()
self._download_queue = gevent.queue.JoinableQueue()
self._delete_queue = gevent.queue.JoinableQueue()
self._verify_queue = gevent.queue.JoinableQueue()
self._storage = storage_obj
self._prefix = prefix
self.db_key = self._prefix + '/' + os.path.basename(self.db_path)
self.max_workers = max_workers
def fits2png(fits_dir, png_dir):
"""
Convert fits to png files based on the D1 method
"""
cmd_tpl = '%s -cmap Cool'\
' -zoom to fit -scale log -scale mode minmax -export %s -exit'
from sh import Command
ds9 = Command(ds9_path)
#fits = '/Users/chen/gitrepos/ml/rgz_rcnn/data/EMU_GAMA23/split_fits/30arcmin/gama_linmos_corrected_clipped0-0.fits'
#png = '/Users/chen/gitrepos/ml/rgz_rcnn/data/EMU_GAMA23/split_png/30arcmin/gama_linmos_corrected_clipped0-0.png'
for fits in os.listdir(fits_dir):
if (fits.endswith('.fits')):
png = fits.replace('.fits', '.png')
cmd = cmd_tpl % (osp.join(fits_dir, fits), osp.join(png_dir, png))
ds9(*(cmd.split()))
