def sketchupl(firmware):
sketchpath = os.path.join(sdir, "sketches", firmware)
if os.path.exists(sketchpath):
errorkey = int(time.time())
cuser = g.user.nickname
if host == "win":
output = subprocess.call(
[rfpath, "-h", host, "-s", sketchpath, "-r", "-e", errorkey, "-c", cuser], shell=True
)
print "********************************************************************"
print output
print "********************************************************************"
else:
output = subprocess.call(
["%s -h %s -s %s -r -e %s -c %s" % (rfpath, host, sketchpath, errorkey, cuser)], shell=True
)
print "********************************************************************"
print output
print "********************************************************************"
if output == 0:
print "Subprocess call complete with " + str(output) + " errors"
return messagereturn(cuser, errorkey, None, "fullcycle")
else:
print "Error uploading firmware to devices"
print "ERROR 4: Subprocess call complete with " + str(output) + " errors"
return messagereturn(cuser, errorkey, None, "fullcycle")
else:
# Firmware specified does not exist, explicitly handled through messagereturn
print sketchpath
return messagereturn(None, None, 34, None)
def runPkcombu(a_path, b_path, oam_path):
FNULL = open(os.devnull, 'w')
cmds = ["pkcombu",
"-A", a_path,
"-B", b_path,
"-oam", oam_path]
subprocess32.call(cmds, stdout=FNULL, stderr=subprocess32.STDOUT)
def trimc_trim (file_fw, file_rv, outdir, trimc_dir=None): if not trimc_dir: trimc_dir = '/home/anna/bioinformatics/bioprograms/Trimmomatic-0.32/' trim_out = outdir + 'trim_out/' if not os.path.exists(trim_out): os.makedirs(trim_out) trimlog = trim_out +'trimlog' paired_out_fw = trim_out + 'paired_out_fw' + '.fastq' unpaired_out_fw = trim_out + 'unpaired_out_fw' + '.fastq' paired_out_rv = trim_out + 'paired_out_rv' + '.fastq' unpaired_out_rv = trim_out + 'unpaired_out_rv' + '.fastq' adapters_file = trimc_dir + 'adapters/'+ "illumina.fasta" trimmomatic = ['java', '-jar', trimc_dir + 'trimmomatic-0.32.jar'] # trim_options = ['PE', '-threads', str(THREADS), '-phred33', '-trimlog', trimlog, file_fw, file_rv, # paired_out_fw, unpaired_out_fw, paired_out_rv, unpaired_out_rv, # 'ILLUMINACLIP:'+ adapters_file + ':2:15:15:8:true', 'LEADING:3', 'TRAILING:3', 'SLIDINGWINDOW:4:5', # 'MAXINFO:200:0.2', 'MINLEN:5' ] trim_options = ['PE', '-threads', str(THREADS), '-phred33', '-trimlog', trimlog, file_fw, file_rv, paired_out_fw, unpaired_out_fw, paired_out_rv, unpaired_out_rv, 'TRAILING:3', 'SLIDINGWINDOW:4:25', 'MINLEN:100' ] trim = trimmomatic + trim_options print ' '.join(trim) call(trim) return trim_out
def timeout_func():
self.logger.debug("%s: time limit exceeded; killing docker container" % (user,))
ran_to_completion[0] = False
try:
subprocess.call(['docker', 'rm', '-f', docker_name])
except:
pass
def spades_assemble(outdir, test=None, reads = None, file_fw=None, file_rv=None, spades_dir=None, bbduk_out = None, trimc_out=None, RAM=6):
if not spades_dir: spades_dir = '/home/anna/bioinformatics/bioprograms/SPAdes-3.1.1-Linux/bin/'
spades_out = outdir + 'spades_out/'
spades = spades_dir + './spades.py'
if test: spades_assemble= [spades, '--test'] # Test SPAdes
else:
if trimc_out:
files = {'PEfw' : 'paired_out_fw.fastq', 'PErv' : 'paired_out_rv.fastq',
'UPfw': 'unpaired_out_fw.fastq', 'UPrv': 'unpaired_out_rv.fastq'}
for key in files:
files[key] = trimc_out + files[key]
spades_options = ['-1', files['PEfw'], '-2', files['PErv'], '-s', files['UPfw'], '-s', files['UPrv'],
'-o', spades_out, '-m '+ str(RAM), '--careful']
spades_assemble= [spades] + spades_options
elif file_fw and file_rv:
spades_options = ['-o', spades_out, '-m '+ str(RAM), '--careful', '--only-assembler']
spades_assemble = [spades, '-1', file_fw, '-2', file_rv] + spades_options
elif reads:
spades_options = ['-o', spades_out, '-m '+ str(RAM), '--only-assembler']
spades_assemble = [spades, '-s', reads] + spades_options
else: print "Error: spades_assemble haven't get needed values"
if not os.path.exists(spades_out): os.makedirs(spades_out)
call(spades_assemble)
return spades_out
def run_apoc(self):
subprocess32.call(["mkdir", "-p", self._mydir()])
paths = [_.output().path for _ in self.requires()]
cmd = [APOC_BIN] + paths
print " ".join(cmd)
stdout = subprocess32.check_output(cmd)
return stdout
def test_case (testname, timeout, prog_name, checker):
import subprocess32
try:
q=subprocess32.call(
("./"+prog_name,),
stdin=open(testname+".in", "r"),
stdout=open(testname+".test", "w"),
timeout=timeout
)
except:
return (testname, 2)
if q!=0:
return (testname, "program returned code {}.".format(q))
if checker=="diff":
return (
testname,
subprocess32.call(
("diff", "-qb", testname+".out", testname+".test"),
stdout=open("/dev/null", "w")
)
)
else:
return (
testname,
subprocess32.call(
("./"+checker, testname+".out", testname+".test", testname+".in"),
stdout=open("/dev/null", "w")
)
)
def run(self, logdir):
"""
Execute the command as a subprocess and log its output in logdir.
:param logdir: Path to a log directory.
"""
env = os.environ.copy()
if "PATH" not in env:
env["PATH"] = "/usr/bin:/bin"
locale = settings.get_value("sysinfo.collect", "locale", str, None)
if locale:
env["LC_ALL"] = locale
logf_path = os.path.join(logdir, self.logf)
stdin = open(os.devnull, "r")
stdout = open(logf_path, "w")
try:
subprocess.call(self.cmd, stdin=stdin, stdout=stdout,
stderr=subprocess.STDOUT, shell=True, env=env)
finally:
for f in (stdin, stdout):
f.close()
if self._compress_log and os.path.exists(logf_path):
process.run('gzip -9 "%s"' % logf_path,
ignore_status=True,
verbose=False)
def perform_git_install(use_pyqt5):
"""
Performs a git-based install.
"""
if not IS_ROOT:
root_warning()
if use_pyqt5:
if PYTHON3_OK:
run_cmd = ("make", "PYQT=5", "PYTHON=python3")
else:
run_cmd = ("make", "PYQT=5")
else:
if PYTHON3_OK:
run_cmd = ("make", "PYTHON=python3")
else:
run_cmd = ("make")
try:
code = subprocess.call(run_cmd)
except OSError as errmsg:
if errmsg.errno == errno.ENOENT:
print("\nError: 'make' not found. It's either not installed or "
"not in the PATH environment variable like expected.")
return
if code == 0:
print("\nCustomizer has been built from git.")
else:
print("\nCustomizer could not build properly. If this is caused"
" by edits you have made to the code you can try the repair"
" option from the Maintenance submenu")
if not IS_ROOT:
code = subprocess.call(("sudo", "make", "install"))
if code == 0:
print("\nCustomizer has been installed from git.")
else:
print("The installation has failed.")
def run_experiment(self, timeout, filename):
status, value, db = self.get_status(filename)
if status == Experiment.DONE: return
if status == Experiment.TIMEOUT and value >= int(timeout): return
# remove output and timeout files
if os.path.isfile(filename): os.unlink(filename)
timeout_filename = "{}.timeout".format(filename)
if os.path.isfile(timeout_filename): os.unlink(timeout_filename)
print("Performing {}... ".format(self.name), end='')
sys.stdout.flush()
try:
with open(filename, 'w+') as out:
call(self.call, stdout=out, stderr=out, timeout=timeout)
except KeyboardInterrupt:
os.unlink(filename)
print("interrupted!")
sys.exit()
except OSError:
os.unlink(filename)
print("OS failure! (missing executable?)")
sys.exit()
except TimeoutExpired:
with open(timeout_filename, 'w') as to: to.write(str(timeout))
print("timeout!")
else:
status, value, db = self.get_status(filename)
if status == Experiment.DONE: print("done; {}!".format(value))
else: print("not done!")
time.sleep(2)
def timeout_func():
self.logger.debug("%s: time limit exceeded; killing docker container" % (user,))
ran_to_completion[0] = False
try:
subprocess.call('docker rm -f %s' % (docker_name,), shell=True)
except:
pass
def superpose(self):
"""superpose and copy the referenced protein and ligand
"""
result = json.loads(QueryVinaResultOnBioLipFixedPocket(
self.lig_pdb).output().open('r').read())
dset = read2Df(result, self.lig_pdb)
dset = similarPocketsLigands(clean(dset), minimum_Tc=0.2)
work_dir = os.path.join(self.workdir(), 'superpose')
try:
os.makedirs(work_dir)
except Exception:
pass
mob_pdb = self.append_ligand()
for template_pdb in dset.index:
ref_pdb = Path(template_pdb).prtPdb
lig_code = Path(template_pdb).lig_code
ref_lig = Path(lig_code).ligPdb() + '.pdb'
sup_pdb = os.path.join(work_dir, lig_code + '.sup.pdb')
cmd = shlex.split("perl %s all %s %s %s" %
(self.superpose_perl, ref_pdb, mob_pdb, sup_pdb))
subprocess32.call(cmd)
shutil.copy(ref_pdb, work_dir)
shutil.copy(ref_lig, work_dir)
def do_cmd(cmd, tmp_file, cargs):
out = []
if cargs.verbose:
print " CMD: %s" % cmd
fh = None
if tmp_file:
fh = open(tmp_file, 'w')
else:
fh = open(os.devnull, 'w')
if cargs.verbose and not tmp_file:
subprocess.call(cmd, stdin=None, shell=True)
else:
subprocess.call(cmd, stdin=None,
stdout=fh, stderr=subprocess.STDOUT, shell=True)
fh.close()
if tmp_file:
with open(tmp_file, 'r') as f:
for line in f:
out.append(line.rstrip('\n'))
return out
def execute_link(link_cmd_args, record_byproducts):
"""
<Purpose>
Executes the passed command plus arguments in a subprocess and returns
the return value of the executed command. If the specified standard output
and standard error of the command are recorded and also returned to the
caller.
<Arguments>
link_cmd_args:
A list where the first element is a command and the remaining
elements are arguments passed to that command.
record_byproducts:
A bool that specifies whether to redirect standard output and
and standard error to a temporary file which is returned to the
caller (True) or not (False).
<Exceptions>
TBA (see https://github.com/in-toto/in-toto/issues/6)
<Side Effects>
Executes passed command in a subprocess and redirects stdout and stderr
if specified.
<Returns>
- A dictionary containg standard output and standard error of the
executed command, called by-products.
Note: If record_byproducts is False, the dict values are empty strings.
- The return value of the executed command.
"""
# XXX: The first approach only redirects the stdout/stderr to a tempfile
# but we actually want to duplicate it, ideas
# - Using a pipe won't work because processes like vi will complain
# - Wrapping stdout/sterr in Python does not work because the suprocess
# will only take the fd and then uses it natively
# - Reading from /dev/stdout|stderr, /dev/tty is *NIX specific
# Until we come up with a proper solution we use a flag and let the user
# decide if s/he wants to see or store stdout/stderr
# btw: we ignore them in the layout anyway
if record_byproducts:
# XXX: Use SpooledTemporaryFile if we expect very large outputs
stdout_file = tempfile.TemporaryFile()
stderr_file = tempfile.TemporaryFile()
return_value = subprocess.call(link_cmd_args,
stdout=stdout_file, stderr=stderr_file)
stdout_file.seek(0)
stderr_file.seek(0)
stdout_str = stdout_file.read()
stderr_str = stderr_file.read()
else:
return_value = subprocess.call(link_cmd_args)
stdout_str = stderr_str = ""
return {"stdout": stdout_str, "stderr": stderr_str}, return_value
def run(self):
nullFile = open('/dev/null','w')
#logging.info("Starting FFMPEG")
logging.info("Start up FFmpeg %s"%Config.FFMPEG_COMMAND)
subprocess32.call(Config.FFMPEG_COMMAND,stdout=nullFile,stderr=nullFile)
logging.info("FFMPEG ends!")
def cli(force):
"""
Update AerisCloud
"""
if not force and config.get('github', 'enabled', default=False) == 'true':
client = Github().gh
repo = client.repository('aeriscloud', 'aeriscloud')
latest_release = repo.iter_releases().next()
latest_version = latest_release.tag_name[1:]
if semver.compare(version, latest_version) != -1:
click.secho('AerisCloud is already up to date!', fg='green')
sys.exit(0)
click.echo('A new version of AerisCloud is available: %s (%s)' % (
click.style(latest_version, fg='green', bold=True),
click.style(latest_release.name, bold=True)
))
# retrieve install script in a tmpfile
tmp = tempfile.NamedTemporaryFile()
r = requests.get('https://raw.githubusercontent.com/' +
'AerisCloud/AerisCloud/develop/scripts/install.sh')
if r.status_code != 200:
fatal('error: update server returned %d (%s)' % (
r.status_code, r.reason))
tmp.write(r.content)
tmp.flush()
os.environ['INSTALL_DIR'] = aeriscloud_path
call(['bash', tmp.name])
tmp.close()
def settings(data):
call(['touch',data['home_dir']+'/.ssh/config'])
config_path = data['home_dir']+'/.ssh/config'
if os.path.isfile(config_path):
config_file = open(config_path,'wt')
config_file.write('Host *\n\tStrictHostKeyChecking no\n')
config_file.close()
def abacas_scaffold (contigs, reference, outdir, abacas_dir=None): if not abacas_dir: abacas_dir = '/home/anna/bioinformatics/bioprograms/Abacas/abacas.pl' abacas_out = outdir + 'abacas_out/' abacas = ['perl', abacas_dir + 'abacas.pl'] abacas_options = ['-r', reference, '-q', contigs, '-b', '-c', '-m', '-p', nucmer] call(abacas + abacas_options) return abacas_out
def test_app_is_uninstalled(appiumSetup, moduleSetup, testSetup):
print 'in test app uninstalled'
#setup.setup_appium_driver(app=config.app)
# time.sleep(5)
subprocess32.call(['adb', 'uninstall', config.app_package])
app_package = subprocess32.check_output(['adb', '-s', config.udid, 'shell', 'pm list packages', '|', 'grep', config.app_package])
assert not app_package, 'app is not uninstalled, %s not found' % config.app_package
def run_command(self, thefile):
new_command = []
for item in self.command:
if item == "%f":
item = thefile
new_command.append(item)
subprocess.call(new_command)
def up_node3():
subproc.check_call(['vagrant', 'destroy', '-f', 'node3'])
subproc.check_call(['vagrant', 'up', 'node3'])
yield "node3 is ready"
print("Destroying node3...")
subproc.call(['vagrant', 'destroy', '-f', 'node3'])
print("Node3 is destroyed.")
def launch_folder(full_path, cleanup = False):
if not cleanup:
subprocess.call(['open',
'-a',
'Finder',
full_path], close_fds=True)
else:
open_folder_in_finder(full_path, cleanup)
def up_node1():
subproc.check_call(['vagrant', 'destroy', '-f', 'node1'])
subproc.check_call(['vagrant', 'up', 'node1', '--no-provision'])
yield "node1 is ready"
print("Destroying node1...")
subproc.call(['vagrant', 'destroy', '-f', 'node1'])
print("Node1 is destroyed.")
def use_fuzznuc (reads, pattern, outdir, max_mismatch = 0, indels = False, name = ''): fuzznuc_file = outdir + 'fuzznuc_report' + name fuzznuc = ['fuzznuc', '-sequence', reads, '-pattern', pattern, '-outfile', fuzznuc_file] fuzznuc_options = ['-pmismatch', str(max_mismatch), '-complement', '-snucleotide1', '-squick1', '-rformat2', 'excel', '-stdout'] fuzznuc = fuzznuc + fuzznuc_options call(fuzznuc) return fuzznuc_file
def prokka_annotate (sequence, outdir, prokka_dir=None): if not prokka_dir: prokka_dir = '/home/anna/bioinformatics/bioprograms/prokka-1.10/bin/' prokka_out = outdir + 'prokka_out/' prokka = prokka_dir + './prokka' prokka_options = ['--centre', 'XXX', '--kingdom', 'Bacteria', '--gram', 'neg', '--addgenes', '--outdir', prokka_out, '--force', sequence, '--genus', 'Pandoraea'] prokka_annotate = [prokka] + prokka_options call(prokka_annotate) return prokka_out
def gen_correct (testname, correct_prog):
import subprocess32
subprocess32.call(
("./"+correct_prog,),
stdin=open(testname+".in", "r"),
stdout=open(testname+".out", "w"),
)
return testname
def use_quast (contigs, outdir, reference = None, quast_dir=None): if not quast_dir: quast_dir = '/home/anna/bioinformatics/bioprograms/quast-2.3/' quast_out = outdir + 'quast_out/' quast = quast_dir + './quast.py' quast_options = ['-o', quast_out] if reference: quast_options = quast_options + ['-R', reference] use_quast = [quast] + quast_options + [contigs] call(use_quast) return quast_out
def compile (source_file, target_file, lazy=True):
compile_return = 0
if os.path.splitext(source_file)[1] == '.c':
compile_return = subprocess.call(["gcc", "-std=c99", source_file, "-o", target_file])
else:
compile_return = subprocess.call(["g++", "-std=c++11", source_file, "-o", target_file])
return compile_return
def upload_image(self, filename):
""" upload the specified image to imgurl"""
self.log.debug("calling imgurbash.sh %s", filename)
# TODO use pure python here
subprocess.call([
'bash',
'./imgurbash.sh',
filename
], timeout=5)
def run_command(self, thefile):
if self.run_once:
if os.path.exists(thefile) and os.path.getmtime(thefile) < self.last_run:
return
new_command = []
for item in self.command:
new_command.append(item.replace('%f', thefile))
subprocess.call(new_command, shell=(len(new_command) == 1))
self.last_run = time.time()
quit()
verbose = False
n1 = int(sys.argv[1])
if len(sys.argv) > 2:
n2 = int(sys.argv[2])
else:
n2 = n1
inname = "input/comp/%s.in"
assumname = "input/comp/%s.assum"
archivename = "archives/comp/%s.zip"
compname = "input/comp/%s.comp"
if not os.path.exists("input"): call(["mkdir", "input"])
if not os.path.exists("input/comp"): call(["mkdir", "input/comp"])
if not os.path.exists("archives"): call(["mkdir", "archives"])
if not os.path.exists("archives/comp"): call(["mkdir", "archives/comp"])
def runstr(n, c, k):
if k == -1 and c == -1:
return "%d" % n
elif k == -1:
return "%d.%d" % (n, c)
elif k == "*" and c == "*":
return "%d.*.*" % n
elif k == "*":
return "%d.%d.*" % (n, c)
else:
return "%d.%d.%d" % (n, c, k)
def test_exit_unknown_option(arg):
if platform.system() == 'Windows':
tool_name = TOOL + '.exe'
else:
tool_name = TOOL
assert subprocess.call([tool_name, arg]) == 1
def kill_job(self):
"""
Attempt to kill a job using qdel.
"""
return call(self.qdel_cmd)
import subprocess32 as subprocess
import shlex
import sys
fonts = [
"/Library/Fonts/Tahoma.ttf",
"/Users/jason/Library/Fonts/FreeSans.ttf",
"/Users/jason/Library/Fonts/FreeSansBold.ttf",
"/Users/jason/Library/Fonts/FreeMono.ttf",
"/Users/jason/Library/Fonts/FreeMonoBold.ttf",
"/Users/jason/Library/Fonts/VCR_OSD_MONO_1.001.ttf",
"/Users/jason/Library/Fonts/Retron2000.ttf",
"/Users/jason/Library/Fonts/pt-mono.regular.ttf",
"/Users/jason/Library/Fonts/pt-mono.bold.ttf",
]
sizes = [6, 8, 10, 12, 14, 13, 16, 18, 22, 24, 26, 32, 36, 40, 42, 46, 60]
source = open("generate.py").read().replace("\n", "\n// ")
print("// Generated with generate.py:")
print("// " + source)
print()
sys.stdout.flush()
for font in fonts:
for size in sizes:
cmd = "./fontconvert %s %d 64 32 176" % (font, size)
cmd = shlex.split(cmd)
subprocess.call(cmd)
def test_exit_unknown_option(arg):
assert subprocess.call([TOOL, arg]) == 1
else:
psl_option = '-j managed'
job_type_option = ''
if job_type:
job_type_option = '-i {0}'.format(job_type)
# wrapper_params = '-a {0} -s {1} -r {2} -q {3} {4} {5} {6}'.format(WORK_DIR, panda_site, panda_queue, panda_queue,
wrapper_params = '-s {0} -r {1} -q {2} {3} {4} {5}'.format(
panda_site, panda_queue, panda_queue, resource_type_option, psl_option,
job_type_option)
if submit_mode == 'PUSH':
# job configuration files need to be copied, because k8s configmap mounts as read-only file system
# and therefore the pilot cannot execute in the same directory
copy_files_in_dir(CONFIG_DIR, WORK_DIR)
pilot_url = "https://raw.githubusercontent.com/yesw2000/Harvester/master/pilot2-gcs.tgz"
command = "/tmp/runpilot2-wrapper.sh {0} -i PR --piloturl {1} -w generic --pilot-user generic --url=https://ai-idds-01.cern.ch -d --harvester-submit-mode {2} --allow-same-user=False -t | tee /tmp/wrapper-wid.log". \
format(wrapper_params, pilot_url, submit_mode)
try:
subprocess.call(command, shell=True)
except:
logging.error(traceback.format_exc())
logging.debug('[main] pilot wrapper done...')
# upload logs to e.g. panda cache or similar
upload_logs(logs_frontend_w, '/tmp/wrapper-wid.log', destination_name,
proxy_path)
logging.debug('[main] FINISHED')
tmp_filename = "tmp_{}.nc".format(EXP)
tmp_file = os.path.join(tmp_dir, tmp_filename)
try:
os.remove(tmp_file)
except OSError:
pass
# Check if request variables are present
nc = CDF(infile, "r")
for m_var in pism_copy_vars:
if m_var not in nc.variables:
print(("Requested variable '{}' missing".format(m_var)))
nc.close()
print(("Copy {} to {}".format(infile, tmp_file)))
cmd = ["ncks", "-O", "-d", "time,1,", "-v", "{}".format(",".join(pism_copy_vars)), infile, tmp_file]
sub.call(cmd)
# Make the file ISMIP6 conforming
make_spatial_vars_ismip6_conforming(tmp_file, ismip6_vars_dict)
# Should be temporary until new runs
ncatted_cmd = [
"ncatted",
"-a",
"""bounds,lat,o,c,lat_bnds""",
"-a",
"""bounds,lon,o,c,lon_bnds""",
"-a",
"""coordinates,lat_bnds,d,,""",
"-a",
"""coordinates,lon_bnds,d,,""",
tmp_file,
def host_cmd(args, cmd_args):
if not cmd_args:
raise TestFormatError('Must specify command to run on the host.')
elif args.deploy_chrome and not args.path_to_outdir:
raise TestFormatError(
'--path-to-outdir must be specified if --deploy-chrome is passed.')
cros_run_test_cmd = [
CROS_RUN_TEST_PATH,
'--board',
args.board,
'--cache-dir',
os.path.join(CHROMIUM_SRC_PATH, args.cros_cache),
]
if args.use_vm:
cros_run_test_cmd += [
'--start',
# Don't persist any filesystem changes after the VM shutsdown.
'--copy-on-write',
]
else:
cros_run_test_cmd += [
'--device', args.device if args.device else LAB_DUT_HOSTNAME
]
if args.verbose:
cros_run_test_cmd.append('--debug')
if args.flash:
cros_run_test_cmd.append('--flash')
if args.public_image:
cros_run_test_cmd += ['--public-image']
if args.logs_dir:
for log in SYSTEM_LOG_LOCATIONS:
cros_run_test_cmd += ['--results-src', log]
cros_run_test_cmd += [
'--results-dest-dir',
os.path.join(args.logs_dir, 'system_logs')
]
test_env = setup_env()
if args.deploy_chrome or args.deploy_lacros:
# Mounting ash-chrome gives it enough disk space to not need stripping.
cros_run_test_cmd.extend([
'--deploy-lacros', '--lacros-launcher-script',
LACROS_LAUNCHER_SCRIPT_PATH
] if args.deploy_lacros else ['--deploy', '--mount', '--nostrip'])
cros_run_test_cmd += [
'--build-dir',
os.path.join(CHROMIUM_SRC_PATH, args.path_to_outdir)
]
cros_run_test_cmd += [
'--host-cmd',
'--',
] + cmd_args
logging.info('Running the following command:')
logging.info(' '.join(cros_run_test_cmd))
return subprocess.call(cros_run_test_cmd,
stdout=sys.stdout,
stderr=sys.stderr,
env=test_env)
save_times = [-20000, -15000, -12500, -11700]
if options.no_refreeze:
pism_config = 'init_config_norefreeze'
else:
pism_config = 'init_config'
pism_config_nc = '.'.join([pism_config, 'nc'])
pism_config_cdl = os.path.join('../config', '.'.join([pism_config, 'cdl']))
# Anaconda libssl problem on chinook
if system in ('chinook'):
ncgen = '/usr/bin/ncgen'
else:
ncgen = 'ncgen'
cmd = [ncgen, '-o',
pism_config_nc, pism_config_cdl]
sub.call(cmd)
if not os.path.isdir(odir):
os.mkdir(odir)
state_dir = 'state'
scalar_dir = 'scalar'
spatial_dir = 'spatial'
snap_dir = 'snap'
for tsdir in (scalar_dir, spatial_dir, snap_dir, state_dir):
if not os.path.isdir(os.path.join(odir, tsdir)):
os.mkdir(os.path.join(odir, tsdir))
odir_tmp = '_'.join([odir, 'tmp'])
if not os.path.isdir(odir_tmp):
os.mkdir(odir_tmp)
# ########################################################
# set up model initialization
import subprocess
import subprocess32
thread_timeout = 30
includeos_src = os.environ.get(
'INCLUDEOS_SRC',
os.path.realpath(os.path.join(
os.getcwd(), os.path.dirname(__file__))).split('/test')[0])
sys.path.insert(0, includeos_src)
from vmrunner import vmrunner
# Get an auto-created VM from the vmrunner
vm = vmrunner.vms[0]
def cleanup():
# Call the cleanup script - let python do the printing to get it synced
print subprocess.check_output(["./fat32_disk.sh", "clean"])
# Setup disk
subprocess32.call(["./fat32_disk.sh"], shell=True, timeout=thread_timeout)
# Clean up on exit
vm.on_exit(cleanup)
# Boot the VM
vm.cmake().boot(thread_timeout).clean()
def place_file(self, id, source_path):
target_dir = '{}/{}'.format(IMAGERY_PATH, id)
if not os.path.exists(target_dir):
os.mkdir(target_dir)
output_file = '{}/index.tif'.format(target_dir)
# rewrite with gdal_translate
gdal_translate = [
'gdal_translate',
source_path,
output_file,
'-co',
'TILED=yes',
'-co',
'COMPRESS=DEFLATE',
'-co',
'PREDICTOR=2',
'-co',
'SPARSE_OK=yes',
'-co',
'BLOCKXSIZE=256',
'-co',
'BLOCKYSIZE=256',
'-co',
'INTERLEAVE=band',
'-co',
'NUM_THREADS=ALL_CPUS',
]
started_at = datetime.utcnow()
self.update_state(state='RUNNING',
meta={
'name': 'preprocess',
'started_at': started_at.isoformat(),
'status': 'Rewriting imagery'
})
try:
returncode = subprocess.call(gdal_translate, timeout=60 * 5)
except subprocess.TimeoutExpired as e:
raise Exception(
json.dumps({
'name': 'proprocess',
'started_at': started_at.isoformat(),
'command': ' '.join(gdal_translate),
'return_code': returncode,
'status': 'Timed out'
}))
if returncode != 0:
raise Exception(
json.dumps({
'name': 'preprocess',
'started_at': started_at.isoformat(),
'command': ' '.join(gdal_translate),
'return_code': returncode,
'status': 'Failed'
}))
# delete original
os.unlink(source_path)
return {
'name': 'preprocess',
'completed_at': datetime.utcnow().isoformat(),
'started_at': started_at,
'status': 'Image pre-processing completed'
}
def node_start(node_image, runtime, log_dir, ip, ip6, as_num, detach,
kubernetes_version, rkt, libnetwork_image):
"""
Create the calico-node container and establish Calico networking on this
host.
:param ip: The IPv4 address of the host.
:param node_image: The calico-node image to use.
:param ip6: The IPv6 address of the host (or None if not configured)
:param as_num: The BGP AS Number to use for this node. If not specified
the global default value will be used.
:param detach: True to run in Docker's "detached" mode, False to run
attached.
:param kubernetes_version: The version of the calico-kubernetes plugin to
install, or None if the plugin should not be installed.
:param rkt: True to install the rkt plugin, False otherwise.
:param libnetwork_image: The name of the Calico libnetwork driver image to
use. None, if not using libnetwork.
:return: None.
"""
# Normally, Felix will load the modules it needs, but when running inside a
# container it might not be able to so ensure the required modules are
# loaded each time the node starts.
# This is just a best error attempt, as the modules might be builtins.
# We'll warn during the check_system() if the modules are unavailable.
try:
call(["modprobe", "-a"] + REQUIRED_MODULES)
except OSError:
pass
# Print warnings for any known system issues before continuing
using_docker = True if runtime == 'docker' else False
(_, _, etcd_ok) = \
check_system(quit_if_error=False, libnetwork=libnetwork_image,
check_docker=using_docker)
if not etcd_ok:
sys.exit(1)
# We will always want to setup IP forwarding
_setup_ip_forwarding()
# Ensure log directory exists
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# Get IP address of host, if none was specified
if not ip:
ips = get_host_ips(exclude=["^docker.*", "^cbr.*"])
try:
ip = ips.pop()
except IndexError:
print "Couldn't autodetect a management IP address. Please provide" \
" an IP by rerunning the command with the --ip=<IP_ADDRESS> flag."
sys.exit(1)
else:
print "No IP provided. Using detected IP: %s" % ip
# Verify that IPs are not already in use by another host.
error_if_bgp_ip_conflict(ip, ip6)
# Verify that the chosen IP exists on the current host
warn_if_unknown_ip(ip, ip6)
# Warn if this hostname conflicts with an existing host
warn_if_hostname_conflict(ip)
# Install Kubernetes plugin
if kubernetes_version:
# Build a URL based on the provided Kubernetes_version.
url = KUBERNETES_BINARY_URL % kubernetes_version
try:
# Attempt to install to the default Kubernetes directory
install_plugin(KUBERNETES_PLUGIN_DIR, url)
except OSError:
# Use the backup directory
install_plugin(KUBERNETES_PLUGIN_DIR_BACKUP, url)
# Install rkt plugin
if rkt:
try:
# Attempt to install to the default rkt directory
install_plugin(RKT_PLUGIN_DIR, RKT_BINARY_URL)
except OSError:
# Use the backup directory
install_plugin(RKT_PLUGIN_DIR_BACKUP, RKT_BINARY_URL)
# Set up etcd
ipv4_pools = client.get_ip_pools(4)
ipv6_pools = client.get_ip_pools(6)
# Create default pools if required
if not ipv4_pools:
client.add_ip_pool(4, DEFAULT_IPV4_POOL)
if not ipv6_pools:
client.add_ip_pool(6, DEFAULT_IPV6_POOL)
client.ensure_global_config()
client.create_host(hostname, ip, ip6, as_num)
# Always try to convert the address(hostname) to an IP. This is a noop if
# the address is already an IP address. Note that the format of the authority
# string has already been validated.
etcd_authority = os.getenv(ETCD_AUTHORITY_ENV, ETCD_AUTHORITY_DEFAULT)
etcd_authority_address, etcd_authority_port = etcd_authority.split(':')
etcd_authority = '%s:%s' % (socket.gethostbyname(etcd_authority_address),
etcd_authority_port)
# Get etcd SSL environment variables if they exist
etcd_scheme = os.getenv(ETCD_SCHEME_ENV, ETCD_SCHEME_DEFAULT)
etcd_key_file = os.getenv(ETCD_KEY_FILE_ENV, None)
etcd_cert_file = os.getenv(ETCD_CERT_FILE_ENV, None)
etcd_ca_cert_file = os.getenv(ETCD_CA_CERT_FILE_ENV, None)
etcd_volumes = []
etcd_binds = {}
etcd_envs = [
"ETCD_AUTHORITY=%s" % etcd_authority,
"ETCD_SCHEME=%s" % etcd_scheme
]
felix_envs = [
"FELIX_ETCDADDR=%s" % etcd_authority,
"FELIX_ETCDSCHEME=%s" % etcd_scheme
]
if etcd_ca_cert_file and etcd_key_file and etcd_cert_file:
etcd_volumes.append(ETCD_CA_CERT_NODE_FILE)
etcd_binds[etcd_ca_cert_file] = {
"bind": ETCD_CA_CERT_NODE_FILE,
"ro": True
}
etcd_envs.append("ETCD_CA_CERT_FILE=%s" % ETCD_CA_CERT_NODE_FILE)
felix_envs.append("FELIX_ETCDCAFILE=%s" % ETCD_CA_CERT_NODE_FILE)
etcd_volumes.append(ETCD_KEY_NODE_FILE)
etcd_binds[etcd_key_file] = {"bind": ETCD_KEY_NODE_FILE, "ro": True}
etcd_envs.append("ETCD_KEY_FILE=%s" % ETCD_KEY_NODE_FILE)
felix_envs.append("FELIX_ETCDKEYFILE=%s" % ETCD_KEY_NODE_FILE)
etcd_volumes.append(ETCD_CERT_NODE_FILE)
etcd_binds[etcd_cert_file] = {"bind": ETCD_CERT_NODE_FILE, "ro": True}
etcd_envs.append("ETCD_CERT_FILE=%s" % ETCD_CERT_NODE_FILE)
felix_envs.append("FELIX_ETCDCERTFILE=%s" % ETCD_CERT_NODE_FILE)
if runtime == 'docker':
_start_node_container(ip, ip6, log_dir, node_image, detach, etcd_envs,
felix_envs, etcd_volumes, etcd_binds)
if libnetwork_image:
_start_libnetwork_container(libnetwork_image, etcd_envs,
etcd_volumes, etcd_binds)
'''
PreCluster submission.
Joblist is created and the first job is send out to test whether the pipeline
runs successfully.
'''
# 1) Create joblist
process = Popen(['jt', 'joblist'], stdin=PIPE, stdout=PIPE, stderr=PIPE)
(stdoutdata, stderrdata) = process.communicate()
print stdoutdata
print stderrdata
if process.returncode > 0 or re.search('Failed', stderrdata):
raise Exception('\n--> Building joblist failed!')
if not os.path.exists('lsf'):
os.mkdir('lsf')
# 2) Run 'PreCluster'
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M-%S')
lsf = os.path.abspath(os.path.join('lsf', '%.5d_%s.precluster' % (1, st)))
if not os.path.exists(lsf):
print('jt - PreCluster submission:')
call([
'bsub', '-W', '8:00', '-o', lsf, '-N', '-R',
'rusage[mem=4000,scratch=4000]', 'jt', 'run', '--job', '1'
])
time_dir = join(output_dir, "time_forcing")
if not os.path.isdir(time_dir):
os.makedirs(time_dir)
# use the actual path of the uq directory
uq_dir = join(output_dir, "uq")
if not os.path.isdir(uq_dir):
os.makedirs(uq_dir)
# generate the config file *after* creating the output directory
pism_config = "pism"
pism_config_nc = join(output_dir, pism_config + ".nc")
cmd = "ncgen -o {output} {input_dir}/config/{config}.cdl".format(
output=pism_config_nc, input_dir=input_dir, config=pism_config)
sub.call(shlex.split(cmd))
# these Bash commands are added to the beginning of the run scrips
run_header = """# stop if a variable is not defined
set -u
# stop on errors
set -e
# path to the config file
config="{config}"
# path to the input directory (input data sets are contained in this directory)
input_dir="{input_dir}"
# output directory
output_dir="{output_dir}"
# temporary directory for spatial files
spatial_tmp_dir="{spatial_tmp_dir}"
import os
import subprocess32
DAT_DIR = "/work/jaydy/dat/fda_pdb_mb"
WORK_DIR = "/work/jaydy/working/clean_pdb"
def readDat(ifn):
with open(ifn, 'r') as f:
return cPickle.load(f)
dat_ifn = "../dat/drugid_checked_pdb.dat"
dat = readDat(dat_ifn)
for drug_id, ligs in dat.iteritems():
pdb_ids = [name.split('.')[0] for name in ligs]
for pdb_id in pdb_ids:
my_work_dir = ""
try:
work_dir = os.path.join(WORK_DIR, pdb_id[1:3])
my_work_dir = work_dir
os.makedirs(work_dir)
except:
pass
pdb_fn = os.path.join(DAT_DIR, pdb_id[1:3], pdb_id + '.pdb')
if os.path.exists(pdb_fn):
os.chdir(my_work_dir)
ctrip_cmd = ['ctrip', '-prm', '2', pdb_fn]
subprocess32.call(ctrip_cmd)
def get_file_dic(path, options):
'''
This function opens supplied VCF files and saves metadata and variants in the dictionary spec_data.
'''
global sample_list
spec_data = [
] # create empty list that will be populated which dictionaries of VCF file information
file_list = [f for f in os.listdir(path) if options.file_pattern in f
] # get file list from input path and fname pattern
for fname in file_list:
if not options.inverse_deletions: # retroseq results
# DO VCF to BED conversion
if fname.endswith(".vcf"):
subprocess32.call(["vcf2bed", "-d"],
stdin=open(os.path.join(path, fname)),
stdout=open(
os.path.join(
options.out_path,
fname.replace(".vcf", ".bed")), "w"))
fname = fname.replace(".vcf", ".bed")
else:
copyfile(os.path.join(path, fname),
os.path.join(options.out_path, fname))
fname_splitted = fname.split(".")
species_dict = {
"sample": fname_splitted[0],
"fname": fname,
"ftype": fname_splitted[-1],
"meitype": fname_splitted[3],
"f_bedtools": BedTool(os.path.join(options.out_path, fname)),
"fname_sm": fname.replace(".gq.bed", ".sorted.merged.gq.bed")
}
elif options.inverse_deletions:
fname_splitted = fname.split(".")
species_dict = {
"sample":
fname_splitted[0],
"fname":
fname,
"ftype":
fname_splitted[-1],
"meitype":
fname_splitted[-2], #fname_splitted[2],
"f_bedtools":
BedTool(os.path.join(path, fname)).saveas(
os.path.join(options.out_path, fname)),
"fname_sm":
fname.replace(".bed", ".sorted.merged.bed")
}
print "\n loading %s" % fname,
print "\t BedTool object length: %i" % (len(
species_dict.get("f_bedtools"))),
if len(species_dict.get("f_bedtools")) < 3 or species_dict.get(
"meitype"
) == "DNA": # filter out empty BedTool object and DNA insetions
continue
print "\t performing analyses: ",
for analyses in prep_analyses: # perform initial analyses
print "\t %s" % analyses.__name__,
species_dict["f_bedtools"] = analyses(
species_dict.get("f_bedtools")
).saveas(
os.path.join(options.out_path, species_dict.get("fname_sm"))
) #.saveas(os.path.join(options.out_path, species_dict.get("fname_sm"))) # save again to dictionary
# species_dict.get("f_bedtools").saveas(
# os.path.join(options.out_path, species_dict.get("fname_sm"))) # save to file
spec_data.append(species_dict) # append to list
sample_list = set([l.get("sample") for l in spec_data])
return spec_data
else:
failed = False
f.close()
return failed
# 1) Check results of 'PreCluster' step
lsf_file = glob.glob('lsf/*.precluster')
lsf_file = lsf_file[-1] # take the latest
failed = check_precluster(lsf_file)
if failed:
raise Exception('\n--> PreCluster step failed!')
else:
print('jt - PreCluster step successfully completed')
# 2) Run 'JTCluster' step
print('jt - JTCluster submission:')
joblist_filename = glob.glob(os.path.join(os.getcwd(), '*.jobs'))[0]
joblist = yaml.load(open(joblist_filename))
for job in joblist.itervalues():
print('jt - Submitting job # %d' % job['jobID'])
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M-%S')
lsf = os.path.abspath(os.path.join('lsf',
'%.5d_%s.jtcluster' % (job['jobID'], st)))
call(['bsub', '-W', '8:00', '-o', lsf,
'-R', 'rusage[mem=4000,scratch=4000]',
'jt', 'run', '--job', '%s' % job['jobID']])
def start_redis_daemon():
cmd = [
'redis-server', '--unixsocket', '/tmp/redis.sock', '--daemonize', 'yes'
]
call(cmd)
def test_exit_no_option():
# When ran by itself, 'sfntedit' prints the usage
assert subprocess.call([TOOL]) == 1
def clearCache():
command = "rm -rf /home/bishal/.cache/mozilla/firefox/mw7he6gs.default/cache2/entries/*"
result = call(command, shell=True)
time.sleep(1)
def node_start(node_image, runtime, log_dir, ip, ip6, as_num, detach,
libnetwork_image):
"""
Create the calico-node container and establish Calico networking on this
host.
:param ip: The IPv4 address of the host.
:param node_image: The calico-node image to use.
:param ip6: The IPv6 address of the host (or None if not configured)
:param as_num: The BGP AS Number to use for this node. If not specified
the global default value will be used.
:param detach: True to run in Docker's "detached" mode, False to run
attached.
:param libnetwork_image: The name of the Calico libnetwork driver image to
use. None, if not using libnetwork.
:return: None.
"""
# Normally, Felix will load the modules it needs, but when running inside a
# container it might not be able to do so. Ensure the required modules are
# loaded each time the node starts.
# We only make a best effort attempt because the command may fail if the
# modules are built in.
# We'll warn during the check_system() if the modules are unavailable.
try:
call(["modprobe", "-a"] + REQUIRED_MODULES)
except OSError:
pass
# Print warnings for any known system issues before continuing
using_docker = True if runtime == 'docker' else False
(_, _, etcd_ok) = \
check_system(quit_if_error=False, libnetwork=libnetwork_image,
check_docker=using_docker)
if not etcd_ok:
sys.exit(1)
# We will always want to setup IP forwarding
_setup_ip_forwarding()
# Ensure log directory exists
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# Get IP address of host, if none was specified
if not ip:
ips = get_host_ips(exclude=[
"^docker.*", "^cbr.*", "virbr.*", "lxcbr.*", "veth.*", "cali.*",
"tunl.*"
])
try:
ip = ips.pop()
except IndexError:
print "Couldn't autodetect a management IP address. Please provide" \
" an IP by rerunning the command with the --ip=<IP_ADDRESS> flag."
sys.exit(1)
else:
print "No IP provided. Using detected IP: %s" % ip
# Verify that IPs are not already in use by another host.
error_if_bgp_ip_conflict(ip, ip6)
# Verify that the chosen IP exists on the current host
warn_if_unknown_ip(ip, ip6)
# Warn if this hostname conflicts with an existing host
warn_if_hostname_conflict(ip)
# Set up etcd
ipv4_pools = client.get_ip_pools(4)
ipv6_pools = client.get_ip_pools(6)
# Create default pools if required
if not ipv4_pools:
client.add_ip_pool(4, DEFAULT_IPV4_POOL)
if not ipv6_pools:
client.add_ip_pool(6, DEFAULT_IPV6_POOL)
client.ensure_global_config()
client.create_host(hostname, ip, ip6, as_num)
# If IPIP is enabled, the host requires an IP address for its tunnel
# device, which is in an IPIP pool. Without this, a host can't originate
# traffic to a pool address because the response traffic would not be
# routed via the tunnel (likely being dropped by RPF checks in the fabric).
ipv4_pools = client.get_ip_pools(4)
ipip_pools = [p for p in ipv4_pools if p.ipip]
if ipip_pools:
# IPIP is enabled, make sure the host has an address for its tunnel.
_ensure_host_tunnel_addr(ipv4_pools, ipip_pools)
else:
# No IPIP pools, clean up any old address.
_remove_host_tunnel_addr()
# The format of the authority string has already been validated.
etcd_authority = os.getenv(ETCD_AUTHORITY_ENV, ETCD_AUTHORITY_DEFAULT)
# Get etcd SSL environment variables if they exist
etcd_scheme = os.getenv(ETCD_SCHEME_ENV, ETCD_SCHEME_DEFAULT)
etcd_key_file = os.getenv(ETCD_KEY_FILE_ENV, None)
etcd_cert_file = os.getenv(ETCD_CERT_FILE_ENV, None)
etcd_ca_cert_file = os.getenv(ETCD_CA_CERT_FILE_ENV, None)
etcd_volumes = []
etcd_binds = {}
etcd_envs = [
"ETCD_AUTHORITY=%s" % etcd_authority,
"ETCD_SCHEME=%s" % etcd_scheme
]
felix_envs = [
"FELIX_ETCDADDR=%s" % etcd_authority,
"FELIX_ETCDSCHEME=%s" % etcd_scheme
]
if etcd_ca_cert_file and etcd_key_file and etcd_cert_file:
etcd_volumes.append(ETCD_CA_CERT_NODE_FILE)
etcd_binds[etcd_ca_cert_file] = {
"bind": ETCD_CA_CERT_NODE_FILE,
"ro": True
}
etcd_envs.append("ETCD_CA_CERT_FILE=%s" % ETCD_CA_CERT_NODE_FILE)
felix_envs.append("FELIX_ETCDCAFILE=%s" % ETCD_CA_CERT_NODE_FILE)
etcd_volumes.append(ETCD_KEY_NODE_FILE)
etcd_binds[etcd_key_file] = {"bind": ETCD_KEY_NODE_FILE, "ro": True}
etcd_envs.append("ETCD_KEY_FILE=%s" % ETCD_KEY_NODE_FILE)
felix_envs.append("FELIX_ETCDKEYFILE=%s" % ETCD_KEY_NODE_FILE)
etcd_volumes.append(ETCD_CERT_NODE_FILE)
etcd_binds[etcd_cert_file] = {"bind": ETCD_CERT_NODE_FILE, "ro": True}
etcd_envs.append("ETCD_CERT_FILE=%s" % ETCD_CERT_NODE_FILE)
felix_envs.append("FELIX_ETCDCERTFILE=%s" % ETCD_CERT_NODE_FILE)
if runtime == 'docker':
_start_node_container(ip, ip6, log_dir, node_image, detach, etcd_envs,
felix_envs, etcd_volumes, etcd_binds)
if libnetwork_image:
_start_libnetwork_container(libnetwork_image, etcd_envs,
etcd_volumes, etcd_binds)
def setta(bot, update, **optional_args):
#composizione del comando da lanciare
comando = 'curl "192.168.43.84:5555/setta?token=386429372:AAG7xgNjODVqBFg517NpbmhDNDbpVEdtO5M&chat=%s&soglia=' % update.message.chat_id
comando = comando + '%s"' % optional_args['args'][0]
#invia la richiesta http ad arduino
subprocess32.call(comando, shell=True)
import sys, os.path
from subprocess32 import call
from mathcheck_common import sq
if len(sys.argv) < 2:
print "need order (or min and max order) of cases to generate"
quit()
n = int(sys.argv[1])
inname = "input/compstring/%d.in"
if not os.path.exists("input"): call(["mkdir", "input"])
if not os.path.exists("input/compstring"): call(["mkdir", "input/compstring"])
indices = [
0 * (n / 2 + 1) + 1, 1 * (n / 2 + 1) + 1, 2 * (n / 2 + 1) + 1,
3 * (n / 2 + 1) + 1
]
f = open(inname % n, "w")
if n % 2 == 1:
f.write("p cnf {0} {1}\n".format(4 * (n / 2) + 4, 8 * (n / 2) + 4))
for j in range(1, n / 2 + 1):
#f.write("cx%d %d %d %d 0\n" % (-(indices[0]+j), (indices[1]+j), (indices[2]+j), (indices[3]+j)))
f.write("%d %d %d %d 0\n" % (-(indices[0] + j), -(indices[1] + j),
-(indices[2] + j), -(indices[3] + j)))
f.write("%d %d %d %d 0\n" % (-(indices[0] + j), -(indices[1] + j),
(indices[2] + j), (indices[3] + j)))
f.write("%d %d %d %d 0\n" % (-(indices[0] + j),
(indices[1] + j), -(indices[2] + j),
(indices[3] + j)))
def remove_veth(ep):
# The veth removal is best effort. If it fails then just log.
rc = call(['ip', 'link', 'del', ep.name], timeout=IP_CMD_TIMEOUT)
if rc != 0:
app.logger.warn("Failed to delete veth %s", ep.name)
return "%d.%d" % (n, c)
elif k == "*" and c == "*":
return "%d.*.*" % n
elif k == "*":
return "%d.%d.*" % (n, c)
else:
return "%d.%d.%d" % (n, c, k)
inname = "input/comp/%s.in"
compname = "input/comp/%s.comp"
assumname = "input/comp/%s.assum"
#prodvarname = "input/prodvar/%d.prodvars"
resultname = "results/compprog/%s.out"
logname = "output/compprog/%s.log"
if not os.path.exists("output"): call(["mkdir", "output"])
if not os.path.exists("output/compprog"): call(["mkdir", "output/compprog"])
if not os.path.exists("results"): call(["mkdir", "results"])
if not os.path.exists("results/compprog"): call(["mkdir", "results/compprog"])
for n in range(n1, n2+1):
# Generate SAT instance
#command = "python generate_compression_instances.py {0}".format(n)
#call(command.split(" "))
files = glob.glob(compname % runstr(n, "*", "*"))
count = 0
tottime = 0
for f in files:
k = int(f.split(".")[-2])
if qemu_use_rr:
qemu_args = ['rr', 'record', project['qemu'], '-replay', isoname]
subprocess32.check_call(qemu_args)
else:
raise
replay_time = tock()
print("taint analysis complete %.2f seconds" % replay_time)
sys.stdout.flush()
tick()
progress("Trying to create database {}...".format(project['name']))
createdb_args = ['createdb', '-U', 'postgres', project['db']]
createdb_result = subprocess32.call(createdb_args,
stdout=sys.stdout,
stderr=sys.stderr)
print()
if createdb_result == 0: # Created new DB; now populate
progress("Database created. Initializing...")
# psql_args = ['psql', '-U', 'postgres', '-d', project['db'],
# '-f', join(join(lavadir, 'include'), 'lava.sql')]
psql_args = [
'psql', '-U', 'postgres', '-d', project['db'], '-f',
join(join(lavadir, 'fbi'), 'lava.sql')
]
dprint("psql invocation: [%s]" % (" ".join(psql_args)))
subprocess32.check_call(psql_args, stdout=sys.stdout, stderr=sys.stderr)
else:
progress("Database already exists.")
def create_warped_vrt(self, id):
raster_path = '{}/{}/index.tif'.format(IMAGERY_PATH, id)
vrt_path = '{}/{}/index.vrt'.format(IMAGERY_PATH, id)
# initialize Flask
# TODO Celery's @worker_init.connect decorator _should_ work for this
app.config['SERVER_NAME'] = SERVER_NAME
meta = get_metadata(id)
approximate_zoom = meta['meta']['approximateZoom']
# create a warped VRT to reproject on the fly
gdalwarp = [
'gdalwarp',
raster_path,
vrt_path,
'-r',
'cubic',
'-t_srs',
'epsg:3857',
'-overwrite',
'-of',
'VRT',
'-te',
'-20037508.34',
'-20037508.34',
'20037508.34',
'20037508.34',
'-ts',
str(2**approximate_zoom * 256),
str(2**approximate_zoom * 256),
]
# add an alpha band (for NODATA) if one wasn't already included
if meta['meta']['bandCount'] < 4:
gdalwarp.append('-dstalpha')
started_at = datetime.utcnow()
self.update_state(state='RUNNING',
meta={
'name': 'warped-vrt',
'started_at': started_at.isoformat(),
'status': 'Creating warped VRT'
})
try:
returncode = subprocess.call(gdalwarp, timeout=60 * 5)
except subprocess.TimeoutExpired as e:
raise Exception(
json.dumps({
'name': 'overviews',
'started_at': started_at.isoformat(),
'command': ' '.join(gdalwarp),
'return_code': returncode,
'status': 'Timed out'
}))
if returncode != 0:
raise Exception(
json.dumps({
'name': 'warped-vrt',
'started_at': started_at.isoformat(),
'command': ' '.join(gdalwarp),
'return_code': returncode,
'status': 'Failed'
}))
return {
'completed_at': datetime.utcnow().isoformat(),
'started_at': started_at,
'status': 'Warped VRT creation completed'
}
def run_autorestart(args=None):
"""
Run B3 in auto-restart mode.
"""
restart_num = 0
if main_is_frozen():
# if we are running the frozen application we do not
# need to run any script, just the executable itself
script = ''
else:
# if we are running from sources, then sys.executable is set to `python`
script = os.path.join(modulePath[:-3], 'b3_run.py')
if not os.path.isfile(script):
# must be running from the wheel, so there is no b3_run
script = os.path.join(modulePath[:-3], 'b3', 'run.py')
if os.path.isfile(script + 'c'):
script += 'c'
if args:
script = '%s %s %s --autorestart' % (sys.executable, script,
' '.join(args))
else:
script = '%s %s --autorestart' % (sys.executable, script)
while True:
try:
try:
import subprocess32 as subprocess
except ImportError:
import subprocess
status = subprocess.call(script, shell=True)
sys.stdout.write('Exited with status: %s ... ' % status)
sys.stdout.flush()
sleep(2)
if status == 221:
restart_num += 1
sys.stdout.write('restart requested (%s)\n' % restart_num)
sys.stdout.flush()
elif status == 222:
sys.stdout.write('shutdown requested!\n')
sys.stdout.flush()
break
elif status == 220 or status == 223:
sys.stdout.write('B3 error (check log file)\n')
sys.stdout.flush()
break
elif status == 224:
sys.stdout.write('B3 error (check console)\n')
sys.stdout.flush()
break
elif status == 256:
sys.stdout.write('python error, (check log file)\n')
sys.stdout.flush()
break
elif status == 0:
sys.stdout.write('normal shutdown\n')
sys.stdout.flush()
break
elif status == 1:
sys.stdout.write('general error (check console)\n')
sys.stdout.flush()
break
else:
restart_num += 1
sys.stdout.write(
'unknown exit code (%s), restarting (%s)...\n' %
(status, restart_num))
sys.stdout.flush()
sleep(4)
except KeyboardInterrupt:
print('Quit')
break
def create_overviews(self, id):
raster_path = '{}/{}/index.tif'.format(IMAGERY_PATH, id)
# initialize Flask
# TODO Celery's @worker_init.connect decorator _should_ work for this
app.config['SERVER_NAME'] = SERVER_NAME
meta = get_metadata(id)
approximate_zoom = meta['meta']['approximateZoom']
# create external overviews
gdaladdo = [
'gdaladdo',
'-r',
'cubic',
'--config',
'GDAL_TIFF_OVR_BLOCKSIZE',
'256',
'--config',
'TILED_OVERVIEW',
'yes',
'--config',
'COMPRESS_OVERVIEW',
'DEFLATE',
'--config',
'PREDICTOR_OVERVIEW',
'2',
'--config',
'SPARSE_OK_OVERVIEW',
'yes',
'--config',
'BLOCKXSIZE_OVERVIEW',
'256',
'--config',
'BLOCKYSIZE_OVERVIEW',
'256',
'--config',
'INTERLEAVE_OVERVIEW',
'band',
'--config',
'NUM_THREADS_OVERVIEW',
'ALL_CPUS',
'-ro',
raster_path,
]
# generate a list of overview values
gdaladdo.extend([str(2**(x + 1)) for x in range(approximate_zoom)])
started_at = datetime.utcnow()
self.update_state(state='RUNNING',
meta={
'name': 'overviews',
'started_at': started_at.isoformat(),
'status': 'Creating external overviews'
})
try:
returncode = subprocess.call(gdaladdo, timeout=60 * 5)
except subprocess.TimeoutExpired as e:
raise Exception(
json.dumps({
'name': 'overviews',
'started_at': started_at.isoformat(),
'command': ' '.join(gdaladdo),
'return_code': returncode,
'status': 'Timed out'
}))
if returncode != 0:
raise Exception(
json.dumps({
'name': 'overviews',
'started_at': started_at.isoformat(),
'command': ' '.join(gdaladdo),
'return_code': returncode,
'status': 'Failed'
}))
return {
'name': 'overviews',
'completed_at': datetime.utcnow().isoformat(),
'started_at': started_at,
'status': 'Overview addition completed'
}
def host_cmd(args, unknown_args):
if not args.cmd:
raise TestFormatError('Must specify command to run on the host.')
elif unknown_args:
raise TestFormatError(
'Args "%s" unsupported. Is your host command correctly formatted?'
% (' '.join(unknown_args)))
elif args.deploy_chrome and not args.path_to_outdir:
raise TestFormatError(
'--path-to-outdir must be specified if --deploy-chrome is passed.')
cros_run_test_cmd = [
CROS_RUN_TEST_PATH,
'--board',
args.board,
'--cache-dir',
args.cros_cache,
]
if args.use_vm:
cros_run_test_cmd += [
'--start',
# Don't persist any filesystem changes after the VM shutsdown.
'--copy-on-write',
'--device',
'localhost',
]
else:
cros_run_test_cmd += [
'--device', args.device if args.device else LAB_DUT_HOSTNAME
]
if args.verbose:
cros_run_test_cmd.append('--debug')
if args.logs_dir:
for log in SYSTEM_LOG_LOCATIONS:
cros_run_test_cmd += ['--results-src', log]
cros_run_test_cmd += ['--results-dest-dir', args.logs_dir]
test_env = setup_env()
if args.deploy_chrome:
cros_run_test_cmd += [
'--deploy',
# Mounting the browser gives it enough disk space to not need stripping.
'--mount',
'--nostrip',
'--build-dir',
os.path.abspath(args.path_to_outdir),
]
cros_run_test_cmd += [
'--host-cmd',
'--',
] + args.cmd
logging.info('Running the following command:')
logging.info(' '.join(cros_run_test_cmd))
return subprocess.call(cros_run_test_cmd,
stdout=sys.stdout,
stderr=sys.stderr,
env=test_env)
