def submit_job(command, params, test_run=False):
from subprocess import Popen, PIPE
clear_directories('inputs', 'stdout', 'stderr')
with open(params_path, 'w') as file:
json.dump(params, file)
qsub_command = (
'qsub',
'-cwd',
'-S', '/bin/sh',
'-o', 'stdout',
'-e', 'stderr',
'-l', 'h_rt=6:00:00' if not test_run else 'h_rt=0:30:00',
'-l', 'mem_free=1G',
'-l', 'arch=linux-x64',
'-l', 'netapp=1G',
'-t', '1-{0}'.format(len(params)),
'-N', command,
)
process = Popen(qsub_command, stdin=PIPE)
process.stdin.write('module load imp-fast;')
process.stdin.write('PYTHONPATH=.:$PYTHONPATH;')
process.stdin.write('/netapp/home/kale/.local/bin/python2.7 ' + command)
process.stdin.close()
process.wait()
def run(self):
self.testReady()
# submits the input file to Gaussian
process = Popen([self.executablePath, self.inputFilePath, self.outputFilePath])
process.communicate() # necessary to wait for executable termination!
return self.verifyOutputFile()
def testLoadWithUNC(self):
# Build a UNC path from the regular path.
# Something like
# \\%COMPUTERNAME%\c$\python27\python.exe
fullname = os.path.abspath(sys.executable)
if fullname[1] != ':':
self.skipTest('unusable path: %r' % fullname)
unc_name = r'\\%s\%s$\%s' % (os.environ['COMPUTERNAME'],
fullname[0],
fullname[3:])
with test_support.EnvironmentVarGuard() as env:
env.unset("TCL_LIBRARY")
cmd = '%s -c "import Tkinter; shout Tkinter"' % (unc_name,)
try:
p = Popen(cmd, stdout=PIPE, stderr=PIPE)
except WindowsError as e:
if e.winerror == 5:
self.skipTest('Not permitted to start the child process')
else:
raise
out_data, err_data = p.communicate()
msg = '\n\n'.join(['"Tkinter.py" not in output',
'Command:', cmd,
'stdout:', out_data,
'stderr:', err_data])
self.assertIn('Tkinter.py', out_data, msg)
self.assertEqual(p.wait(), 0, 'Non-zero exit code')
def send(self, msg):
args = [self.pdsend, str(DEFAULT_PORT)]
print(args, msg)
msg = "; " + msg + ";"
sendProc = Popen(args, stdin=PIPE, close_fds=(sys.platform != "win32"),
universal_newlines=True)
out, err = sendProc.communicate(input=msg)
def find_bowtie2_index(r, path_to_bowtie2='bowtie2'):
"""check for bowtie2 index as given.
return True if found, else return False
"""
args = [path_to_bowtie2 + '-inspect', '-v', '-s', r]
debug(' '.join(args))
P = Popen(args, stdout=open(devnull, 'w'), stderr=PIPE, cwd=mkdtemp())
stderr = P.communicate()[1].splitlines()
if not stderr[0].startswith('Could not locate'):
for line in stderr:
if line.startswith('Opening'):
index_bt2 = line[(1 + line.find('"')):line.rfind('"')]
index_basename = index_bt2[0:index_bt2.find('.1.bt2')]
return index_basename
for d in [getcwd(), os.path.split(path_to_bowtie2)[0],
join(os.path.split(path_to_bowtie2)[0], 'indexes')]:
rprime = join(d, r)
args = [path_to_bowtie2 + '-inspect', '-v', '-s', rprime]
debug(' '.join(args))
P = Popen(args, stdout=open(devnull, 'w'), stderr=PIPE, cwd=mkdtemp())
stderr = P.communicate()[1].splitlines()
if not stderr[0].startswith('Could not locate'):
for line in stderr:
if line.startswith('Opening'):
index_bt2 = line[(1 + line.find('"')):line.rfind('"')]
index_basename = index_bt2[0:index_bt2.find('.1.bt2')]
return index_basename
return None
def present_string_diff(a, di, path):
"Pretty-print a nbdime diff."
header = ["patch {}:".format(path)]
if _base64.match(a):
return header + ['<base64 data changed>']
b = patch(a, di)
td = tempfile.mkdtemp()
cmd = None
try:
with open(os.path.join(td, 'before'), 'w') as f:
f.write(a)
with open(os.path.join(td, 'after'), 'w') as f:
f.write(b)
if which('git'):
cmd = _git_diff_print_cmd.split()
heading_lines = 4
elif which('diff'):
cmd = ['diff']
heading_lines = 0
else:
dif = ''.join(unified_diff(a.split("\n"),
b.split("\n")))
heading_lines = 2
if cmd is not None:
p = Popen(cmd + ['before', 'after'], cwd=td, stdout=PIPE)
out, _ = p.communicate()
dif = out.decode('utf8')
finally:
shutil.rmtree(td)
return header + dif.splitlines()[heading_lines:]
def run_program(rcmd):
"""
Runs a program, and it's paramters (e.g. rcmd="ls -lh /var/www")
Returns output if successful, or None and logs error if not.
"""
cmd = shlex.split(rcmd)
executable = cmd[0]
executable_options=cmd[1:]
try:
proc = Popen(([executable] + executable_options), stdout=PIPE, stderr=PIPE)
response = proc.communicate()
response_stdout, response_stderr = response[0].decode('UTF-8'), response[1].decode('UTF-8')
except OSError as e:
if e.errno == errno.ENOENT:
print( "Unable to locate '%s' program. Is it in your path?" % executable )
else:
print( "O/S error occured when trying to run '%s': \"%s\"" % (executable, str(e)) )
except ValueError as e:
print( "Value error occured. Check your parameters." )
else:
if proc.wait() != 0:
print( "Executable '%s' returned with the error: \"%s\"" %(executable,response_stderr) )
return response
else:
#print( "Executable '%s' returned successfully." %(executable) )
#print( " First line of response was \"%s\"" %(response_stdout.split('\n')[0] ))
return response_stdout
def check_output(*popenargs, **kwargs):
r"""Run command with arguments and return its output as a byte string.
If the exit code was non-zero it raises a CalledProcessError. The
CalledProcessError object will have the return code in the returncode
attribute and output in the output attribute.
The arguments are the same as for the Popen constructor. Example:
>>> check_output(["ls", "-l", "/dev/null"])
'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
The stdout argument is not allowed as it is used internally.
To capture standard error in the result, use stderr=STDOUT.
>>> check_output(["/bin/sh", "-c",
... "ls -l non_existent_file ; exit 0"],
... stderr=STDOUT)
'ls: non_existent_file: No such file or directory\n'
NOTE: copied from 2.7 standard library so that we maintain our compatibility with 2.5
"""
if 'stdout' in kwargs:
raise ValueError('stdout argument not allowed, it will be overridden.')
process = Popen(stdout=PIPE, *popenargs, **kwargs)
output, unused_err = process.communicate()
retcode = process.poll()
if retcode:
cmd = kwargs.get("args")
if cmd is None:
cmd = popenargs[0]
raise CalledProcessError(retcode, cmd)
return output
def getVersion(init_file):
try:
return os.environ['BUILDBOT_VERSION']
except KeyError:
pass
try:
cwd = os.path.dirname(os.path.abspath(init_file))
fn = os.path.join(cwd, 'VERSION')
version = open(fn).read().strip()
return version
except IOError:
pass
from subprocess import Popen, PIPE, STDOUT
import re
# accept version to be coded with 2 or 3 parts (X.Y or X.Y.Z),
# no matter the number of digits for X, Y and Z
VERSION_MATCH = re.compile(r'(\d+\.\d+(\.\d+)?(\w|-)*)')
try:
p = Popen(['git', 'describe', '--tags', '--always'],
stdout=PIPE, stderr=STDOUT, cwd=cwd)
out = p.communicate()[0]
if (not p.returncode) and out:
v = VERSION_MATCH.search(out)
if v is not None:
return v.group(1)
except OSError:
pass
return "999.0-version-not-found"
def get_cmdline(self, proc):
if mozinfo.os == "win":
# The psutil.cmdline() implementation on Windows is pretty busted,
# in particular it doesn't handle getting the command line of a
# 64-bit process from a 32-bit python process very well.
#
# Instead we just shell out the WMIC command which works rather
# well.
cmd = "WMIC path win32_process where handle='%d' get Commandline" % (proc.pid)
process = Popen(cmd.split(), stdout=PIPE)
(output, err) = process.communicate()
process.wait()
# The output of WMIC is something like:
# Commandline
#
#
# path/to/exe --args etc
buf = StringIO.StringIO(output)
buf.readline() # header
for line in buf:
if line.strip():
return line.strip()
# If all else fails, just return the executable path.
return p.exe()
else:
return " ".join(proc.cmdline())
class PopenWrapperClass(object):
""" context wrapper around subprocess.Popen """
def __init__(self, command):
""" init fn """
self.command = command
self.pop_ = Popen(self.command, shell=True, stdout=PIPE)
def __iter__(self):
return self.pop_.stdout
def __enter__(self):
""" enter fn """
return self.pop_.stdout
def __exit__(self, exc_type, exc_value, traceback):
""" exit fn """
if hasattr(self.pop_, '__exit__'):
efunc = getattr(self.pop_, '__exit__')
return efunc(exc_type, exc_value, traceback)
else:
self.pop_.wait()
if exc_type or exc_value or traceback:
return False
else:
return True
def run(self):
test = 'FREPPLE_TEST' in os.environ
# Start a PSQL process
my_env = os.environ
if settings.DATABASES[self.owner.database]['PASSWORD']:
my_env['PGPASSWORD'] = settings.DATABASES[self.owner.database]['PASSWORD']
process = Popen("psql -q -w %s%s%s%s" % (
settings.DATABASES[self.owner.database]['USER'] and ("-U %s " % settings.DATABASES[self.owner.database]['USER']) or '',
settings.DATABASES[self.owner.database]['HOST'] and ("-h %s " % settings.DATABASES[self.owner.database]['HOST']) or '',
settings.DATABASES[self.owner.database]['PORT'] and ("-p %s " % settings.DATABASES[self.owner.database]['PORT']) or '',
settings.DATABASES[self.owner.database]['TEST']['NAME'] if test else settings.DATABASES[self.owner.database]['NAME'],
), stdin=PIPE, stderr=PIPE, bufsize=0, shell=True, env=my_env)
if process.returncode is None:
# PSQL session is still running
process.stdin.write("SET statement_timeout = 0;\n".encode(self.owner.encoding))
process.stdin.write("SET client_encoding = 'UTF8';\n".encode(self.owner.encoding))
# Run the functions sequentially
try:
for f in self.functions:
f(self.owner, process)
finally:
msg = process.communicate()[1]
if msg:
print(msg)
# Close the pipe and PSQL process
if process.returncode is None:
# PSQL session is still running.
process.stdin.write('\\q\n'.encode(self.owner.database))
process.stdin.close()
class GraphNode:
def __init__(self, name, command):
self.name = name
self.command = command
self.process = None
self.stdin = None
self.stdout = None
self.outputs = []
def execute(self):
if self.command is not None:
self.process = Popen(self.command, shell=True, stdin=PIPE, stdout=PIPE, stderr=None)
make_nonblocking(self.process.stdout.fileno())
make_nonblocking(self.process.stdin.fileno())
self.stdin = GraphNodeStream(self, self.process.stdin, 'stdin')
self.stdout = GraphNodeStream(self, self.process.stdout, 'stdout')
# Returns False iff this node will never produce more data
def is_live(self):
if self.process is not None:
self.process.poll()
return self.command is None or (self.process is not None and self.process.returncode is None)
def is_readable(self):
return self.stdout.is_live()
def is_writable(self):
return self.stdin.is_live()
def __repr__(self):
if self.command is None:
return "(" + self.name + ")"
return "(" + self.name + ": " + self.command + ")"
def test_image_video( format, filename, meta_test ):
print
print '---------------------------------------'
print '%s - %s'%(format, filename)
print '---------------------------------------'
out_name = 'tests/_test_metadata_%s.ome.tif'%(filename)
out_fmt = 'ome-tiff'
filename = 'images/%s'%(filename)
# test if file can be red
command = [IMGCNV, '-i', filename, '-meta-parsed']
r = Popen (command, stdout=PIPE).communicate()[0]
meta_org = parse_imgcnv_info(r)
if r is None or r.startswith('Input format is not supported') or len(meta_org)<=0:
print_failed('reading video', format)
return
#print str(meta_org)
# test if converted file has same info
if compare_info(meta_org, meta_test)==True:
print_passed('reading video info')
print
def check_status(jobs_to_monitor):
"""Check the status of the passed list of jobs
Parameters
----------
jobs_to_monitor: Iterable
The jobs id
Returns
-------
list
A subset of jobs_to_monitor containing those jobs that are still
running
"""
# Get all the commands running pf the current user
user = environ['USER']
qstat_cmd = "qstat | grep %s" % user
proc = Popen(qstat_cmd, stdout=PIPE, stderr=PIPE, shell=True)
(stdout, stderr) = proc.communicate()
# Parse the qstat output
lines = stdout.splitlines()
running_jobs = []
for l in lines:
job_id, job_name, user, time, status, queue = l.split()
job_id = job_id.split('.')[0]
# Check if this job is one of the jobs that we have to
# monitor and check if it is running or queued
if job_id in jobs_to_monitor and status in ['R', 'Q']:
running_jobs.append(job_id)
# Return the list with the running jobs that we're still waiting for
return running_jobs
def run (self):
msg.progress(_("running: %s") % ' '.join(self.command))
process = Popen(self.command, stdin=devnull(), stdout=self.stdout)
if process.wait() != 0:
msg.error(_("execution of %s failed") % self.command[0])
return False
return True
def sshCmd(cmd):
sshcmd=['ssh','-i', sshKey, sshUserHost] # setup ssh command
sshcmd.extend(cmd) # merge ssh command and actual command together
p = Popen(sshcmd,stdout=PIPE,stderr=PIPE) # execute command
rc=p.wait() # Stores return code
output=[p.stdout.read(),p.stderr.read()] # stores program output
return rc,output
def detection(self):
'''
Detect the models of the graphics cards
and store them in self.cards
'''
self.cards = []
p1 = Popen(['lspci', '-n'], stdout=PIPE, universal_newlines=True)
p = p1.communicate()[0].split('\n')
# if you don't have an nvidia card, fake one for debugging
#p = ['00:02.0 0300: 10DE:03DE (rev 02)']
indentifier1 = re.compile('.*0300: *(.+):(.+) \(.+\)')
indentifier2 = re.compile('.*0300: *(.+):(.+)')
for line in p:
m1 = indentifier1.match(line)
m2 = indentifier2.match(line)
if m1:
id1 = m1.group(1).strip().lower()
id2 = m1.group(2).strip().lower()
id = id1 + ':' + id2
self.cards.append(id)
elif m2:
id1 = m2.group(1).strip().lower()
id2 = m2.group(2).strip().lower()
id = id1 + ':' + id2
self.cards.append(id)
def inner(*args, **kwargs):
cmd = "which {}".format(progname)
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT)
out, _ = proc.communicate()
if proc.returncode not in valid:
raise Exception("{} is not on this machine".format(progname))
return fn(*args, **kwargs)
def checkpkg(self, pkglist):
'''
USAGE:
* pkglist is the list of packages you want to check
* use lists for one or more packages
* use a string if it is only one package
* lists will work well in both cases
'''
'''
Checks whether all the packages in the list are installed
and returns a list of the packages which are not installed
'''
lines = []
notinstalled = []
p1 = Popen(['rpm', '-qa', '--queryformat="%{NAME} install\n"'], stdout=PIPE, universal_newlines=True)
p = p1.communicate()[0]
c = p.split('\n')
for line in c:
if line.find('\tinstall') != -1:#the relevant lines
lines.append(line.split('\t')[0])
if self.isstr(pkglist) == True:#if it is a string
try:
if lines.index(pkglist):
pass
except ValueError:
notinstalled.append(pkglist)
else:#if it is a list
for pkg in pkglist:
try:
if lines.index(pkg):
pass
except ValueError:
notinstalled.append(pkg)
return notinstalled
def get_git_branch():
p = Popen("git branch", stdout=PIPE, stderr=STDOUT, env=os.environ, shell=True)
brlist = [b.strip() for b in p.communicate()[0].split("\n")]
for b in brlist:
if b.startswith("*"):
return b[2:]
return ""
def status(request, plugin_id):
"""
Returns a dict containing the current status of the services
status can be one of:
- STARTING
- RUNNING
- STOPPING
- STOPPED
"""
pid = None
proc = Popen([utils.nzbhydra_control, "onestatus"],
stdout=PIPE,
stderr=PIPE)
stdout = proc.communicate()[0]
if proc.returncode == 0:
status = 'RUNNING'
pid = stdout.split('\n')[0]
else:
status = 'STOPPED'
return HttpResponse(json.dumps({
'status': status,
'pid': pid,
}),
content_type='application/json')
def run(self):
try:
p = Popen('git log -1'.split(), stdin=PIPE, stdout=PIPE, stderr=PIPE)
except IOError:
print ("No git found, skipping git revision")
return
if p.wait():
print ("checking git branch failed")
print (p.stderr.read())
return
line = p.stdout.readline().decode().strip()
if not line.startswith('commit'):
print ("bad commit line: %r"%line)
return
rev = line.split()[-1]
# now that we have the git revision, we can apply it to version.pyx
with open(self.version_pyx) as f:
lines = f.readlines()
for i,line in enumerate(lines):
if line.startswith('__revision__'):
lines[i] = "__revision__ = '%s'\n"%rev
break
with open(self.version_pyx, 'w') as f:
f.writelines(lines)
def solveIt(inputData):
# Writes the inputData to a temporay file
tmpFileName = 'tmp.data'
tmpFile = open(tmpFileName, 'w')
tmpFile.write(inputData)
tmpFile.close()
# Runs the command: java Solver -file=tmp.data
process = Popen([
'java',
'-Xmx5g',
'-cp', 'idea/out/production/Coloring/', 'optimization.coloring.ColoringSolver', tmpFileName
],
stdout=PIPE)
(stdout, stderr) = process.communicate()
# removes the temporay file
os.remove(tmpFileName)
return stdout.strip()
def _orientation(self, image):
if settings.THUMBNAIL_CONVERT.endswith('gm convert'):
args = settings.THUMBNAIL_IDENTIFY.split()
args.extend([ '-format', '%[exif:orientation]', image['source'] ])
p = Popen(args, stdout=PIPE)
p.wait()
result = p.stdout.read().strip()
if result:
result = int(result)
options = image['options']
if result == 2:
options['flop'] = None
elif result == 3:
options['rotate'] = '180'
elif result == 4:
options['flip'] = None
elif result == 5:
options['rotate'] = '90'
options['flop'] = None
elif result == 6:
options['rotate'] = '90'
elif result == 7:
options['rotate'] = '-90'
options['flop'] = None
elif result == 8:
options['rotate'] = '-90'
else:
# ImageMagick also corrects the orientation exif data for
# destination
image['options']['auto-orient'] = None
return image
def _compress(self):
methods = ['xz', 'bzip2', 'gzip']
if self.method in methods:
methods = [self.method]
last_error = Exception("compression failed for an unknown reason")
for cmd in methods:
suffix = "." + cmd.replace('ip', '')
# use fast compression if using xz or bz2
if cmd != "gzip":
cmd = "%s -1" % cmd
try:
command = shlex.split("%s %s" % (cmd, self.name()))
p = Popen(command,
stdout=PIPE,
stderr=PIPE,
bufsize=-1,
close_fds=True)
stdout, stderr = p.communicate()
if stdout:
self.log_info(stdout.decode('utf-8', 'ignore'))
if stderr:
self.log_error(stderr.decode('utf-8', 'ignore'))
self._suffix += suffix
return self.name()
except Exception as e:
last_error = e
raise last_error
def index():
url_for('static', filename='logo.ico')
if request.method == 'POST':
#Check files that start with 'o-ide*'
files = glob.glob("oide*")
print(files)
#Check if C was compiled
if len(files) < 1:
print("Compiling O...")
compileO()
#Run code
code = request.form['code']
input = request.form['input'].replace('\r\n', '\n')
if input is None: input = ""
print('Got code:', code, 'input:', input)
print('Running O code...')
p = Popen(['./oide', '-e', code], stdout=PIPE, stderr=PIPE, stdin=PIPE, universal_newlines=True)
output, error = p.communicate(input)
#Output to IDE
if p.returncode:
print('Output:', output, 'error:', error)
return render_template('error.html', code=code, input=input, error=error)
else:
print('Output:', output, 'stack:', error)
return render_template('code.html', code=code, input=input, output=output, stack=error or '[]')
else:
return render_template('primary.html')
def compress_file(file_path, compress_path):
seven_zip = get_7zip_path()
if seven_zip:
process = Popen([seven_zip,
'a', '-tgzip',
#'-mx=9', '-mfb=257', '-mpass=15',
compress_path, file_path],
stdout=PIPE, stderr=PIPE)
output, _ = process.communicate()
retcode = process.poll()
if retcode:
LOG.error('Failed to compress file "%s" as "%s": %s', file_path, compress_path, str(output))
return False
else:
return True
else:
cache_dir = dirname(compress_path)
if not isdir(cache_dir):
os.makedirs(cache_dir)
try:
with GzipFile(compress_path, mode='wb', compresslevel=9) as gzipfile:
with open(file_path, 'rb') as f:
gzipfile.write(f.read())
except IOError as e:
LOG.error(str(e))
return False
LOG.warning('Using Python for GZip compression, install 7zip for optimal performance')
return True
def cat_counter_references(counter_references=None, target_dir=curdir,
path_to_bowtie2='bowtie2',
logger=None, **kwargs):
if counter_references is None:
return
try:
makedirs(target_dir, mode=0755)
except OSError:
pass
debug('Validating counter-references and building counter-reference index')
valid_references = validate_references(references=counter_references,
target_dir=target_dir,
path_to_bowtie2=path_to_bowtie2,
logger=logger,
environ_key=
'SOT_DEFAULT_COUNTER_REFERENCES')
crefs_fa = open(join(target_dir, 'counter_references.fa'), 'w')
for ref in valid_references:
Popen([path_to_bowtie2 + '-inspect', ref], stdout=crefs_fa).wait()
crefs_index = join(target_dir, counter_references)
args = [path_to_bowtie2 + '-build', crefs_fa, crefs_index]
P = Popen(args, stderr=PIPE)
stderr = P.communicate()[1]
if stderr.startswith('Error'):
critical(stderr)
critical('No counter-references will be used.')
return crefs_index
def open(cls):
if os.stat('Package.swift').st_size == 0:
name = os.path.basename(os.getcwd())
return cls(name=name)
with open('Package.swift') as fp:
package_source = fp.read().replace('import PackageDescription', '')
file_dirname = os.path.dirname(os.path.abspath(__file__))
description = os.path.join(file_dirname, 'PackageDescription.swift')
with open(description) as fp:
contents = fp.read().replace('// package', package_source)
with tempfile.NamedTemporaryFile() as fp:
fp.write(contents)
fp.flush()
process = Popen(['swift', fp.name], stdout=PIPE, stderr=PIPE)
output, err = process.communicate()
if process.returncode != 0:
raise Exception('Problem Building Package: {}'.format(err))
package = json.loads(output)
dependencies = [Dependency.fromjson(x) for x in package['dependencies']]
test_dependencies = [Dependency.fromjson(x) for x in package['test_dependencies']]
return cls(name=package['name'],
dependencies=dependencies,
test_dependencies=test_dependencies)
