def run_python(path,
env=None,
args=None,
timeout=None,
pythonpath_extend=None,
expect_pass=False):
new_argv = [sys.executable]
if sys.version_info[:2] <= (2, 7):
new_argv += ['-W', 'ignore:Python 2 is no longer supported']
new_env = os.environ.copy()
new_env.setdefault('eventlet_test_in_progress', 'yes')
src_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if path:
path = os.path.abspath(path)
new_argv.append(path)
new_env['PYTHONPATH'] = os.pathsep.join(sys.path + [src_dir])
if env:
new_env.update(env)
if pythonpath_extend:
new_path = [
p for p in new_env.get('PYTHONPATH', '').split(os.pathsep) if p
]
new_path.extend(p if os.path.isabs(p) else os.path.join(src_dir, p)
for p in pythonpath_extend)
new_env['PYTHONPATH'] = os.pathsep.join(new_path)
if args:
new_argv.extend(args)
p = subprocess.Popen(
new_argv,
env=new_env,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
)
if timeout is None:
timeout = 10
try:
output, _ = p.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
p.kill()
output, _ = p.communicate(timeout=timeout)
if expect_pass:
sys.stderr.write('Program {0} output:\n---\n{1}\n---\n'.format(
path, output.decode()))
assert False, 'timed out'
return '{0}\nFAIL - timed out'.format(output).encode()
if expect_pass:
if output.startswith(b'skip'):
parts = output.rstrip().split(b':', 1)
skip_args = []
if len(parts) > 1:
skip_args.append(parts[1])
raise SkipTest(*skip_args)
ok = output.rstrip() == b'pass'
if not ok:
sys.stderr.write('Program {0} output:\n---\n{1}\n---\n'.format(
path, output.decode()))
assert ok, 'Expected single line "pass" in stdout'
return output
def plot_raja_mmultgpu(cali_file,
baseFileName,
cali_query,
title=True,
legend=True):
ANNOTATIONS = ':'.join(
['iteration', 'size', 'mode', 'time.inclusive.duration'])
CALI_QUERY = _get_cali_query(cali_query, ANNOTATIONS, cali_file)
p = subprocess.Popen(CALI_QUERY, stdout=subprocess.PIPE)
out, err = p.communicate()
data = {}
for line in out.split():
line = dict(map(lambda x: x.split('='), line.split(',')))
if len(line) != 4:
continue
time = int(line['time.inclusive.duration'])
size = int(line['size'])
mode = line['mode']
if mode not in data:
data[mode] = {}
if size not in data[mode]:
data[mode][size] = []
data[mode][size].append(time)
colors = ('r', 'b', 'k', 'g')
markers = ('o', 'v', 's', '*')
namelist = (('Mmult1_cuda', 'RAJA w/ CUDA Loop Fusion'),
('Mmult2_cuda', 'RAJA w/ CUDA normal'),
('Mmult1_agency', 'RAJA w/ Agency Loop Fusion'),
('Mmult2_agency', 'RAJA w/ Agency normal'))
names = {
key: (name, colors[i], markers[i])
for i, (key, name) in enumerate(namelist)
}
fig = plt.figure()
legendNames = []
for mode, datum in data.iteritems():
sizes = sorted(datum.keys())
xerr = [0] * len(sizes)
means = [mean(datum[size]) for size in sizes]
sems = [sem(datum[size]) for size in sizes]
name, color, marker = names[mode]
legendNames.append(name)
plt.errorbar(sizes,
means,
xerr=xerr,
yerr=sems,
marker=marker,
color=color)
if legend:
plt.legend(legendNames, loc='best', fontsize=12)
if title:
plt.title('Matrix Multiplication Performance', fontsize=22)
plt.xlabel('Matrix Size', fontsize=16)
plt.ylabel('Time Taken (ms)', fontsize=16)
plt.yscale('log')
plt.grid(b=True, which='major', color='k', linestyle='dotted')
filename = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'figs',
'{}_gpu.pdf'.format(baseFileName))
plt.savefig(filename)
def plot_raja_dependent_bench(cali_file, filename, cali_query):
ANNOTATIONS = ':'.join(
['iteration', 'size', 'loop', 'time.inclusive.duration'])
CALI_QUERY = _get_cali_query(cali_query, ANNOTATIONS, cali_file)
p = subprocess.Popen(CALI_QUERY, stdout=subprocess.PIPE)
out, err = p.communicate()
data = {
'Agency': {},
'OMP': {},
}
for line in out.split():
line = dict(map(lambda x: x.split('='), line.split(',')))
loop = None
time = None
mode = None
size = None
for key, value in line.iteritems():
if key == 'time.inclusive.duration':
time = int(value)
elif key == 'iteration':
mode = value
elif key == 'size':
size = int(value)
elif key == 'loop':
loop = int(value)
if mode == 'Agency' or mode == 'OMP':
if size not in data[mode]:
data[mode][size] = []
data[mode][size].append(time)
OMP = data['OMP']
agency = data['Agency']
fig = plt.figure()
dat = [[size, omp, a] for size in OMP
for omp, a in zip(OMP[size], agency[size])]
dataframe = pd.DataFrame(data=dat, columns=['Size', 'OMP', 'Agency'])
d = [[
size,
dataframe[dataframe['Size'] == size]['OMP'].mean(),
stats.sem(dataframe[dataframe['Size'] == size]['OMP']),
dataframe[dataframe['Size'] == size]['Agency'].mean(),
stats.sem(dataframe[dataframe['Size'] == size]['Agency']),
] for size in sorted(OMP.keys())]
realDataframe = pd.DataFrame(
data=d,
columns=['Size', 'OMPmean', 'OMPsem', 'Agencymean', 'Agencysem'])
fig = plt.figure()
sizes = sorted(OMP)
legendNames = []
legendNames.append('OMP RAJA')
legendNames.append('Agency RAJA')
plt.xscale('log')
plt.yscale('log')
plt.errorbar(sizes,
realDataframe['OMPmean'],
color='r',
xerr=[0] * len(sizes),
yerr=realDataframe['OMPsem'])
plt.errorbar(sizes,
realDataframe['Agencymean'],
color='g',
xerr=[0] * len(sizes),
yerr=realDataframe['Agencysem'])
plt.legend(legendNames, loc='center right')
plt.xlabel('Array Size', fontsize=16)
plt.ylabel('Total Time (ms)', fontsize=16)
plt.grid(b=True, which='major', color='k', linestyle='dotted')
plt.title('Dependent Index Sets Benchmark', fontsize=22)
plt.savefig(filename)
def execute(*args, **kwargs):
logger = logging.getLogger(logger_name)
command = 'simpleflow.execute' # name of a module.
sys.stdout.flush()
sys.stderr.flush()
result_str = None # useless
context = kwargs.pop('context', {})
with tempfile.TemporaryFile() as result_fd, tempfile.TemporaryFile(
) as error_fd:
dup_result_fd = os.dup(result_fd.fileno()) # remove FD_CLOEXEC
dup_error_fd = os.dup(error_fd.fileno()) # remove FD_CLOEXEC
arguments_json = format_arguments_json(*args, **kwargs)
full_command = [
interpreter,
'-m',
command, # execute module a script.
get_name(func),
'--logger-name={}'.format(logger_name),
'--result-fd={}'.format(dup_result_fd),
'--error-fd={}'.format(dup_error_fd),
'--context={}'.format(json_dumps(context)),
]
if len(
arguments_json
) < MAX_ARGUMENTS_JSON_LENGTH: # command-line limit on Linux: 128K
full_command.append(arguments_json)
arg_file = None
arg_fd = None
else:
arg_file = tempfile.TemporaryFile()
arg_file.write(arguments_json.encode('utf-8'))
arg_file.flush()
arg_file.seek(0)
arg_fd = os.dup(arg_file.fileno())
full_command.append(
'--arguments-json-fd={}'.format(arg_fd))
full_command.append('foo') # dummy funcarg
if kill_children:
full_command.append('--kill-children')
if is_buggy_subprocess32(
): # close_fds doesn't work with subprocess32 < 3.5.0
close_fds = False
pass_fds = []
else:
close_fds = True
pass_fds = [dup_result_fd, dup_error_fd]
if arg_file:
pass_fds.append(arg_fd)
process = subprocess.Popen(
full_command,
bufsize=-1,
close_fds=close_fds,
pass_fds=pass_fds,
)
rc = wait_subprocess(process,
timeout=timeout,
command_info=full_command)
os.close(dup_result_fd)
os.close(dup_error_fd)
if arg_file:
arg_file.close()
if rc:
error_fd.seek(0)
err_output = error_fd.read()
if err_output:
if not compat.PY2:
err_output = err_output.decode('utf-8',
errors='replace')
raise ExecutionError(err_output)
result_fd.seek(0)
result_str = result_fd.read()
if not result_str:
return None
try:
if not compat.PY2:
result_str = result_str.decode('utf-8', errors='replace')
result = format.decode(result_str)
return result
except BaseException as ex:
logger.exception('Exception in python.execute: {} {}'.format(
ex.__class__.__name__, ex))
logger.warning('%r', result_str)
def run(command,
timeout=60,
ignore_status=False,
env=None,
io_encoding='utf-8'):
"""Execute a command in a subproccess and return its output.
Commands can be either shell commands (given as strings) or the
path and arguments to an executable (given as a list). This function
will block until the subprocess finishes or times out.
Args:
command: The command to execute. Can be either a string or a list.
timeout: number seconds to wait for command to finish.
ignore_status: bool True to ignore the exit code of the remote
subprocess. Note that if you do ignore status codes,
you should handle non-zero exit codes explicitly.
env: dict enviroment variables to setup on the remote host.
io_encoding: str unicode encoding of command output.
Returns:
A job.Result containing the results of the ssh command.
Raises:
job.TimeoutError: When the remote command took to long to execute.
Error: When the ssh connection failed to be created.
CommandError: Ssh worked, but the command had an error executing.
"""
start_time = time.time()
proc = subprocess.Popen(command,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=not isinstance(command, list))
# Wait on the process terminating
timed_out = False
out = bytes()
err = bytes()
try:
(out, err) = proc.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
timed_out = True
proc.kill()
proc.wait()
result = Result(command=command,
stdout=out,
stderr=err,
exit_status=proc.returncode,
duration=time.time() - start_time,
encoding=io_encoding,
did_timeout=timed_out)
logging.debug(result)
if timed_out:
logging.error("Command %s with %s timeout setting timed out", command,
timeout)
raise TimeoutError(result)
if not ignore_status and proc.returncode != 0:
raise Error(result)
return result
def run(self):
self._popen = subprocess.Popen(self._cmd, env=self._env, cwd=self._cwd)
def runMD(wdir,
localExe=True,
sshConn=None,
remoteSettings=None,
scriptExe='lie_mdrun.sh',
killer=None,
killSig='/KILL'):
"""
Kill of the process can take place via: killing job manager(runLIE)
or creation of a file (killSig)
In the first case status is updated till this step
In the last, job become CANCELLED
"""
if killer == None:
killer = jobHandler.WarmKiller()
currDir = os.getcwd()
#get list of directory and execute MDs
listMDs = [x[0] for x in os.walk(wdir) if x[0] != wdir]
listResults = []
for simDir in listMDs:
simInfo = {}
simInfo['wdir'] = simDir
logging.debug('Running simulation %s' % simDir)
if localExe:
### LOCAL EXECUTION
os.chdir(simDir)
#save stdout and stderr of the script in files for debug
logfile = 'runMD.log'
errfile = 'runMD.err'
logMD = open(logfile, 'w')
errMD = open(errfile, 'w')
cmd = ['bash', scriptExe]
try:
proc = sp.Popen(cmd, stdout=logMD, stderr=errMD)
while True:
if proc.poll() != None:
break
if os.path.exists(killSig) or killer.kill_now:
logging.info("%s Warm Killing runMD function")
jobHandler.killProcs(proc.pid)
if os.path.exists(killSig):
os.remove(killSig)
return (True, 'CANCELLED')
else:
return (True, 'MD READY')
simInfo['status'] = 'Done'
except Exception, e:
logging.error(e)
#if simDir.endswith('0-0'):
# msg='Simulation of ligand in solvent failed: %s'%e
# return (False, msg)
simInfo['status'] = 'Failed'
os.chdir(currDir)
else:
### REMOTE EXECUTION
# 1. Tar files
tarName = 'MDfiles.tar'
if os.path.exists(tarName):
os.remove(tarName)
os.chdir(simDir)
listfiles = glob('*')
tar = tarfile.open(tarName, 'w')
for fname in listfiles:
tar.add(fname)
tar.close()
# 2. copy on the remote dir
rwdir = os.path.join(remoteSettings['wdir'],
simDir.split(os.sep)[-3],
simDir.split(os.sep)[-1])
sshConn.putFile(tarName, rwdir)
# 3. submit
sshConn.execCmd('cd %s; tar xvf %s' % (rwdir, tarName))
success, (status, stdOut) = sshConn.execCmd(
'cd %s; %s %s' % (rwdir, remoteSettings['submit'], scriptExe))
simInfo['jobid'] = stdOut.rstrip()
simInfo['rdir'] = rwdir
if status == 0:
simInfo['status'] = 'Submitted'
else:
simInfo['status'] = 'Failed'
listResults.append(simInfo)
def trigger_stage_out(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_stage_out')
tmpLog.debug('start')
# loop over all files
allChecked = True
ErrMsg = 'These files failed to upload : '
zip_datasetName = 'harvester_stage_out.{0}'.format(str(uuid.uuid4()))
fileAttrs = jobspec.get_output_file_attributes()
for fileSpec in jobspec.outFiles:
# fileSpec.fileAttributes['transferID'] = None # synchronius transfer
# skip already done
tmpLog.debug('file: %s status: %s' %
(fileSpec.lfn, fileSpec.status))
if fileSpec.status in ['finished', 'failed']:
continue
fileSpec.pathConvention = self.pathConvention
fileSpec.objstoreID = self.objstoreID
# set destination RSE
if fileSpec.fileType in ['es_output', 'zip_output', 'output']:
dstRSE = self.dstRSE_Out
elif fileSpec.fileType == 'log':
dstRSE = self.dstRSE_Log
else:
errMsg = 'unsupported file type {0}'.format(fileSpec.fileType)
tmpLog.error(errMsg)
return (False, errMsg)
# skip if destination is None
if dstRSE is None:
continue
# get/set scope and dataset name
if fileSpec.fileType == 'log':
if fileSpec.lfn in fileAttrs:
scope = fileAttrs[fileSpec.lfn]['scope']
datasetName = fileAttrs[fileSpec.lfn]['dataset']
else:
lfnWithoutWorkerID = ".".join(fileSpec.lfn.split('.')[:-1])
scope = fileAttrs[lfnWithoutWorkerID]['scope']
datasetName = fileAttrs[lfnWithoutWorkerID]['dataset']
elif fileSpec.fileType != 'zip_output' and fileSpec.lfn in fileAttrs:
scope = fileAttrs[fileSpec.lfn]['scope']
datasetName = fileAttrs[fileSpec.lfn]['dataset']
else:
# use panda scope for zipped files
scope = self.scopeForTmp
datasetName = zip_datasetName
# for now mimic behaviour and code of pilot v2 rucio copy tool (rucio download) change when needed
executable_prefix = None
pfn_prefix = None
if self.objectstore_additions and dstRSE in self.objectstore_additions:
if 'storage_id' in self.objectstore_additions[dstRSE]:
fileSpec.objstoreID = self.objectstore_additions[dstRSE][
'storage_id']
if 'access_key' in self.objectstore_additions[dstRSE] and \
'secret_key' in self.objectstore_additions[dstRSE] and \
'is_secure' in self.objectstore_additions[dstRSE]:
executable_prefix = "export S3_ACCESS_KEY=%s; export S3_SECRET_KEY=%s; export S3_IS_SECURE=%s" % (
self.objectstore_additions[dstRSE]['access_key'],
self.objectstore_additions[dstRSE]['secret_key'],
self.objectstore_additions[dstRSE]['is_secure'])
if 'pfn_prefix' in self.objectstore_additions[dstRSE]:
pfn_prefix = self.objectstore_additions[dstRSE][
'pfn_prefix']
executable = ['/usr/bin/env', 'rucio', '-v', 'upload']
executable += ['--no-register']
if hasattr(self, 'lifetime'):
executable += ['--lifetime', ('%d' % self.lifetime)]
if fileSpec.fileAttributes is not None and 'guid' in fileSpec.fileAttributes:
executable += ['--guid', fileSpec.fileAttributes['guid']]
executable += ['--rse', dstRSE]
executable += ['--scope', scope]
if pfn_prefix:
executable += [
'--pfn %s' %
os.path.join(pfn_prefix, os.path.basename(fileSpec.path))
]
else:
executable += [('%s:%s' % (scope, datasetName))]
executable += [('%s' % fileSpec.path)]
tmpLog.debug('rucio upload command: {0} '.format(executable))
tmpLog.debug('rucio upload command (for human): %s ' %
' '.join(executable))
if executable_prefix:
cmd = executable_prefix + "; " + ' '.join(executable)
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True)
else:
process = subprocess.Popen(executable,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stdout, stderr = process.communicate()
fileSpec.attemptNr += 1
stdout = stdout.decode() + " attemptNr: %s" % fileSpec.attemptNr
tmpLog.debug("stdout: %s" % stdout)
tmpLog.debug("stderr: %s" % stderr)
if process.returncode == 0:
fileSpec.status = 'finished'
else:
# check what failed
file_exists = False
rucio_sessions_limit_error = False
for line in stdout.split('\n'):
if 'File name in specified scope already exists' in line:
file_exists = True
break
elif 'File already exists on RSE' in line:
# can skip if file exist on RSE since no register
tmpLog.warning(
'rucio skipped upload and returned stdout: %s' %
stdout)
file_exists = True
break
elif 'exceeded simultaneous SESSIONS_PER_USER limit' in line:
rucio_sessions_limit_error = True
if file_exists:
tmpLog.debug('file exists, marking transfer as finished')
fileSpec.status = 'finished'
elif rucio_sessions_limit_error:
# do nothing
tmpLog.warning(
'rucio returned error, will retry: stdout: %s' %
stdout)
# do not change fileSpec.status and Harvester will retry if this function returns False
allChecked = False
continue
else:
tmpLog.error('rucio upload failed with stdout: %s' %
stdout)
ErrMsg += '%s failed with rucio error stdout="%s"' % (
fileSpec.lfn, stdout)
allChecked = False
if fileSpec.attemptNr >= self.maxAttempts:
tmpLog.error(
'reached maxattempts: %s, marked it as failed' %
self.maxAttempts)
fileSpec.status = 'failed'
# force update
fileSpec.force_update('status')
tmpLog.debug('file: %s status: %s' %
(fileSpec.lfn, fileSpec.status))
# return
tmpLog.debug('done')
if allChecked:
return True, ''
else:
return False, ErrMsg
def copy_data(source='',target=''):
cmd = 'cp '+ mkpath('data/' + source) + ' ' + mkpath(target)
p = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
return p.communicate()
username = re.sub('/CN=Robot:[^/]+', '', username)
pat = re.compile('.*/CN=([^\/]+)/CN=([^\/]+)')
mat = pat.match(username)
if mat:
username = mat.group(2)
else:
username = username.replace('/CN=', '')
if username.lower().find('/email') > 0:
username = username[:username.lower().find('/email')]
pat = re.compile('.*(limited.*proxy).*')
mat = pat.match(username)
if mat:
username = mat.group(1)
username = username.replace('(', '')
username = username.replace(')', '')
username = username.replace("'", '')
return username
except:
return id
certFile = sys.argv[1]
com = "openssl x509 -noout -subject -in"
p = subprocess.Popen(com.split() + [certFile], stdout=subprocess.PIPE)
out, err = p.communicate()
out = re.sub('^subject=', '', out)
out = out.strip()
print('DN: "{0}"'.format(out))
print('extracted: "{0}"'.format(clean_user_id(out)))
def run(cmd,
cwd=Settings.TEST_RUN_HOME,
wait=True,
timeout=600,
fail_safe=False,
register=True,
log_level=logging.DEBUG):
# Init result values
time_string = datetime.now().strftime('%Y_%m_%d_%H_%M_%S_%f')
log_file = os.path.join(Settings.TEST_OUT_LOGS,
'command_{0}.txt'.format(time_string))
complete = False
duration = None
output = ''
# Ensure logs folder exists
dir_path = os.path.dirname(os.path.realpath(log_file))
Folder.create(dir_path)
# Command settings
if not wait:
# Redirect output to file
File.write(path=log_file, text=cmd + os.linesep + '====>' + os.linesep)
cmd = cmd + ' >> ' + log_file + ' 2>&1 &'
# Log command that will be executed:
Log.log(level=log_level, msg='Execute command: ' + cmd)
Log.log(level=logging.DEBUG, msg='CWD: ' + cwd)
# Execute command:
if wait:
start = time.time()
with open(log_file, mode='w') as log:
if Settings.HOST_OS == OSType.WINDOWS:
process = subprocess.Popen(cmd,
cwd=cwd,
shell=True,
stdout=log,
stderr=log)
else:
process = subprocess.Popen(cmd,
cwd=cwd,
shell=True,
stdout=subprocess.PIPE,
stderr=log)
# Wait until command complete
try:
process.wait(timeout=timeout)
complete = True
out, err = process.communicate()
if out is not None:
if Settings.PYTHON_VERSION < 3:
output = str(out.decode('utf8').encode('utf8')).strip()
else:
output = out.decode("utf-8").strip()
except subprocess.TimeoutExpired:
process.kill()
if fail_safe:
Log.error('Command "{0}" timeout after {1} seconds.'.format(
cmd, timeout))
else:
raise
# Append stderr to output
stderr = File.read(path=log_file)
if stderr:
output = output + os.linesep + File.read(path=log_file)
# noinspection PyBroadException
try:
File.delete(path=log_file)
except Exception:
Log.debug('Failed to clean log file: {0}'.format(log_file))
log_file = None
end = time.time()
duration = end - start
else:
process = psutil.Popen(cmd,
cwd=cwd,
shell=True,
stdin=None,
stdout=None,
stderr=None,
close_fds=True)
# Get result
pid = process.pid
exit_code = process.returncode
# Log output of the process
if wait:
Log.log(level=log_level,
msg='OUTPUT: ' + os.linesep + output + os.linesep)
else:
Log.log(level=log_level,
msg='OUTPUT REDIRECTED: ' + log_file + os.linesep)
# Construct result
result = ProcessInfo(cmd=cmd,
pid=pid,
exit_code=exit_code,
output=output,
log_file=log_file,
complete=complete,
duration=duration)
# Register in TestContext
if psutil.pid_exists(result.pid) and register:
TestContext.STARTED_PROCESSES.append(result)
# Return the result
return result
def unmount_ramdisk(dir_path):
subprocess.Popen(['sudo', 'umount', dir_path]).wait()
def __doBackgroundDispatch(self, batch):
if self.__getStatus(batch) == LocalDispatcher.Job.Status.Complete:
return True
for upstreamBatch in batch.preTasks():
if not self.__doBackgroundDispatch(upstreamBatch):
return False
if batch.blindData().get("killed"):
self.__reportKilled(batch)
return False
if not batch.plug():
self.__reportCompleted(batch)
return True
if len(batch.frames()) == 0:
# This case occurs for nodes like TaskList and TaskContextProcessors,
# because they don't do anything in execute (they have empty hashes).
# Their batches exist only to depend on upstream batches. We don't need
# to do any work here, but we still signal completion for the task to
# provide progress feedback to the user.
self.__setStatus(batch, LocalDispatcher.Job.Status.Complete)
IECore.msg(IECore.MessageHandler.Level.Info,
self.__messageTitle,
"Finished " + batch.blindData()["nodeName"].value)
return True
taskContext = batch.context()
frames = str(
IECore.frameListFromList([int(x) for x in batch.frames()]))
args = [
"gaffer",
"execute",
"-script",
self.__scriptFile,
"-nodes",
batch.blindData()["nodeName"].value,
"-frames",
frames,
]
if self.__ignoreScriptLoadErrors:
args.append("-ignoreScriptLoadErrors")
contextArgs = []
for entry in [
k for k in taskContext.keys()
if k != "frame" and not k.startswith("ui:")
]:
if entry not in self.__context.keys(
) or taskContext[entry] != self.__context[entry]:
contextArgs.extend(["-" + entry, repr(taskContext[entry])])
if contextArgs:
args.extend(["-context"] + contextArgs)
self.__setStatus(batch, LocalDispatcher.Job.Status.Running)
IECore.msg(IECore.MessageHandler.Level.Info, self.__messageTitle,
" ".join(args))
process = subprocess.Popen(args, start_new_session=True)
batch.blindData()["pid"] = IECore.IntData(process.pid)
while process.poll() is None:
if batch.blindData().get("killed"):
os.killpg(process.pid, signal.SIGTERM)
self.__reportKilled(batch)
return False
time.sleep(0.01)
if process.returncode:
self.__reportFailed(batch)
return False
self.__setStatus(batch, LocalDispatcher.Job.Status.Complete)
return True
def __init__(self, resource_id, log=""):
# required in the local connection mode
import Pyro.core
from Pyro.errors import ConnectionClosedError
try:
import subprocess32 as subprocess
except ImportError:
import subprocess
login = getpass.getuser()
pyro_objet_name = "workflow_engine_" + login
# run the workflow engine process and get back the
# workflow_engine and ConnectionChecker URIs
# command = "python -m cProfile -o /home/soizic/profile/profile /home/soizic/svn/brainvisa/source/soma/soma-workflow/trunk/python/soma_workflow/start_workflow_engine.py %s %s %s" %(
# resource_id,
# pyro_objet_name,
# log)
command = "python -m soma_workflow.start_workflow_engine %s %s %s" % (
resource_id, pyro_objet_name, log)
engine_process = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
line = engine_process.stdout.readline()
stdout_content = line
while line and line.split()[0] != pyro_objet_name:
line = engine_process.stdout.readline()
stdout_content = stdout_content + "\n" + line
if not line: # A problem occured while starting the engine.
line = engine_process.stderr.readline()
stderr_content = line
while line:
line = engine_process.stderr.readline()
stderr_content = stderr_content + "\n" + line
raise ConnectionError(
"A problem occured while starting the engine "
"process on the local machine. \n"
"**More details:**\n"
"**Start engine process command line:** \n"
"\n" + command + "\n\n"
"**Engine process standard output:** \n"
"\n" + stdout_content + "**Engine process standard error:** \n"
"\n" + stderr_content)
workflow_engine_uri = line.split()[1]
line = engine_process.stdout.readline()
stdout_content = stdout_content + "\n" + line
while line and line.split()[0] != "connection_checker":
line = engine_process.stdout.readline()
stdout_content = stdout_content + "\n" + line
if not line: # A problem occured while starting the engine.
line = engine_process.stderr.readline()
stderr_content = line
while line:
line = engine_process.stderr.readline()
stderr_content = stderr_content + "\n" + line
raise ConnectionError(
"A problem occured while starting the engine "
"process on the local machine. \n"
"**More details:**\n"
"**Start engine process command line:** \n"
"\n" + command + "\n\n"
"**Engine process standard output:** \n"
"\n" + stdout_content + "**Engine process standard error:** \n"
"\n" + stderr_content)
connection_checker_uri = line.split()[1]
line = engine_process.stdout.readline()
stdout_content = stdout_content + "\n" + line
while line and line.split()[0] != "configuration":
line = engine_process.stdout.readline()
stdout_content = stdout_content + "\n" + line
if not line: # A problem occured while starting the engine.
line = engine_process.stderr.readline()
stderr_content = line
while line:
line = engine_process.stderr.readline()
stderr_content = stderr_content + "\n" + line
raise ConnectionError(
"A problem occured while starting the engine "
"process on the local machine. \n"
"**More details:**\n"
"**Start engine process command line:** \n"
"\n" + command + "\n\n"
"**Engine process standard output:** \n"
"\n" + stdout_content + "**Engine process standard error:** \n"
"\n" + stderr_content)
configuration_uri = line.split()[1]
print("workflow_engine_uri: " + workflow_engine_uri)
print("connection_checker_uri: " + connection_checker_uri)
print("configuration_uri: " + configuration_uri)
# create the proxies #
self.workflow_engine = Pyro.core.getProxyForURI(workflow_engine_uri)
connection_checker = Pyro.core.getAttrProxyForURI(
connection_checker_uri)
self.configuration = Pyro.core.getAttrProxyForURI(configuration_uri)
# create the connection holder objet for #
# a clean disconnection in any case #
self.__connection_holder = ConnectionHolder(connection_checker)
self.__connection_holder.start()
def start_test(mr):
tests_passed = True
error_lines = []
print 'Starting testing for MR %s ...' % mr['source_branch']
# Add remote of user
resp = requests.get(url='https://gitlab.cern.ch/api/v4/projects/%s' %
str(mr['source_project_id']),
params={'private_token': private_token})
proj = json.loads(resp.text)
commands.getstatusoutput(
'git remote add %s %s' %
(proj['namespace']['name'], proj['http_url_to_repo']))
# Fetch all
print ' git fetch --all --prune'
s, o = commands.getstatusoutput('git fetch --all --prune')
if s != 0:
print 'Error while fetching all: %s' % o
sys.exit(-1)
# Rebase master/next
print ' git rebase origin/next next'
s, o = commands.getstatusoutput('git rebase origin/next next')
if s != 0:
print 'Error while rebaseing next: %s' % o
sys.exit(-1)
print ' git rebase origin/master master'
s, o = commands.getstatusoutput('git rebase origin/master master')
if s != 0:
print 'Error while rebaseing master: %s' % o
sys.exit(-1)
# Check for Cross Merges
if mr['source_branch'].lower().startswith('patch'):
print ' Checking for cross-merges:'
commits = commands.getoutput(
'git log master..remotes/%s/%s | grep ^commit' %
(proj['namespace']['name'], mr['source_branch']))
for commit in commits.splitlines():
commit = commit.partition(' ')[2]
if commands.getstatusoutput(
'git branch --contains %s | grep next' % commit)[0] == 0:
print ' Found cross-merge problem with commit %s' % commit
tests_passed = False
error_lines.append('##### CROSS-MERGE TESTS:\n')
error_lines.append('```\n')
error_lines.append(
'This patch is suspicious. It looks like there are feature-commits pulled into the master branch!\n'
)
error_lines.append('```\n')
break
# Checkout the branch to test
print ' git checkout remotes/%s/%s' % (proj['namespace']['name'],
mr['source_branch'])
if commands.getstatusoutput(
'git checkout remotes/%s/%s' %
(proj['namespace']['name'], mr['source_branch']))[0] != 0:
print 'Error while checking out branch'
sys.exit(-1)
# ACTUAL TESTS START HERE #
print ' Installing .venv'
# Try re-install .venv (This validates the python packages
commands.getstatusoutput('rm -R --force .venv/') # Remove old .venv
if commands.getstatusoutput('python tools/install_venv.py')[0] != 0:
print 'Error while installing .venv'
tests_passed = False
error_lines.append('##### INSTALLING .VENV FAILED\n')
# Restart apache and memcached
print ' /sbin/service memcached restart'
if commands.getstatusoutput('/sbin/service memcached restart')[0] != 0:
print 'Error while restarting memcached'
sys.exit(-1)
# Restart apache and memcached
print ' service httpd restart'
if commands.getstatusoutput('/sbin/service httpd restart')[0] != 0:
print 'Error while restarting httpd'
sys.exit(-1)
changed_files = commands.getoutput(
'git diff-tree --no-commit-id --name-only -r --diff-filter=AMRT HEAD | grep .py | grep -v .py.mako'
).splitlines()
command = """
cd %s; source .venv/bin/activate;
cd .venv/lib/python2.7/site-packages/;
ln -s %s/lib/rucio/;
cd %s;
pip install cx_oracle==5.2.1;
python ../purge_bin.py;
find lib -iname "*.pyc" | xargs rm; rm -rf /tmp/.rucio_*/;
tools/reset_database.py;
tools/sync_rses.py;
tools/sync_meta.py;
tools/bootstrap_tests.py;
nosetests -v --logging-filter=-sqlalchemy,-requests,-rucio.client.baseclient --exclude=.*test_rse_protocol_.* --exclude=test_alembic --exclude=test_rucio_cache --exclude=test_rucio_server --exclude=test_dq2* --exclude=test_objectstore > /tmp/rucio_nose.txt 2> /tmp/rucio_nose.txt;
tools/reset_database.py;
nosetests -v lib/rucio/tests/test_alembic.py > /tmp/rucio_alembic.txt 2> /tmp/rucio_alembic.txt;
flake8 --exclude=*.cfg bin/* lib/ tools/*.py > /tmp/rucio_flake8.txt;
pylint %s > /tmp/rucio_pylint.txt;
python ../purge_bin.py;
""" % (root_git_dir, root_git_dir, root_git_dir, ' '.join(changed_files)
) # NOQA
print ' %s' % command
if tests_passed:
proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
try:
outs, errs = proc.communicate(timeout=60 * 20)
except TimeoutExpired:
print 'Timeout reached, killing test'
proc.kill()
outs, errs = proc.communicate()
os.remove('/tmp/rucio_test.pid')
sys.exit(-1)
with open('/tmp/rucio_nose.txt', 'r') as f:
lines = f.readlines()
if lines[-1] != 'OK\n':
tests_passed = False
error_lines.append('##### UNIT TESTS:\n')
error_lines.append('```\n')
error_lines.extend(lines)
error_lines.append('```\n')
with open('/tmp/rucio_alembic.txt', 'r') as f:
lines = f.readlines()
if lines[-1] != 'OK\n':
tests_passed = False
error_lines.append('##### ALEMBIC:\n')
error_lines.append('```\n')
error_lines.extend(lines)
error_lines.append('```\n')
if os.stat('/tmp/rucio_flake8.txt').st_size != 0:
with open('/tmp/rucio_flake8.txt', 'r') as f:
lines = f.readlines()
tests_passed = False
error_lines.append('##### FLAKE8:\n')
error_lines.append('```\n')
error_lines.extend(lines)
error_lines.append('```\n')
# PYLINT
if os.stat('/tmp/rucio_pylint.txt').st_size != 0:
with open('/tmp/rucio_pylint.txt', 'r') as f:
lines = f.readlines()
pylint_passed = True
# Check if there is an Error in PYLINT
for line in lines:
if line.startswith('E:'):
tests_passed = False
pylint_passed = False
if not pylint_passed:
error_lines.append('##### PYLINT\n')
error_lines.append('```\n')
for line in lines:
if line.startswith('E:'):
error_lines.append(line)
error_lines.append('```\n')
error_lines.append(lines[-2])
if tests_passed:
error_lines.insert(0, '#### BUILD-BOT TEST RESULT: OK\n\n')
else:
error_lines.insert(0, '#### BUILD-BOT TEST RESULT: FAIL\n\n')
commands.getstatusoutput('/sbin/service httpd stop')
update_merg_request(mr=mr, test_result=tests_passed, comment=error_lines)
# Checkout original master
print ' git checkout master'
if commands.getstatusoutput('git checkout master')[0] != 0:
print 'Error while checking out master'
sys.exit(-1)
def execute(*args, **kwargs):
logger = logging.getLogger(logger_name)
command = 'simpleflow.execute' # name of a module.
sys.stdout.flush()
sys.stderr.flush()
result_str = None # useless
context = kwargs.pop('context', {})
with tempfile.TemporaryFile() as result_fd, tempfile.TemporaryFile(
) as error_fd:
dup_result_fd = os.dup(result_fd.fileno()) # remove FD_CLOEXEC
dup_error_fd = os.dup(error_fd.fileno()) # remove FD_CLOEXEC
# print('error_fd: {}'.format(dup_error_fd))
full_command = [
interpreter,
'-m',
command, # execute module a script.
get_name(func),
format_arguments_json(*args, **kwargs),
'--logger-name={}'.format(logger_name),
'--result-fd={}'.format(dup_result_fd),
'--error-fd={}'.format(dup_error_fd),
'--context={}'.format(json_dumps(context)),
]
if kill_children:
full_command.append('--kill-children')
if compat.PY2: # close_fds doesn't work with python2 (using its C _posixsubprocess helper)
close_fds = False
pass_fds = ()
else:
close_fds = True
pass_fds = (dup_result_fd, dup_error_fd)
process = subprocess.Popen(
full_command,
bufsize=-1,
close_fds=close_fds,
pass_fds=pass_fds,
)
rc = wait_subprocess(process,
timeout=timeout,
command_info=full_command)
os.close(dup_result_fd)
os.close(dup_error_fd)
if rc:
error_fd.seek(0)
err_output = error_fd.read()
if err_output:
if not compat.PY2:
err_output = err_output.decode('utf-8',
errors='replace')
raise ExecutionError(err_output)
result_fd.seek(0)
result_str = result_fd.read()
if not result_str:
return None
try:
if not compat.PY2:
result_str = result_str.decode('utf-8', errors='replace')
result = format.decode(result_str)
return result
except BaseException as ex:
logger.exception('Exception in python.execute: {} {}'.format(
ex.__class__.__name__, ex))
logger.warning('%r', result_str)
def run(cmd, asynchronous=False):
if asynchronous:
return subprocess.Popen(cmd, shell=True)
return subprocess.check_output(cmd, shell=True)
#メッセージを送信
ws.send(commands["register"])
while True:
#受信したメッセージを表示
recv_str = ws.recv()
print recv_str
try:
logging.info("ws.recv() = {}".format(recv_str))
ms = json.loads(recv_str)
if ms["order"] == "exec_bash":
print("exec {}".format(ms["cmd_str"]))
logging.info("exec {}".format(ms["cmd_str"]))
p = subprocess.Popen(ms["cmd_str"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
std_out, std_err = p.communicate(None, timeout=60)
result = std_out.strip()
error = std_err.strip()
logging.info("result = {}".format(result))
logging.info("error = {}".format(error))
result_json = json.dumps({
"order": "response",
"result": result,
"error": error
})
logging.info("result_json = {}".format(result_json))
def run(cmd,
print_error=True,
asynchronous=False,
stdin=False,
stderr=subprocess.STDOUT,
outfile=None,
env_vars=None,
inherit_cwd=False,
inherit_env=True,
tty=False):
# don't use subprocess module in Python 2 as it is not thread-safe
# http://stackoverflow.com/questions/21194380/is-subprocess-popen-not-thread-safe
# TODO: should be removed, now that Python 2 has reached its EOL
if six.PY2:
import subprocess32 as subprocess
else:
import subprocess
env_dict = os.environ.copy() if inherit_env else {}
if env_vars:
env_dict.update(env_vars)
env_dict = dict([(k, to_str(str(v))) for k, v in env_dict.items()])
if tty:
asynchronous = True
stdin = True
try:
cwd = os.getcwd() if inherit_cwd else None
if not asynchronous:
if stdin:
return subprocess.check_output(cmd,
shell=True,
stderr=stderr,
env=env_dict,
stdin=subprocess.PIPE,
cwd=cwd)
output = subprocess.check_output(cmd,
shell=True,
stderr=stderr,
env=env_dict,
cwd=cwd)
return output.decode(config.DEFAULT_ENCODING)
# subprocess.Popen is not thread-safe, hence use a mutex here.. (TODO: mutex still needed?)
with mutex_popen:
stdin_arg = subprocess.PIPE if stdin else None
stdout_arg = open(outfile, 'ab') if isinstance(
outfile, six.string_types) else outfile
stderr_arg = stderr
if tty:
# Note: leave the "pty" import here (not supported in Windows)
import pty
master_fd, slave_fd = pty.openpty()
stdin_arg = slave_fd
stdout_arg = stderr_arg = None
# start the actual sub process
kwargs = {}
if is_linux() or is_mac_os():
kwargs['preexec_fn'] = os.setsid
process = subprocess.Popen(cmd,
shell=True,
stdin=stdin_arg,
bufsize=-1,
stderr=stderr_arg,
stdout=stdout_arg,
env=env_dict,
cwd=cwd,
**kwargs)
if tty:
# based on: https://stackoverflow.com/questions/41542960
def pipe_streams(*args):
while process.poll() is None:
r, w, e = select.select([sys.stdin, master_fd], [], [])
if sys.stdin in r:
d = os.read(sys.stdin.fileno(), 10240)
os.write(master_fd, d)
elif master_fd in r:
o = os.read(master_fd, 10240)
if o:
os.write(sys.stdout.fileno(), o)
FuncThread(pipe_streams).start()
return process
except subprocess.CalledProcessError as e:
if print_error:
print("ERROR: '%s': exit code %s; output: %s" %
(cmd, e.returncode, e.output))
sys.stdout.flush()
raise e
def do_capture(status_code, the_record, base_url, model='capture'):
"""
Create a screenshot, text scrape, from a provided html file.
This depends on phantomjs and an associated javascript file to perform the captures.
In the event an error occurs, an exception is raised and handled by the celery task
or the controller that called this method.
"""
# Make sure the the_record
db.session.add(the_record)
# If the capture is for static content, use a differnet PhantomJS config file
if model == 'static':
capture_name = the_record.filename
service_args = [
app.config['PHANTOMJS'], '--ssl-protocol=any',
'--ignore-ssl-errors=yes',
os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +
'/assets/static.js', app.config['LOCAL_STORAGE_FOLDER'],
capture_name
]
content_to_parse = os.path.join(app.config['LOCAL_STORAGE_FOLDER'],
capture_name)
else:
capture_name = grab_domain(the_record.url) + '_' + str(the_record.id)
service_args = [
app.config['PHANTOMJS'], '--ssl-protocol=any',
'--ignore-ssl-errors=yes',
os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +
'/assets/capture.js', the_record.url,
os.path.join(app.config['LOCAL_STORAGE_FOLDER'], capture_name)
]
content_to_parse = os.path.join(app.config['LOCAL_STORAGE_FOLDER'],
capture_name + '.html')
# Using subprocess32 backport, call phantom and if process hangs kill it
pid = subprocess32.Popen(service_args, stdout=PIPE, stderr=PIPE)
try:
stdout, stderr = pid.communicate(timeout=35)
except subprocess32.TimeoutExpired:
pid.kill()
stdout, stderr = pid.communicate()
app.logger.error('PhantomJS Static Capture timeout')
raise Exception('PhantomJS Static Capture timeout')
# If the subprocess has an error, raise an exception
if stderr or stdout:
raise Exception(stderr)
# Strip tags and parse out all text
ignore_tags = ('script', 'noscript', 'style')
with open(content_to_parse, 'r') as content_file:
content = content_file.read()
cleaner = clean.Cleaner()
content = cleaner.clean_html(content)
doc = LH.fromstring(content)
output = ""
for elt in doc.iterdescendants():
if elt.tag in ignore_tags:
continue
text = elt.text or ''
tail = elt.tail or ''
wordz = " ".join((text, tail)).strip('\t')
if wordz and len(wordz) >= 2 and not re.match("^[ \t\n]*$", wordz):
output += wordz.encode('utf-8')
# Since the filename format is different for static captures, update the filename
# This will ensure the URLs are pointing to the correct resources
if model == 'static':
capture_name = capture_name.split('.')[0]
# Wite our html text that was parsed into our capture folder
parsed_text = open(
os.path.join(app.config['LOCAL_STORAGE_FOLDER'],
capture_name + '.txt'), 'wb')
parsed_text.write(output)
# Update the sketch record with the local URLs for the sketch, scrape, and html captures
the_record.sketch_url = base_url + '/files/' + capture_name + '.png'
the_record.scrape_url = base_url + '/files/' + capture_name + '.txt'
the_record.html_url = base_url + '/files/' + capture_name + '.html'
# Create a dict that contains what files may need to be written to S3
files_to_write = defaultdict(list)
files_to_write['sketch'] = capture_name + '.png'
files_to_write['scrape'] = capture_name + '.txt'
files_to_write['html'] = capture_name + '.html'
# If we are not writing to S3, update the capture_status that we are completed.
if not app.config['USE_S3']:
the_record.job_status = "COMPLETED"
the_record.capture_status = "LOCAL_CAPTURES_CREATED"
else:
the_record.capture_status = "LOCAL_CAPTURES_CREATED"
db.session.commit()
return files_to_write
def main():
parser = argparse.ArgumentParser()
parser.add_argument('dataset_dir', help='Dataset directory')
args = parser.parse_args()
dataset_dir = args.dataset_dir
if not osp.exists(dataset_dir):
print('Please install JSKV1 dataset to: %s' % dataset_dir)
quit(1)
PKG_DIR = rospkg.RosPack().get_path('jsk_arc2017_common')
objlist_file = osp.join(PKG_DIR, 'data/others/object_list_5x8.jpg')
objlist = cv2.imread(objlist_file)
scale = 1000. / max(objlist.shape[:2])
objlist = cv2.resize(objlist, dsize=None, fx=scale, fy=scale)
cmap = labelme.utils.labelcolormap(41)
for stamp_dir in sorted(os.listdir(dataset_dir)):
stamp = datetime.datetime.fromtimestamp(int(stamp_dir) / 1e9)
stamp_dir = osp.join(dataset_dir, stamp_dir)
if not osp.isdir(stamp_dir):
continue
json_file = osp.join(stamp_dir, 'label.json')
img_file = osp.join(stamp_dir, 'image.jpg')
lbl_file = osp.join(stamp_dir, 'label.npz')
lbl_viz_file = osp.join(stamp_dir, 'label_viz.jpg')
lock_file = osp.join(stamp_dir, 'annotation.lock')
if osp.exists(lock_file):
continue
if not osp.exists(json_file):
open(lock_file, 'w')
print('%s: %s' % (stamp.isoformat(), stamp_dir))
cmd = 'labelme %s -O %s' % (img_file, json_file)
output = subprocess.Popen(shlex.split(cmd))
returncode = None
while returncode is None:
try:
output.communicate(timeout=1)
returncode = output.returncode
except subprocess.TimeoutExpired:
pass
except KeyboardInterrupt:
break
cv2.imshow('object list', objlist)
cv2.waitKey(50)
os.remove(lock_file)
if returncode != 0:
output.kill()
break
if not (osp.exists(lbl_file) and osp.exists(lbl_viz_file)):
img = skimage.io.imread(img_file)
lbl = json_to_label(json_file)
lbl_viz = skimage.color.label2rgb(lbl,
img,
bg_label=0,
colors=cmap[1:])
np.savez_compressed(lbl_file, lbl)
skimage.io.imsave(lbl_viz_file, lbl_viz)
def trigger_preparation(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_preparation')
tmpLog.debug('start')
# loop over all inputs
inFileInfo = jobspec.get_input_file_attributes()
gucInput = None
for tmpFileSpec in jobspec.inFiles:
# construct source and destination paths
srcPath = mover_utils.construct_file_path(
self.srcBasePath, inFileInfo[tmpFileSpec.lfn]['scope'],
tmpFileSpec.lfn)
dstPath = mover_utils.construct_file_path(
self.dstBasePath, inFileInfo[tmpFileSpec.lfn]['scope'],
tmpFileSpec.lfn)
# local access path
accPath = mover_utils.construct_file_path(
self.localBasePath, inFileInfo[tmpFileSpec.lfn]['scope'],
tmpFileSpec.lfn)
if self.checkLocalPath:
# check if already exits
if os.path.exists(accPath):
# calculate checksum
checksum = core_utils.calc_adler32(accPath)
checksum = 'ad:{0}'.format(checksum)
if checksum == inFileInfo[tmpFileSpec.lfn]['checksum']:
continue
# make directories if needed
if not os.path.isdir(os.path.dirname(accPath)):
os.makedirs(os.path.dirname(accPath))
# make input for globus-url-copy
if gucInput is None:
gucInput = tempfile.NamedTemporaryFile(mode='w',
delete=False,
suffix='_guc_in.tmp')
gucInput.write("{0} {1}\n".format(srcPath, dstPath))
tmpFileSpec.attemptNr += 1
# nothing to transfer
if gucInput is None:
tmpLog.debug('done with no transfers')
return True, ''
# transfer
tmpLog.debug('execute globus-url-copy')
gucInput.close()
args = ['globus-url-copy', '-f', gucInput.name, '-cd']
if self.gulOpts is not None:
args += self.gulOpts.split()
try:
tmpLog.debug('execute: ' + ' '.join(args))
p = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
stdout, stderr = p.communicate(timeout=self.timeout)
except subprocess.TimeoutExpired:
p.kill()
stdout, stderr = p.communicate()
tmpLog.warning('command timeout')
return_code = p.returncode
if stdout is not None:
if not isinstance(stdout, str):
stdout = stdout.decode()
stdout = stdout.replace('\n', ' ')
if stderr is not None:
if not isinstance(stderr, str):
stderr = stderr.decode()
stderr = stderr.replace('\n', ' ')
tmpLog.debug("stdout: %s" % stdout)
tmpLog.debug("stderr: %s" % stderr)
except Exception:
core_utils.dump_error_message(tmpLog)
return_code = 1
os.remove(gucInput.name)
if return_code == 0:
tmpLog.debug('succeeded')
return True, ''
else:
errMsg = 'failed with {0}'.format(return_code)
tmpLog.error(errMsg)
# check attemptNr
for tmpFileSpec in jobspec.inFiles:
if tmpFileSpec.attemptNr >= self.maxAttempts:
errMsg = 'gave up due to max attempts'
tmpLog.error(errMsg)
return (False, errMsg)
return None, errMsg
try:
import pty # pylint: disable=unused-import
except ImportError:
sys.exit('Unsupported operating system!')
# version string
__version__ = '0.3.3.post1'
# install requires
try:
subprocess.check_call(['ps', 'axo', 'pid=,stat='],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
with subprocess.Popen(['yes'],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL) as pipe:
subprocess.check_call(['pgrep', '-P', str(os.getpid())],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
pipe.terminate()
pipe.kill()
except subprocess.CalledProcessError:
requirements = ['psutil']
else:
requirements = None
# README
with open('README.rst', 'rb') as file:
long_desc = file.read().decode('utf-8')
from easyprocess import EasyProcess
import subprocess32
timeout = 3
print "#### use subprocess ####"
# use subprocess
# timeout can not set
# stdout is display in realtime
command = ['ping', 'localhost', '-t', '5']
child = subprocess.Popen(command)
# s = child.communicate(timeout=timeout)[0] timeout can not set
s = child.communicate()[0]
print "#### use easyprocess ####"
# use easyprocess
# timeout can set
# stdout is display when finished
s = EasyProcess('ping localhost -t 5').call(timeout=timeout).stdout
print(s)
print "#### use subprocess32 ####"
# use subprocess32
# timeout can set
# stdout is display in realtime
command = ['ping', 'localhost', '-t', '5']
child = subprocess32.Popen(command)
try:
s = child.communicate(timeout=timeout)[0]
except subprocess32.TimeoutExpired:
print "timeout"
def plot_raja_mmult(cali_file,
baseFileName,
cali_query,
legend=True,
title=True):
ANNOTATIONS = ':'.join([
'iteration', 'loop', 'size', 'depth', 'perm', 'mode',
'time.inclusive.duration'
])
CALI_QUERY = _get_cali_query(cali_query, ANNOTATIONS, cali_file)
p = subprocess.Popen(CALI_QUERY, stdout=subprocess.PIPE)
out, err = p.communicate()
data = {}
for line in out.split():
line = dict(map(lambda x: x.split('='), line.split(',')))
if len(line) != 6:
continue
depth = line['depth']
mode = line['mode']
perm = line['perm']
size = int(line['size'])
time = int(line['time.inclusive.duration'])
if depth and perm and mode:
if depth not in data:
data[depth] = {}
if perm not in data[depth]:
data[depth][perm] = {}
if mode not in data[depth][perm]:
data[depth][perm][mode] = {}
if size not in data[depth][perm][mode]:
data[depth][perm][mode][size] = []
data[depth][perm][mode][size].append(time)
colors = ['r', 'b', 'k', 'g', 'c', 'm']
markers = ('o', 'v', '8', 's', 'h', '*')
namelist = (('OMP', 'RAJA w/ OpenMP'), ('Serial', 'RAJA Serial'),
('Agency', 'RAJA w/ Agency'), ('AgencyOMP',
'RAJA w/ Agency w/ OpenMP'),
('control', 'Control Serial'), ('rawOMP', 'Control OpenMP'))
names = {
key: (name, colors[i], markers[i])
for i, (key, name) in enumerate(namelist)
}
for depth, perm_dict in data.iteritems():
for perm, mode_dict in perm_dict.iteritems():
fig = plt.figure()
legendNames = []
for i, (mode, datum) in enumerate(mode_dict.iteritems()):
sizes = sorted(datum.keys())
xerr = [0] * len(sizes)
means = [mean(datum[size]) for size in sizes]
sems = [sem(datum[size]) for size in sizes]
name, color, marker = names[mode]
legendNames.append(name)
plt.errorbar(sizes,
means,
xerr=xerr,
yerr=sems,
marker=marker,
color=color)
if legend:
plt.legend(legendNames, loc='best', fontsize=12)
if title:
plt.title('Matrix Multiplication Performance', fontsize=22)
plt.xlabel('Matrix Size', fontsize=16)
plt.ylabel('Time Taken (ms)', fontsize=16)
plt.yscale('log')
plt.grid(b=True, which='major', color='k', linestyle='dotted')
filename = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'figs',
"{}_PERM_{}_Depth_{}.pdf".format(baseFileName, perm, depth))
plt.savefig(filename)
def word_to_pdf(in_file,
in_format,
out_file,
pdfa=False,
password=None,
update_refs=False,
tagged=False):
tempdir = tempfile.mkdtemp()
from_file = os.path.join(tempdir, "file." + in_format)
to_file = os.path.join(tempdir, "file.pdf")
shutil.copyfile(in_file, from_file)
tries = 0
if pdfa:
method = 'pdfa'
elif tagged:
method = 'tagged'
else:
method = 'default'
while tries < 5:
use_libreoffice = True
if update_refs:
if daconfig.get('convertapi secret', None) is not None:
update_references(from_file)
try:
convertapi_to_pdf(from_file, to_file)
result = 0
except:
logmessage("Call to convertapi failed")
result = 1
use_libreoffice = False
else:
subprocess_arguments = [
LIBREOFFICE_PATH, '--headless', '--invisible',
'macro:///Standard.Module1.ConvertToPdf(' + from_file +
',' + to_file + ',True,' + method + ')'
]
elif daconfig.get('convertapi secret', None) is not None:
try:
convertapi_to_pdf(from_file, to_file)
result = 0
except:
logmessage("Call to convertapi failed")
result = 1
use_libreoffice = False
else:
if method == 'default':
subprocess_arguments = [
LIBREOFFICE_PATH, '--headless', '--convert-to', 'pdf',
from_file
]
else:
subprocess_arguments = [
LIBREOFFICE_PATH, '--headless', '--invisible',
'macro:///Standard.Module1.ConvertToPdf(' + from_file +
',' + to_file + ',False,' + method + ')'
]
if use_libreoffice:
initialize_libreoffice()
#logmessage("Trying libreoffice with " + repr(subprocess_arguments))
p = subprocess.Popen(subprocess_arguments, cwd=tempdir)
result = p.wait()
if os.path.isfile(to_file):
break
result = 1
tries += 1
time.sleep(2 + tries * random.random())
if use_libreoffice:
logmessage("Retrying libreoffice with " +
repr(subprocess_arguments))
else:
logmessage("Retrying convertapi")
continue
if result == 0:
if password:
pdf_encrypt(to_file, password)
shutil.copyfile(to_file, out_file)
if tempdir is not None:
shutil.rmtree(tempdir)
if result != 0:
return False
return True
def plot_raja_reducer(cali_file, filename, cali_query):
ANNOTATIONS = ':'.join([
'iteration', 'loop', 'size', 'initialization', 'baseline',
'RajaSerial', 'OMP', 'AgencyParallel', 'AgencySerial', 'RawAgency',
'RawOMP', 'time.inclusive.duration'
])
CALI_QUERY = _get_cali_query(cali_query, ANNOTATIONS, cali_file)
p = subprocess.Popen(CALI_QUERY, stdout=subprocess.PIPE)
out, err = p.communicate()
data = {
'init': {},
'baseline': {},
'OMP': {},
'RajaSerial': {},
'AgencyParallel': {},
'AgencySerial': {},
'RawAgency': {},
'RawOMP': {}
}
for line in out.split():
line = dict(map(lambda x: x.split('='), line.split(',')))
loop = None
time = None
mode = None
size = None
for key, value in line.iteritems():
if key == 'time.inclusive.duration':
time = int(value)
elif key == 'iteration':
loop = int(value)
elif key == 'size':
size = int(value)
else:
mode = key
if mode == 'initialization':
data['init'][size] = time
elif mode:
if size not in data[mode]:
data[mode][size] = [None] * 10
if size is not None and loop is not None:
data[mode][size][loop] = time
OMP = data['OMP']
control = data['baseline']
Serial = data['RajaSerial']
AgencyParallel = data['AgencyParallel']
AgencySerial = data['AgencySerial']
RawAgency = data['RawAgency']
RawOMP = data['RawOMP']
dat = [[size, c, omp, serial, agp, ags, ra, ro]
for size in OMP for omp, serial, c, agp, ags, ra, ro in zip(
OMP[size], Serial[size], control[size], AgencyParallel[size],
AgencySerial[size], RawAgency[size], RawOMP[size])]
dataframe = pd.DataFrame(data=dat,
columns=[
'Size', 'baseline', 'OMP', 'RajaSerial',
'AgencyParallel', 'AgencySerial', 'RawAgency',
'RawOMP'
])
d = [[
size, dataframe[dataframe['Size'] == size]['OMP'].mean(),
stats.sem(dataframe[dataframe['Size'] == size]['OMP']),
dataframe[dataframe['Size'] == size]['RajaSerial'].mean(),
stats.sem(dataframe[dataframe['Size'] == size]['RajaSerial']),
dataframe[dataframe['Size'] == size]['baseline'].mean(),
stats.sem(dataframe[dataframe['Size'] == size]['baseline']),
dataframe[dataframe['Size'] == size]['AgencyParallel'].mean(),
stats.sem(dataframe[dataframe['Size'] == size]['AgencyParallel']),
dataframe[dataframe['Size'] == size]['AgencySerial'].mean(),
stats.sem(dataframe[dataframe['Size'] == size]['AgencySerial']),
dataframe[dataframe['Size'] == size]['RawAgency'].mean(),
stats.sem(dataframe[dataframe['Size'] == size]['RawAgency']),
dataframe[dataframe['Size'] == size]['RawOMP'].mean(),
stats.sem(dataframe[dataframe['Size'] == size]['RawOMP'])
] for size in sorted(OMP)]
realDataframe = pd.DataFrame(
data=d,
columns=[
'Size', 'OMPmean', 'OMPsem', 'RajaSerialmean', 'RajaSerialsem',
'baselinemean', 'baselinesem', 'AgencyParallelmean',
'AgencyParallelsem', 'AgencySerialmean', 'AgencySerialsem',
'RawAgencymean', 'RawAgencysem', 'RawOMPmean', 'RawOMPsem'
])
fig = plt.figure()
sizes = sorted(OMP)
legendNames = []
plt.errorbar(sizes,
realDataframe['OMPmean'],
color='k',
xerr=[0] * len(sizes),
yerr=realDataframe['OMPsem'])
legendNames.append('OMP RAJA')
plt.errorbar(sizes,
realDataframe['RawOMPmean'],
color='k',
xerr=[0] * len(sizes),
yerr=realDataframe['RawOMPsem'],
linestyle='dashed')
legendNames.append('Raw OMP')
plt.errorbar(sizes,
realDataframe['RajaSerialmean'],
color='r',
xerr=[0] * len(sizes),
yerr=realDataframe['RajaSerialsem'])
legendNames.append('Serial RAJA')
plt.errorbar(sizes,
realDataframe['baselinemean'],
color='r',
xerr=[0] * len(sizes),
yerr=realDataframe['baselinesem'],
linestyle='dashed')
legendNames.append('Control')
plt.errorbar(sizes,
realDataframe['AgencySerialmean'],
color='c',
xerr=[0] * len(sizes),
yerr=realDataframe['AgencySerialsem'])
legendNames.append('Serial Agency RAJA')
plt.errorbar(sizes,
realDataframe['AgencyParallelmean'],
color='b',
xerr=[0] * len(sizes),
yerr=realDataframe['AgencyParallelsem'])
legendNames.append('Parallel Agency RAJA')
plt.errorbar(sizes,
realDataframe['RawAgencymean'],
color='b',
xerr=[0] * len(sizes),
yerr=realDataframe['RawAgencysem'],
linestyle='dashed')
legendNames.append('Raw Agency')
plt.legend(legendNames, loc='best', fontsize=8)
plt.xlabel('Reduction Size', fontsize=16)
plt.ylabel('Absolute Time', fontsize=16)
plt.grid(b=True, which='major', color='k', linestyle='dotted')
plt.title('Reducer Time Taken (32 cores)', fontsize=22)
plt.yscale('log')
plt.savefig(filename)
# Get the code
totp = subprocess.check_output(['oathtool', '-b', '--totp', \
otpSecrets['otpsecrets'][args.label]]).rstrip('\n')
# Is this a number? Print it.
try:
print '%s\t(%dsec)' % (totp,
(30 - (datetime.now().second % 30)))
# Try and put it on the clipboard unless disabled
if 'use_clipboard' not in otpSecrets or otpSecrets[
'use_clipboard'] != False:
try:
program = [
'xclip', '-selection', 'clipboard'
] if platform.system() == 'Linux' else ['pbcopy']
process = subprocess.Popen(program,
stdin=subprocess.PIPE)
process.stdin.write(totp)
process.stdin.close()
except subprocess.CalledProcessError, err:
print 'Couldn\'t put on the clipboard'
# Wasn't parsed as int, but maybe the output is useful?
except ValueError:
print 'Output from oathtool doesn\'t seem to be valid'
print 'Here\'s the output anyway:\n'
print totp
# Call to oathtool failed
except subprocess.CalledProcessError, err:
print err.output
sys.exit(err.returncode)
else:
print 'Couldn\'t find label \'%s\' in the yaml. (Try the -l switch?)' % args.label
def start_job(self):
self._logger.info("Starting Executable")
job_error_msg = None
job_template = self.job_template
try:
# create job UVE and log
self.result_handler = JobResultHandler(self.job_template_id,
self.job_execution_id,
self.fabric_fq_name,
self._logger,
self.job_utils,
self.job_log_utils,
self.device_name,
self.job_description,
self.job_transaction_id,
self.job_transaction_descr)
msg = MsgBundle.getMessage(MsgBundle.START_JOB_MESSAGE,
job_execution_id=self.job_execution_id,
job_template_name=job_template.fq_name[
-1])
self._logger.debug(msg)
timestamp = int(round(time.time() * 1000))
self.job_log_utils.send_job_log(
job_template.fq_name,
self.job_execution_id,
self.fabric_fq_name,
msg,
JobStatus.STARTING.value,
timestamp=timestamp,
description=self.job_description,
transaction_id=self.job_transaction_id,
transaction_descr=self.job_transaction_descr)
# validate job input if required by job_template input_schema
input_schema = job_template.get_job_template_input_schema()
if input_schema:
self._validate_job_input(input_schema, self.job_data)
executable_list = job_template.get_job_template_executables()\
.get_executable_info()
for executable in executable_list:
exec_path = executable.get_executable_path()
executable.get_executable_args()
job_input_args = self.gather_job_args()
try:
exec_process = subprocess32.Popen(
[exec_path, "--job-input",
json.dumps(job_input_args)], close_fds=True, cwd='/',
stdout=subprocess32.PIPE, stderr=subprocess32.PIPE)
self.job_file_write.write_to_file(
self.job_execution_id,
"job_summary",
JobFileWrite.JOB_LOG,
{"job_status": JobStatus.IN_PROGRESS.value})
msg = "Child process pid = " + str(exec_process.pid)
self._logger.info(msg)
(out, err) = exec_process.communicate(
timeout=self.executable_timeout)
self._logger.notice(str(out))
self._logger.notice(str(err))
timestamp = int(round(time.time() * 1000))
self.job_log_utils.send_job_log(
job_template.fq_name,
self.job_execution_id,
self.fabric_fq_name,
str(err),
JobStatus.IN_PROGRESS.value,
timestamp=timestamp,
description=self.job_description,
transaction_id=self.job_transaction_id,
transaction_descr=self.job_transaction_descr)
except subprocess32.TimeoutExpired as timeout_exp:
if exec_process is not None:
os.kill(exec_process.pid, 9)
msg = MsgBundle.getMessage(
MsgBundle.RUN_EXECUTABLE_PROCESS_TIMEOUT,
exec_path=exec_path, exc_msg=repr(timeout_exp))
raise JobException(msg, self.job_execution_id)
self._logger.info(exec_process.returncode)
self._logger.info("Executable Completed")
if exec_process.returncode != 0:
job_status = JobStatus.FAILURE.value
msg = MsgBundle.getMessage(
MsgBundle.EXECUTABLE_RETURN_WITH_ERROR,
exec_uri=exec_path)
self._logger.error(msg)
else:
job_status = JobStatus.SUCCESS.value
msg = MsgBundle.getMessage(
MsgBundle.JOB_EXECUTION_COMPLETE,
job_execution_id=self.job_execution_id,
job_template_name=\
job_template.fq_name[-1])
self.job_file_write.write_to_file(
self.job_execution_id,
"job_summary",
JobFileWrite.JOB_LOG,
{"job_status": job_status})
self._logger.debug(msg)
timestamp = int(round(time.time() * 1000))
self.job_log_utils.send_job_log(
job_template.fq_name,
self.job_execution_id,
self.fabric_fq_name,
msg,
job_status,
timestamp=timestamp,
description=self.job_description,
transaction_id=self.job_transaction_id,
transaction_descr=self.job_transaction_descr)
except JobException as exp:
err_msg = "Job Exception recieved: %s " % repr(exp)
self._logger.error(err_msg)
self._logger.error("%s" % traceback.format_exc())
self.result_handler.update_job_status(JobStatus.FAILURE,
err_msg)
if job_template:
self.result_handler.create_job_summary_log(
job_template.fq_name)
job_error_msg = err_msg
except Exception as exp:
err_msg = "Error while executing job %s " % repr(exp)
self._logger.error(err_msg)
self._logger.error("%s" % traceback.format_exc())
self.result_handler.update_job_status(JobStatus.FAILURE,
err_msg)
self.result_handler.create_job_summary_log(job_template.fq_name)
job_error_msg = err_msg
finally:
# need to wait for the last job log and uve update to complete
# via sandesh and then close sandesh connection
sandesh_util = SandeshUtils(self._logger)
sandesh_util.close_sandesh_connection()
self._logger.info("Closed Sandesh connection")
if job_error_msg is not None:
sys.exit(job_error_msg)
def cutadaptit_pairs(data, sample):
"""
Applies trim & filters to pairs, including adapter detection. If we have
barcode information then we use it to trim reversecut+bcode+adapter from
reverse read, if not then we have to apply a more general cut to make sure
we remove the barcode, this uses wildcards and so will have more false
positives that trim a little extra from the ends of reads. Should we add
a warning about this when filter_adapters=2 and no barcodes?
"""
LOGGER.debug("Entering cutadaptit_pairs - {}".format(sample.name))
sname = sample.name
## applied to read pairs
#trim_r1 = str(data.paramsdict["edit_cutsites"][0])
#trim_r2 = str(data.paramsdict["edit_cutsites"][1])
finput_r1 = sample.files.concat[0][0]
finput_r2 = sample.files.concat[0][1]
## Get adapter sequences. This is very important. For the forward adapter
## we don't care all that much about getting the sequence just before the
## Illumina adapter, b/c it will either be random (in RAD), or the reverse
## cut site of cut1 or cut2 (gbs or ddrad). Either way, we can still trim it
## off later in step7 with trim overhang if we want. And it should be invar-
## iable unless the cut site has an ambiguous char. The reverse adapter is
## super important, however b/c it can contain the inline barcode and
## revcomp cut site. We def want to trim out the barcode, and ideally the
## cut site too to be safe. Problem is we don't always know the barcode if
## users demultiplexed their data elsewhere. So, if barcode is missing we
## do a very fuzzy match before the adapter and trim it out.
if not data.barcodes:
## try linking barcodes again in case user just added a barcodes path
## after receiving the warning
try:
data._link_barcodes()
except Exception as inst:
LOGGER.warning(" error adding barcodes info: %s", inst)
if data.barcodes:
try:
adapter1 = fullcomp(data.paramsdict["restriction_overhang"][1])[::-1]+\
data._hackersonly["p3_adapter"]
adapter2 = fullcomp(data.paramsdict["restriction_overhang"][0])[::-1]+\
fullcomp(data.barcodes[sample.name])[::-1]+\
data._hackersonly["p5_adapter"] ## <- should this be revcomp?
except KeyError as inst:
msg = """
Sample name does not exist in the barcode file. The name in the barcode file
for each sample must exactly equal the raw file name for the sample minus
`_R1`. So for example a sample called WatDo_PipPrep_R1_100.fq.gz must
be referenced in the barcode file as WatDo_PipPrep_100. The name in your
barcode file for this sample must match: {}
""".format(sample.name)
LOGGER.error(msg)
raise IPyradWarningExit(msg)
else:
print(NO_BARS_GBS_WARNING)
#adapter1 = fullcomp(data.paramsdict["restriction_overhang"][1])[::-1]+\
# data._hackersonly["p3_adapter"]
#adapter2 = "XXX"
adapter1 = data._hackersonly["p3_adapter"]
adapter2 = fullcomp(data._hackersonly["p5_adapter"])
## parse trim_reads
trim5r1 = trim5r2 = trim3r1 = trim3r2 = []
if data.paramsdict.get("trim_reads"):
trimlen = data.paramsdict.get("trim_reads")
## trim 5' end
if trimlen[0]:
trim5r1 = ["-u", str(trimlen[0])]
if trimlen[1] < 0:
trim3r1 = ["-u", str(trimlen[1])]
if trimlen[1] > 0:
trim3r1 = ["--length", str(trimlen[1])]
## legacy support for trimlen = 0,0 default
if len(trimlen) > 2:
if trimlen[2]:
trim5r2 = ["-U", str(trimlen[2])]
if len(trimlen) > 3:
if trimlen[3]:
if trimlen[3] < 0:
trim3r2 = ["-U", str(trimlen[3])]
if trimlen[3] > 0:
trim3r2 = ["--length", str(trimlen[3])]
else:
## legacy support
trimlen = data.paramsdict.get("edit_cutsites")
trim5r1 = ["-u", str(trimlen[0])]
trim5r2 = ["-U", str(trimlen[1])]
## testing new 'trim_reads' setting
cmdf1 = ["cutadapt"]
if trim5r1:
cmdf1 += trim5r1
if trim3r1:
cmdf1 += trim3r1
if trim5r2:
cmdf1 += trim5r2
if trim3r2:
cmdf1 += trim3r2
cmdf1 += ["--trim-n",
"--max-n", str(data.paramsdict["max_low_qual_bases"]),
"--minimum-length", str(data.paramsdict["filter_min_trim_len"]),
"-o", OPJ(data.dirs.edits, sname+".trimmed_R1_.fastq.gz"),
"-p", OPJ(data.dirs.edits, sname+".trimmed_R2_.fastq.gz"),
finput_r1,
finput_r2]
## additional args
if int(data.paramsdict["filter_adapters"]) < 2:
## add a dummy adapter to let cutadapt know whe are not using legacy-mode
cmdf1.insert(1, "XXX")
cmdf1.insert(1, "-A")
if int(data.paramsdict["filter_adapters"]):
cmdf1.insert(1, "20,20")
cmdf1.insert(1, "-q")
cmdf1.insert(1, str(data.paramsdict["phred_Qscore_offset"]))
cmdf1.insert(1, "--quality-base")
if int(data.paramsdict["filter_adapters"]) > 1:
## first enter extra cuts
zcut1 = data._hackersonly["p3_adapters_extra"][::-1]
zcut2 = data._hackersonly["p5_adapters_extra"][::-1]
for ecut1, ecut2 in zip(zcut1, zcut2):
cmdf1.insert(1, ecut1)
cmdf1.insert(1, "-a")
cmdf1.insert(1, ecut2)
cmdf1.insert(1, "-A")
## then put the main cut first
cmdf1.insert(1, adapter1)
cmdf1.insert(1, '-a')
cmdf1.insert(1, adapter2)
cmdf1.insert(1, '-A')
## do modifications to read1 and write to tmp file
LOGGER.debug(" ".join(cmdf1))
#sys.exit()
try:
proc1 = sps.Popen(cmdf1, stderr=sps.STDOUT, stdout=sps.PIPE, close_fds=True)
res1 = proc1.communicate()[0]
except KeyboardInterrupt:
proc1.kill()
LOGGER.info("this is where I want it to interrupt")
raise KeyboardInterrupt()
## raise errors if found
if proc1.returncode:
raise IPyradWarningExit(" error [returncode={}]: {}\n{}".format(proc1.returncode, " ".join(cmdf1), res1))
LOGGER.debug("Exiting cutadaptit_pairs - {}".format(sname))
## return results string to be parsed outside of engine
return res1
