def run_python(path, env=None, args=None, timeout=None, pythonpath_extend=None, expect_pass=False): new_argv = [sys.executable] if sys.version_info[:2] <= (2, 7): new_argv += ['-W', 'ignore:Python 2 is no longer supported'] new_env = os.environ.copy() new_env.setdefault('eventlet_test_in_progress', 'yes') src_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if path: path = os.path.abspath(path) new_argv.append(path) new_env['PYTHONPATH'] = os.pathsep.join(sys.path + [src_dir]) if env: new_env.update(env) if pythonpath_extend: new_path = [ p for p in new_env.get('PYTHONPATH', '').split(os.pathsep) if p ] new_path.extend(p if os.path.isabs(p) else os.path.join(src_dir, p) for p in pythonpath_extend) new_env['PYTHONPATH'] = os.pathsep.join(new_path) if args: new_argv.extend(args) p = subprocess.Popen( new_argv, env=new_env, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, ) if timeout is None: timeout = 10 try: output, _ = p.communicate(timeout=timeout) except subprocess.TimeoutExpired: p.kill() output, _ = p.communicate(timeout=timeout) if expect_pass: sys.stderr.write('Program {0} output:\n---\n{1}\n---\n'.format( path, output.decode())) assert False, 'timed out' return '{0}\nFAIL - timed out'.format(output).encode() if expect_pass: if output.startswith(b'skip'): parts = output.rstrip().split(b':', 1) skip_args = [] if len(parts) > 1: skip_args.append(parts[1]) raise SkipTest(*skip_args) ok = output.rstrip() == b'pass' if not ok: sys.stderr.write('Program {0} output:\n---\n{1}\n---\n'.format( path, output.decode())) assert ok, 'Expected single line "pass" in stdout' return output
def plot_raja_mmultgpu(cali_file, baseFileName, cali_query, title=True, legend=True): ANNOTATIONS = ':'.join( ['iteration', 'size', 'mode', 'time.inclusive.duration']) CALI_QUERY = _get_cali_query(cali_query, ANNOTATIONS, cali_file) p = subprocess.Popen(CALI_QUERY, stdout=subprocess.PIPE) out, err = p.communicate() data = {} for line in out.split(): line = dict(map(lambda x: x.split('='), line.split(','))) if len(line) != 4: continue time = int(line['time.inclusive.duration']) size = int(line['size']) mode = line['mode'] if mode not in data: data[mode] = {} if size not in data[mode]: data[mode][size] = [] data[mode][size].append(time) colors = ('r', 'b', 'k', 'g') markers = ('o', 'v', 's', '*') namelist = (('Mmult1_cuda', 'RAJA w/ CUDA Loop Fusion'), ('Mmult2_cuda', 'RAJA w/ CUDA normal'), ('Mmult1_agency', 'RAJA w/ Agency Loop Fusion'), ('Mmult2_agency', 'RAJA w/ Agency normal')) names = { key: (name, colors[i], markers[i]) for i, (key, name) in enumerate(namelist) } fig = plt.figure() legendNames = [] for mode, datum in data.iteritems(): sizes = sorted(datum.keys()) xerr = [0] * len(sizes) means = [mean(datum[size]) for size in sizes] sems = [sem(datum[size]) for size in sizes] name, color, marker = names[mode] legendNames.append(name) plt.errorbar(sizes, means, xerr=xerr, yerr=sems, marker=marker, color=color) if legend: plt.legend(legendNames, loc='best', fontsize=12) if title: plt.title('Matrix Multiplication Performance', fontsize=22) plt.xlabel('Matrix Size', fontsize=16) plt.ylabel('Time Taken (ms)', fontsize=16) plt.yscale('log') plt.grid(b=True, which='major', color='k', linestyle='dotted') filename = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'figs', '{}_gpu.pdf'.format(baseFileName)) plt.savefig(filename)
def plot_raja_dependent_bench(cali_file, filename, cali_query): ANNOTATIONS = ':'.join( ['iteration', 'size', 'loop', 'time.inclusive.duration']) CALI_QUERY = _get_cali_query(cali_query, ANNOTATIONS, cali_file) p = subprocess.Popen(CALI_QUERY, stdout=subprocess.PIPE) out, err = p.communicate() data = { 'Agency': {}, 'OMP': {}, } for line in out.split(): line = dict(map(lambda x: x.split('='), line.split(','))) loop = None time = None mode = None size = None for key, value in line.iteritems(): if key == 'time.inclusive.duration': time = int(value) elif key == 'iteration': mode = value elif key == 'size': size = int(value) elif key == 'loop': loop = int(value) if mode == 'Agency' or mode == 'OMP': if size not in data[mode]: data[mode][size] = [] data[mode][size].append(time) OMP = data['OMP'] agency = data['Agency'] fig = plt.figure() dat = [[size, omp, a] for size in OMP for omp, a in zip(OMP[size], agency[size])] dataframe = pd.DataFrame(data=dat, columns=['Size', 'OMP', 'Agency']) d = [[ size, dataframe[dataframe['Size'] == size]['OMP'].mean(), stats.sem(dataframe[dataframe['Size'] == size]['OMP']), dataframe[dataframe['Size'] == size]['Agency'].mean(), stats.sem(dataframe[dataframe['Size'] == size]['Agency']), ] for size in sorted(OMP.keys())] realDataframe = pd.DataFrame( data=d, columns=['Size', 'OMPmean', 'OMPsem', 'Agencymean', 'Agencysem']) fig = plt.figure() sizes = sorted(OMP) legendNames = [] legendNames.append('OMP RAJA') legendNames.append('Agency RAJA') plt.xscale('log') plt.yscale('log') plt.errorbar(sizes, realDataframe['OMPmean'], color='r', xerr=[0] * len(sizes), yerr=realDataframe['OMPsem']) plt.errorbar(sizes, realDataframe['Agencymean'], color='g', xerr=[0] * len(sizes), yerr=realDataframe['Agencysem']) plt.legend(legendNames, loc='center right') plt.xlabel('Array Size', fontsize=16) plt.ylabel('Total Time (ms)', fontsize=16) plt.grid(b=True, which='major', color='k', linestyle='dotted') plt.title('Dependent Index Sets Benchmark', fontsize=22) plt.savefig(filename)
def execute(*args, **kwargs): logger = logging.getLogger(logger_name) command = 'simpleflow.execute' # name of a module. sys.stdout.flush() sys.stderr.flush() result_str = None # useless context = kwargs.pop('context', {}) with tempfile.TemporaryFile() as result_fd, tempfile.TemporaryFile( ) as error_fd: dup_result_fd = os.dup(result_fd.fileno()) # remove FD_CLOEXEC dup_error_fd = os.dup(error_fd.fileno()) # remove FD_CLOEXEC arguments_json = format_arguments_json(*args, **kwargs) full_command = [ interpreter, '-m', command, # execute module a script. get_name(func), '--logger-name={}'.format(logger_name), '--result-fd={}'.format(dup_result_fd), '--error-fd={}'.format(dup_error_fd), '--context={}'.format(json_dumps(context)), ] if len( arguments_json ) < MAX_ARGUMENTS_JSON_LENGTH: # command-line limit on Linux: 128K full_command.append(arguments_json) arg_file = None arg_fd = None else: arg_file = tempfile.TemporaryFile() arg_file.write(arguments_json.encode('utf-8')) arg_file.flush() arg_file.seek(0) arg_fd = os.dup(arg_file.fileno()) full_command.append( '--arguments-json-fd={}'.format(arg_fd)) full_command.append('foo') # dummy funcarg if kill_children: full_command.append('--kill-children') if is_buggy_subprocess32( ): # close_fds doesn't work with subprocess32 < 3.5.0 close_fds = False pass_fds = [] else: close_fds = True pass_fds = [dup_result_fd, dup_error_fd] if arg_file: pass_fds.append(arg_fd) process = subprocess.Popen( full_command, bufsize=-1, close_fds=close_fds, pass_fds=pass_fds, ) rc = wait_subprocess(process, timeout=timeout, command_info=full_command) os.close(dup_result_fd) os.close(dup_error_fd) if arg_file: arg_file.close() if rc: error_fd.seek(0) err_output = error_fd.read() if err_output: if not compat.PY2: err_output = err_output.decode('utf-8', errors='replace') raise ExecutionError(err_output) result_fd.seek(0) result_str = result_fd.read() if not result_str: return None try: if not compat.PY2: result_str = result_str.decode('utf-8', errors='replace') result = format.decode(result_str) return result except BaseException as ex: logger.exception('Exception in python.execute: {} {}'.format( ex.__class__.__name__, ex)) logger.warning('%r', result_str)
def run(command, timeout=60, ignore_status=False, env=None, io_encoding='utf-8'): """Execute a command in a subproccess and return its output. Commands can be either shell commands (given as strings) or the path and arguments to an executable (given as a list). This function will block until the subprocess finishes or times out. Args: command: The command to execute. Can be either a string or a list. timeout: number seconds to wait for command to finish. ignore_status: bool True to ignore the exit code of the remote subprocess. Note that if you do ignore status codes, you should handle non-zero exit codes explicitly. env: dict enviroment variables to setup on the remote host. io_encoding: str unicode encoding of command output. Returns: A job.Result containing the results of the ssh command. Raises: job.TimeoutError: When the remote command took to long to execute. Error: When the ssh connection failed to be created. CommandError: Ssh worked, but the command had an error executing. """ start_time = time.time() proc = subprocess.Popen(command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=not isinstance(command, list)) # Wait on the process terminating timed_out = False out = bytes() err = bytes() try: (out, err) = proc.communicate(timeout=timeout) except subprocess.TimeoutExpired: timed_out = True proc.kill() proc.wait() result = Result(command=command, stdout=out, stderr=err, exit_status=proc.returncode, duration=time.time() - start_time, encoding=io_encoding, did_timeout=timed_out) logging.debug(result) if timed_out: logging.error("Command %s with %s timeout setting timed out", command, timeout) raise TimeoutError(result) if not ignore_status and proc.returncode != 0: raise Error(result) return result
def run(self): self._popen = subprocess.Popen(self._cmd, env=self._env, cwd=self._cwd)
def runMD(wdir, localExe=True, sshConn=None, remoteSettings=None, scriptExe='lie_mdrun.sh', killer=None, killSig='/KILL'): """ Kill of the process can take place via: killing job manager(runLIE) or creation of a file (killSig) In the first case status is updated till this step In the last, job become CANCELLED """ if killer == None: killer = jobHandler.WarmKiller() currDir = os.getcwd() #get list of directory and execute MDs listMDs = [x[0] for x in os.walk(wdir) if x[0] != wdir] listResults = [] for simDir in listMDs: simInfo = {} simInfo['wdir'] = simDir logging.debug('Running simulation %s' % simDir) if localExe: ### LOCAL EXECUTION os.chdir(simDir) #save stdout and stderr of the script in files for debug logfile = 'runMD.log' errfile = 'runMD.err' logMD = open(logfile, 'w') errMD = open(errfile, 'w') cmd = ['bash', scriptExe] try: proc = sp.Popen(cmd, stdout=logMD, stderr=errMD) while True: if proc.poll() != None: break if os.path.exists(killSig) or killer.kill_now: logging.info("%s Warm Killing runMD function") jobHandler.killProcs(proc.pid) if os.path.exists(killSig): os.remove(killSig) return (True, 'CANCELLED') else: return (True, 'MD READY') simInfo['status'] = 'Done' except Exception, e: logging.error(e) #if simDir.endswith('0-0'): # msg='Simulation of ligand in solvent failed: %s'%e # return (False, msg) simInfo['status'] = 'Failed' os.chdir(currDir) else: ### REMOTE EXECUTION # 1. Tar files tarName = 'MDfiles.tar' if os.path.exists(tarName): os.remove(tarName) os.chdir(simDir) listfiles = glob('*') tar = tarfile.open(tarName, 'w') for fname in listfiles: tar.add(fname) tar.close() # 2. copy on the remote dir rwdir = os.path.join(remoteSettings['wdir'], simDir.split(os.sep)[-3], simDir.split(os.sep)[-1]) sshConn.putFile(tarName, rwdir) # 3. submit sshConn.execCmd('cd %s; tar xvf %s' % (rwdir, tarName)) success, (status, stdOut) = sshConn.execCmd( 'cd %s; %s %s' % (rwdir, remoteSettings['submit'], scriptExe)) simInfo['jobid'] = stdOut.rstrip() simInfo['rdir'] = rwdir if status == 0: simInfo['status'] = 'Submitted' else: simInfo['status'] = 'Failed' listResults.append(simInfo)
def trigger_stage_out(self, jobspec): # make logger tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID), method_name='trigger_stage_out') tmpLog.debug('start') # loop over all files allChecked = True ErrMsg = 'These files failed to upload : ' zip_datasetName = 'harvester_stage_out.{0}'.format(str(uuid.uuid4())) fileAttrs = jobspec.get_output_file_attributes() for fileSpec in jobspec.outFiles: # fileSpec.fileAttributes['transferID'] = None # synchronius transfer # skip already done tmpLog.debug('file: %s status: %s' % (fileSpec.lfn, fileSpec.status)) if fileSpec.status in ['finished', 'failed']: continue fileSpec.pathConvention = self.pathConvention fileSpec.objstoreID = self.objstoreID # set destination RSE if fileSpec.fileType in ['es_output', 'zip_output', 'output']: dstRSE = self.dstRSE_Out elif fileSpec.fileType == 'log': dstRSE = self.dstRSE_Log else: errMsg = 'unsupported file type {0}'.format(fileSpec.fileType) tmpLog.error(errMsg) return (False, errMsg) # skip if destination is None if dstRSE is None: continue # get/set scope and dataset name if fileSpec.fileType == 'log': if fileSpec.lfn in fileAttrs: scope = fileAttrs[fileSpec.lfn]['scope'] datasetName = fileAttrs[fileSpec.lfn]['dataset'] else: lfnWithoutWorkerID = ".".join(fileSpec.lfn.split('.')[:-1]) scope = fileAttrs[lfnWithoutWorkerID]['scope'] datasetName = fileAttrs[lfnWithoutWorkerID]['dataset'] elif fileSpec.fileType != 'zip_output' and fileSpec.lfn in fileAttrs: scope = fileAttrs[fileSpec.lfn]['scope'] datasetName = fileAttrs[fileSpec.lfn]['dataset'] else: # use panda scope for zipped files scope = self.scopeForTmp datasetName = zip_datasetName # for now mimic behaviour and code of pilot v2 rucio copy tool (rucio download) change when needed executable_prefix = None pfn_prefix = None if self.objectstore_additions and dstRSE in self.objectstore_additions: if 'storage_id' in self.objectstore_additions[dstRSE]: fileSpec.objstoreID = self.objectstore_additions[dstRSE][ 'storage_id'] if 'access_key' in self.objectstore_additions[dstRSE] and \ 'secret_key' in self.objectstore_additions[dstRSE] and \ 'is_secure' in self.objectstore_additions[dstRSE]: executable_prefix = "export S3_ACCESS_KEY=%s; export S3_SECRET_KEY=%s; export S3_IS_SECURE=%s" % ( self.objectstore_additions[dstRSE]['access_key'], self.objectstore_additions[dstRSE]['secret_key'], self.objectstore_additions[dstRSE]['is_secure']) if 'pfn_prefix' in self.objectstore_additions[dstRSE]: pfn_prefix = self.objectstore_additions[dstRSE][ 'pfn_prefix'] executable = ['/usr/bin/env', 'rucio', '-v', 'upload'] executable += ['--no-register'] if hasattr(self, 'lifetime'): executable += ['--lifetime', ('%d' % self.lifetime)] if fileSpec.fileAttributes is not None and 'guid' in fileSpec.fileAttributes: executable += ['--guid', fileSpec.fileAttributes['guid']] executable += ['--rse', dstRSE] executable += ['--scope', scope] if pfn_prefix: executable += [ '--pfn %s' % os.path.join(pfn_prefix, os.path.basename(fileSpec.path)) ] else: executable += [('%s:%s' % (scope, datasetName))] executable += [('%s' % fileSpec.path)] tmpLog.debug('rucio upload command: {0} '.format(executable)) tmpLog.debug('rucio upload command (for human): %s ' % ' '.join(executable)) if executable_prefix: cmd = executable_prefix + "; " + ' '.join(executable) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) else: process = subprocess.Popen(executable, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout, stderr = process.communicate() fileSpec.attemptNr += 1 stdout = stdout.decode() + " attemptNr: %s" % fileSpec.attemptNr tmpLog.debug("stdout: %s" % stdout) tmpLog.debug("stderr: %s" % stderr) if process.returncode == 0: fileSpec.status = 'finished' else: # check what failed file_exists = False rucio_sessions_limit_error = False for line in stdout.split('\n'): if 'File name in specified scope already exists' in line: file_exists = True break elif 'File already exists on RSE' in line: # can skip if file exist on RSE since no register tmpLog.warning( 'rucio skipped upload and returned stdout: %s' % stdout) file_exists = True break elif 'exceeded simultaneous SESSIONS_PER_USER limit' in line: rucio_sessions_limit_error = True if file_exists: tmpLog.debug('file exists, marking transfer as finished') fileSpec.status = 'finished' elif rucio_sessions_limit_error: # do nothing tmpLog.warning( 'rucio returned error, will retry: stdout: %s' % stdout) # do not change fileSpec.status and Harvester will retry if this function returns False allChecked = False continue else: tmpLog.error('rucio upload failed with stdout: %s' % stdout) ErrMsg += '%s failed with rucio error stdout="%s"' % ( fileSpec.lfn, stdout) allChecked = False if fileSpec.attemptNr >= self.maxAttempts: tmpLog.error( 'reached maxattempts: %s, marked it as failed' % self.maxAttempts) fileSpec.status = 'failed' # force update fileSpec.force_update('status') tmpLog.debug('file: %s status: %s' % (fileSpec.lfn, fileSpec.status)) # return tmpLog.debug('done') if allChecked: return True, '' else: return False, ErrMsg
def copy_data(source='',target=''): cmd = 'cp '+ mkpath('data/' + source) + ' ' + mkpath(target) p = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE) return p.communicate()
username = re.sub('/CN=Robot:[^/]+', '', username) pat = re.compile('.*/CN=([^\/]+)/CN=([^\/]+)') mat = pat.match(username) if mat: username = mat.group(2) else: username = username.replace('/CN=', '') if username.lower().find('/email') > 0: username = username[:username.lower().find('/email')] pat = re.compile('.*(limited.*proxy).*') mat = pat.match(username) if mat: username = mat.group(1) username = username.replace('(', '') username = username.replace(')', '') username = username.replace("'", '') return username except: return id certFile = sys.argv[1] com = "openssl x509 -noout -subject -in" p = subprocess.Popen(com.split() + [certFile], stdout=subprocess.PIPE) out, err = p.communicate() out = re.sub('^subject=', '', out) out = out.strip() print('DN: "{0}"'.format(out)) print('extracted: "{0}"'.format(clean_user_id(out)))
def run(cmd, cwd=Settings.TEST_RUN_HOME, wait=True, timeout=600, fail_safe=False, register=True, log_level=logging.DEBUG): # Init result values time_string = datetime.now().strftime('%Y_%m_%d_%H_%M_%S_%f') log_file = os.path.join(Settings.TEST_OUT_LOGS, 'command_{0}.txt'.format(time_string)) complete = False duration = None output = '' # Ensure logs folder exists dir_path = os.path.dirname(os.path.realpath(log_file)) Folder.create(dir_path) # Command settings if not wait: # Redirect output to file File.write(path=log_file, text=cmd + os.linesep + '====>' + os.linesep) cmd = cmd + ' >> ' + log_file + ' 2>&1 &' # Log command that will be executed: Log.log(level=log_level, msg='Execute command: ' + cmd) Log.log(level=logging.DEBUG, msg='CWD: ' + cwd) # Execute command: if wait: start = time.time() with open(log_file, mode='w') as log: if Settings.HOST_OS == OSType.WINDOWS: process = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=log, stderr=log) else: process = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=subprocess.PIPE, stderr=log) # Wait until command complete try: process.wait(timeout=timeout) complete = True out, err = process.communicate() if out is not None: if Settings.PYTHON_VERSION < 3: output = str(out.decode('utf8').encode('utf8')).strip() else: output = out.decode("utf-8").strip() except subprocess.TimeoutExpired: process.kill() if fail_safe: Log.error('Command "{0}" timeout after {1} seconds.'.format( cmd, timeout)) else: raise # Append stderr to output stderr = File.read(path=log_file) if stderr: output = output + os.linesep + File.read(path=log_file) # noinspection PyBroadException try: File.delete(path=log_file) except Exception: Log.debug('Failed to clean log file: {0}'.format(log_file)) log_file = None end = time.time() duration = end - start else: process = psutil.Popen(cmd, cwd=cwd, shell=True, stdin=None, stdout=None, stderr=None, close_fds=True) # Get result pid = process.pid exit_code = process.returncode # Log output of the process if wait: Log.log(level=log_level, msg='OUTPUT: ' + os.linesep + output + os.linesep) else: Log.log(level=log_level, msg='OUTPUT REDIRECTED: ' + log_file + os.linesep) # Construct result result = ProcessInfo(cmd=cmd, pid=pid, exit_code=exit_code, output=output, log_file=log_file, complete=complete, duration=duration) # Register in TestContext if psutil.pid_exists(result.pid) and register: TestContext.STARTED_PROCESSES.append(result) # Return the result return result
def unmount_ramdisk(dir_path): subprocess.Popen(['sudo', 'umount', dir_path]).wait()
def __doBackgroundDispatch(self, batch): if self.__getStatus(batch) == LocalDispatcher.Job.Status.Complete: return True for upstreamBatch in batch.preTasks(): if not self.__doBackgroundDispatch(upstreamBatch): return False if batch.blindData().get("killed"): self.__reportKilled(batch) return False if not batch.plug(): self.__reportCompleted(batch) return True if len(batch.frames()) == 0: # This case occurs for nodes like TaskList and TaskContextProcessors, # because they don't do anything in execute (they have empty hashes). # Their batches exist only to depend on upstream batches. We don't need # to do any work here, but we still signal completion for the task to # provide progress feedback to the user. self.__setStatus(batch, LocalDispatcher.Job.Status.Complete) IECore.msg(IECore.MessageHandler.Level.Info, self.__messageTitle, "Finished " + batch.blindData()["nodeName"].value) return True taskContext = batch.context() frames = str( IECore.frameListFromList([int(x) for x in batch.frames()])) args = [ "gaffer", "execute", "-script", self.__scriptFile, "-nodes", batch.blindData()["nodeName"].value, "-frames", frames, ] if self.__ignoreScriptLoadErrors: args.append("-ignoreScriptLoadErrors") contextArgs = [] for entry in [ k for k in taskContext.keys() if k != "frame" and not k.startswith("ui:") ]: if entry not in self.__context.keys( ) or taskContext[entry] != self.__context[entry]: contextArgs.extend(["-" + entry, repr(taskContext[entry])]) if contextArgs: args.extend(["-context"] + contextArgs) self.__setStatus(batch, LocalDispatcher.Job.Status.Running) IECore.msg(IECore.MessageHandler.Level.Info, self.__messageTitle, " ".join(args)) process = subprocess.Popen(args, start_new_session=True) batch.blindData()["pid"] = IECore.IntData(process.pid) while process.poll() is None: if batch.blindData().get("killed"): os.killpg(process.pid, signal.SIGTERM) self.__reportKilled(batch) return False time.sleep(0.01) if process.returncode: self.__reportFailed(batch) return False self.__setStatus(batch, LocalDispatcher.Job.Status.Complete) return True
def __init__(self, resource_id, log=""): # required in the local connection mode import Pyro.core from Pyro.errors import ConnectionClosedError try: import subprocess32 as subprocess except ImportError: import subprocess login = getpass.getuser() pyro_objet_name = "workflow_engine_" + login # run the workflow engine process and get back the # workflow_engine and ConnectionChecker URIs # command = "python -m cProfile -o /home/soizic/profile/profile /home/soizic/svn/brainvisa/source/soma/soma-workflow/trunk/python/soma_workflow/start_workflow_engine.py %s %s %s" %( # resource_id, # pyro_objet_name, # log) command = "python -m soma_workflow.start_workflow_engine %s %s %s" % ( resource_id, pyro_objet_name, log) engine_process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) line = engine_process.stdout.readline() stdout_content = line while line and line.split()[0] != pyro_objet_name: line = engine_process.stdout.readline() stdout_content = stdout_content + "\n" + line if not line: # A problem occured while starting the engine. line = engine_process.stderr.readline() stderr_content = line while line: line = engine_process.stderr.readline() stderr_content = stderr_content + "\n" + line raise ConnectionError( "A problem occured while starting the engine " "process on the local machine. \n" "**More details:**\n" "**Start engine process command line:** \n" "\n" + command + "\n\n" "**Engine process standard output:** \n" "\n" + stdout_content + "**Engine process standard error:** \n" "\n" + stderr_content) workflow_engine_uri = line.split()[1] line = engine_process.stdout.readline() stdout_content = stdout_content + "\n" + line while line and line.split()[0] != "connection_checker": line = engine_process.stdout.readline() stdout_content = stdout_content + "\n" + line if not line: # A problem occured while starting the engine. line = engine_process.stderr.readline() stderr_content = line while line: line = engine_process.stderr.readline() stderr_content = stderr_content + "\n" + line raise ConnectionError( "A problem occured while starting the engine " "process on the local machine. \n" "**More details:**\n" "**Start engine process command line:** \n" "\n" + command + "\n\n" "**Engine process standard output:** \n" "\n" + stdout_content + "**Engine process standard error:** \n" "\n" + stderr_content) connection_checker_uri = line.split()[1] line = engine_process.stdout.readline() stdout_content = stdout_content + "\n" + line while line and line.split()[0] != "configuration": line = engine_process.stdout.readline() stdout_content = stdout_content + "\n" + line if not line: # A problem occured while starting the engine. line = engine_process.stderr.readline() stderr_content = line while line: line = engine_process.stderr.readline() stderr_content = stderr_content + "\n" + line raise ConnectionError( "A problem occured while starting the engine " "process on the local machine. \n" "**More details:**\n" "**Start engine process command line:** \n" "\n" + command + "\n\n" "**Engine process standard output:** \n" "\n" + stdout_content + "**Engine process standard error:** \n" "\n" + stderr_content) configuration_uri = line.split()[1] print("workflow_engine_uri: " + workflow_engine_uri) print("connection_checker_uri: " + connection_checker_uri) print("configuration_uri: " + configuration_uri) # create the proxies # self.workflow_engine = Pyro.core.getProxyForURI(workflow_engine_uri) connection_checker = Pyro.core.getAttrProxyForURI( connection_checker_uri) self.configuration = Pyro.core.getAttrProxyForURI(configuration_uri) # create the connection holder objet for # # a clean disconnection in any case # self.__connection_holder = ConnectionHolder(connection_checker) self.__connection_holder.start()
def start_test(mr): tests_passed = True error_lines = [] print 'Starting testing for MR %s ...' % mr['source_branch'] # Add remote of user resp = requests.get(url='https://gitlab.cern.ch/api/v4/projects/%s' % str(mr['source_project_id']), params={'private_token': private_token}) proj = json.loads(resp.text) commands.getstatusoutput( 'git remote add %s %s' % (proj['namespace']['name'], proj['http_url_to_repo'])) # Fetch all print ' git fetch --all --prune' s, o = commands.getstatusoutput('git fetch --all --prune') if s != 0: print 'Error while fetching all: %s' % o sys.exit(-1) # Rebase master/next print ' git rebase origin/next next' s, o = commands.getstatusoutput('git rebase origin/next next') if s != 0: print 'Error while rebaseing next: %s' % o sys.exit(-1) print ' git rebase origin/master master' s, o = commands.getstatusoutput('git rebase origin/master master') if s != 0: print 'Error while rebaseing master: %s' % o sys.exit(-1) # Check for Cross Merges if mr['source_branch'].lower().startswith('patch'): print ' Checking for cross-merges:' commits = commands.getoutput( 'git log master..remotes/%s/%s | grep ^commit' % (proj['namespace']['name'], mr['source_branch'])) for commit in commits.splitlines(): commit = commit.partition(' ')[2] if commands.getstatusoutput( 'git branch --contains %s | grep next' % commit)[0] == 0: print ' Found cross-merge problem with commit %s' % commit tests_passed = False error_lines.append('##### CROSS-MERGE TESTS:\n') error_lines.append('```\n') error_lines.append( 'This patch is suspicious. It looks like there are feature-commits pulled into the master branch!\n' ) error_lines.append('```\n') break # Checkout the branch to test print ' git checkout remotes/%s/%s' % (proj['namespace']['name'], mr['source_branch']) if commands.getstatusoutput( 'git checkout remotes/%s/%s' % (proj['namespace']['name'], mr['source_branch']))[0] != 0: print 'Error while checking out branch' sys.exit(-1) # ACTUAL TESTS START HERE # print ' Installing .venv' # Try re-install .venv (This validates the python packages commands.getstatusoutput('rm -R --force .venv/') # Remove old .venv if commands.getstatusoutput('python tools/install_venv.py')[0] != 0: print 'Error while installing .venv' tests_passed = False error_lines.append('##### INSTALLING .VENV FAILED\n') # Restart apache and memcached print ' /sbin/service memcached restart' if commands.getstatusoutput('/sbin/service memcached restart')[0] != 0: print 'Error while restarting memcached' sys.exit(-1) # Restart apache and memcached print ' service httpd restart' if commands.getstatusoutput('/sbin/service httpd restart')[0] != 0: print 'Error while restarting httpd' sys.exit(-1) changed_files = commands.getoutput( 'git diff-tree --no-commit-id --name-only -r --diff-filter=AMRT HEAD | grep .py | grep -v .py.mako' ).splitlines() command = """ cd %s; source .venv/bin/activate; cd .venv/lib/python2.7/site-packages/; ln -s %s/lib/rucio/; cd %s; pip install cx_oracle==5.2.1; python ../purge_bin.py; find lib -iname "*.pyc" | xargs rm; rm -rf /tmp/.rucio_*/; tools/reset_database.py; tools/sync_rses.py; tools/sync_meta.py; tools/bootstrap_tests.py; nosetests -v --logging-filter=-sqlalchemy,-requests,-rucio.client.baseclient --exclude=.*test_rse_protocol_.* --exclude=test_alembic --exclude=test_rucio_cache --exclude=test_rucio_server --exclude=test_dq2* --exclude=test_objectstore > /tmp/rucio_nose.txt 2> /tmp/rucio_nose.txt; tools/reset_database.py; nosetests -v lib/rucio/tests/test_alembic.py > /tmp/rucio_alembic.txt 2> /tmp/rucio_alembic.txt; flake8 --exclude=*.cfg bin/* lib/ tools/*.py > /tmp/rucio_flake8.txt; pylint %s > /tmp/rucio_pylint.txt; python ../purge_bin.py; """ % (root_git_dir, root_git_dir, root_git_dir, ' '.join(changed_files) ) # NOQA print ' %s' % command if tests_passed: proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) try: outs, errs = proc.communicate(timeout=60 * 20) except TimeoutExpired: print 'Timeout reached, killing test' proc.kill() outs, errs = proc.communicate() os.remove('/tmp/rucio_test.pid') sys.exit(-1) with open('/tmp/rucio_nose.txt', 'r') as f: lines = f.readlines() if lines[-1] != 'OK\n': tests_passed = False error_lines.append('##### UNIT TESTS:\n') error_lines.append('```\n') error_lines.extend(lines) error_lines.append('```\n') with open('/tmp/rucio_alembic.txt', 'r') as f: lines = f.readlines() if lines[-1] != 'OK\n': tests_passed = False error_lines.append('##### ALEMBIC:\n') error_lines.append('```\n') error_lines.extend(lines) error_lines.append('```\n') if os.stat('/tmp/rucio_flake8.txt').st_size != 0: with open('/tmp/rucio_flake8.txt', 'r') as f: lines = f.readlines() tests_passed = False error_lines.append('##### FLAKE8:\n') error_lines.append('```\n') error_lines.extend(lines) error_lines.append('```\n') # PYLINT if os.stat('/tmp/rucio_pylint.txt').st_size != 0: with open('/tmp/rucio_pylint.txt', 'r') as f: lines = f.readlines() pylint_passed = True # Check if there is an Error in PYLINT for line in lines: if line.startswith('E:'): tests_passed = False pylint_passed = False if not pylint_passed: error_lines.append('##### PYLINT\n') error_lines.append('```\n') for line in lines: if line.startswith('E:'): error_lines.append(line) error_lines.append('```\n') error_lines.append(lines[-2]) if tests_passed: error_lines.insert(0, '#### BUILD-BOT TEST RESULT: OK\n\n') else: error_lines.insert(0, '#### BUILD-BOT TEST RESULT: FAIL\n\n') commands.getstatusoutput('/sbin/service httpd stop') update_merg_request(mr=mr, test_result=tests_passed, comment=error_lines) # Checkout original master print ' git checkout master' if commands.getstatusoutput('git checkout master')[0] != 0: print 'Error while checking out master' sys.exit(-1)
def execute(*args, **kwargs): logger = logging.getLogger(logger_name) command = 'simpleflow.execute' # name of a module. sys.stdout.flush() sys.stderr.flush() result_str = None # useless context = kwargs.pop('context', {}) with tempfile.TemporaryFile() as result_fd, tempfile.TemporaryFile( ) as error_fd: dup_result_fd = os.dup(result_fd.fileno()) # remove FD_CLOEXEC dup_error_fd = os.dup(error_fd.fileno()) # remove FD_CLOEXEC # print('error_fd: {}'.format(dup_error_fd)) full_command = [ interpreter, '-m', command, # execute module a script. get_name(func), format_arguments_json(*args, **kwargs), '--logger-name={}'.format(logger_name), '--result-fd={}'.format(dup_result_fd), '--error-fd={}'.format(dup_error_fd), '--context={}'.format(json_dumps(context)), ] if kill_children: full_command.append('--kill-children') if compat.PY2: # close_fds doesn't work with python2 (using its C _posixsubprocess helper) close_fds = False pass_fds = () else: close_fds = True pass_fds = (dup_result_fd, dup_error_fd) process = subprocess.Popen( full_command, bufsize=-1, close_fds=close_fds, pass_fds=pass_fds, ) rc = wait_subprocess(process, timeout=timeout, command_info=full_command) os.close(dup_result_fd) os.close(dup_error_fd) if rc: error_fd.seek(0) err_output = error_fd.read() if err_output: if not compat.PY2: err_output = err_output.decode('utf-8', errors='replace') raise ExecutionError(err_output) result_fd.seek(0) result_str = result_fd.read() if not result_str: return None try: if not compat.PY2: result_str = result_str.decode('utf-8', errors='replace') result = format.decode(result_str) return result except BaseException as ex: logger.exception('Exception in python.execute: {} {}'.format( ex.__class__.__name__, ex)) logger.warning('%r', result_str)
def run(cmd, asynchronous=False): if asynchronous: return subprocess.Popen(cmd, shell=True) return subprocess.check_output(cmd, shell=True)
#メッセージを送信 ws.send(commands["register"]) while True: #受信したメッセージを表示 recv_str = ws.recv() print recv_str try: logging.info("ws.recv() = {}".format(recv_str)) ms = json.loads(recv_str) if ms["order"] == "exec_bash": print("exec {}".format(ms["cmd_str"])) logging.info("exec {}".format(ms["cmd_str"])) p = subprocess.Popen(ms["cmd_str"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) std_out, std_err = p.communicate(None, timeout=60) result = std_out.strip() error = std_err.strip() logging.info("result = {}".format(result)) logging.info("error = {}".format(error)) result_json = json.dumps({ "order": "response", "result": result, "error": error }) logging.info("result_json = {}".format(result_json))
def run(cmd, print_error=True, asynchronous=False, stdin=False, stderr=subprocess.STDOUT, outfile=None, env_vars=None, inherit_cwd=False, inherit_env=True, tty=False): # don't use subprocess module in Python 2 as it is not thread-safe # http://stackoverflow.com/questions/21194380/is-subprocess-popen-not-thread-safe # TODO: should be removed, now that Python 2 has reached its EOL if six.PY2: import subprocess32 as subprocess else: import subprocess env_dict = os.environ.copy() if inherit_env else {} if env_vars: env_dict.update(env_vars) env_dict = dict([(k, to_str(str(v))) for k, v in env_dict.items()]) if tty: asynchronous = True stdin = True try: cwd = os.getcwd() if inherit_cwd else None if not asynchronous: if stdin: return subprocess.check_output(cmd, shell=True, stderr=stderr, env=env_dict, stdin=subprocess.PIPE, cwd=cwd) output = subprocess.check_output(cmd, shell=True, stderr=stderr, env=env_dict, cwd=cwd) return output.decode(config.DEFAULT_ENCODING) # subprocess.Popen is not thread-safe, hence use a mutex here.. (TODO: mutex still needed?) with mutex_popen: stdin_arg = subprocess.PIPE if stdin else None stdout_arg = open(outfile, 'ab') if isinstance( outfile, six.string_types) else outfile stderr_arg = stderr if tty: # Note: leave the "pty" import here (not supported in Windows) import pty master_fd, slave_fd = pty.openpty() stdin_arg = slave_fd stdout_arg = stderr_arg = None # start the actual sub process kwargs = {} if is_linux() or is_mac_os(): kwargs['preexec_fn'] = os.setsid process = subprocess.Popen(cmd, shell=True, stdin=stdin_arg, bufsize=-1, stderr=stderr_arg, stdout=stdout_arg, env=env_dict, cwd=cwd, **kwargs) if tty: # based on: https://stackoverflow.com/questions/41542960 def pipe_streams(*args): while process.poll() is None: r, w, e = select.select([sys.stdin, master_fd], [], []) if sys.stdin in r: d = os.read(sys.stdin.fileno(), 10240) os.write(master_fd, d) elif master_fd in r: o = os.read(master_fd, 10240) if o: os.write(sys.stdout.fileno(), o) FuncThread(pipe_streams).start() return process except subprocess.CalledProcessError as e: if print_error: print("ERROR: '%s': exit code %s; output: %s" % (cmd, e.returncode, e.output)) sys.stdout.flush() raise e
def do_capture(status_code, the_record, base_url, model='capture'): """ Create a screenshot, text scrape, from a provided html file. This depends on phantomjs and an associated javascript file to perform the captures. In the event an error occurs, an exception is raised and handled by the celery task or the controller that called this method. """ # Make sure the the_record db.session.add(the_record) # If the capture is for static content, use a differnet PhantomJS config file if model == 'static': capture_name = the_record.filename service_args = [ app.config['PHANTOMJS'], '--ssl-protocol=any', '--ignore-ssl-errors=yes', os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/assets/static.js', app.config['LOCAL_STORAGE_FOLDER'], capture_name ] content_to_parse = os.path.join(app.config['LOCAL_STORAGE_FOLDER'], capture_name) else: capture_name = grab_domain(the_record.url) + '_' + str(the_record.id) service_args = [ app.config['PHANTOMJS'], '--ssl-protocol=any', '--ignore-ssl-errors=yes', os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/assets/capture.js', the_record.url, os.path.join(app.config['LOCAL_STORAGE_FOLDER'], capture_name) ] content_to_parse = os.path.join(app.config['LOCAL_STORAGE_FOLDER'], capture_name + '.html') # Using subprocess32 backport, call phantom and if process hangs kill it pid = subprocess32.Popen(service_args, stdout=PIPE, stderr=PIPE) try: stdout, stderr = pid.communicate(timeout=35) except subprocess32.TimeoutExpired: pid.kill() stdout, stderr = pid.communicate() app.logger.error('PhantomJS Static Capture timeout') raise Exception('PhantomJS Static Capture timeout') # If the subprocess has an error, raise an exception if stderr or stdout: raise Exception(stderr) # Strip tags and parse out all text ignore_tags = ('script', 'noscript', 'style') with open(content_to_parse, 'r') as content_file: content = content_file.read() cleaner = clean.Cleaner() content = cleaner.clean_html(content) doc = LH.fromstring(content) output = "" for elt in doc.iterdescendants(): if elt.tag in ignore_tags: continue text = elt.text or '' tail = elt.tail or '' wordz = " ".join((text, tail)).strip('\t') if wordz and len(wordz) >= 2 and not re.match("^[ \t\n]*$", wordz): output += wordz.encode('utf-8') # Since the filename format is different for static captures, update the filename # This will ensure the URLs are pointing to the correct resources if model == 'static': capture_name = capture_name.split('.')[0] # Wite our html text that was parsed into our capture folder parsed_text = open( os.path.join(app.config['LOCAL_STORAGE_FOLDER'], capture_name + '.txt'), 'wb') parsed_text.write(output) # Update the sketch record with the local URLs for the sketch, scrape, and html captures the_record.sketch_url = base_url + '/files/' + capture_name + '.png' the_record.scrape_url = base_url + '/files/' + capture_name + '.txt' the_record.html_url = base_url + '/files/' + capture_name + '.html' # Create a dict that contains what files may need to be written to S3 files_to_write = defaultdict(list) files_to_write['sketch'] = capture_name + '.png' files_to_write['scrape'] = capture_name + '.txt' files_to_write['html'] = capture_name + '.html' # If we are not writing to S3, update the capture_status that we are completed. if not app.config['USE_S3']: the_record.job_status = "COMPLETED" the_record.capture_status = "LOCAL_CAPTURES_CREATED" else: the_record.capture_status = "LOCAL_CAPTURES_CREATED" db.session.commit() return files_to_write
def main(): parser = argparse.ArgumentParser() parser.add_argument('dataset_dir', help='Dataset directory') args = parser.parse_args() dataset_dir = args.dataset_dir if not osp.exists(dataset_dir): print('Please install JSKV1 dataset to: %s' % dataset_dir) quit(1) PKG_DIR = rospkg.RosPack().get_path('jsk_arc2017_common') objlist_file = osp.join(PKG_DIR, 'data/others/object_list_5x8.jpg') objlist = cv2.imread(objlist_file) scale = 1000. / max(objlist.shape[:2]) objlist = cv2.resize(objlist, dsize=None, fx=scale, fy=scale) cmap = labelme.utils.labelcolormap(41) for stamp_dir in sorted(os.listdir(dataset_dir)): stamp = datetime.datetime.fromtimestamp(int(stamp_dir) / 1e9) stamp_dir = osp.join(dataset_dir, stamp_dir) if not osp.isdir(stamp_dir): continue json_file = osp.join(stamp_dir, 'label.json') img_file = osp.join(stamp_dir, 'image.jpg') lbl_file = osp.join(stamp_dir, 'label.npz') lbl_viz_file = osp.join(stamp_dir, 'label_viz.jpg') lock_file = osp.join(stamp_dir, 'annotation.lock') if osp.exists(lock_file): continue if not osp.exists(json_file): open(lock_file, 'w') print('%s: %s' % (stamp.isoformat(), stamp_dir)) cmd = 'labelme %s -O %s' % (img_file, json_file) output = subprocess.Popen(shlex.split(cmd)) returncode = None while returncode is None: try: output.communicate(timeout=1) returncode = output.returncode except subprocess.TimeoutExpired: pass except KeyboardInterrupt: break cv2.imshow('object list', objlist) cv2.waitKey(50) os.remove(lock_file) if returncode != 0: output.kill() break if not (osp.exists(lbl_file) and osp.exists(lbl_viz_file)): img = skimage.io.imread(img_file) lbl = json_to_label(json_file) lbl_viz = skimage.color.label2rgb(lbl, img, bg_label=0, colors=cmap[1:]) np.savez_compressed(lbl_file, lbl) skimage.io.imsave(lbl_viz_file, lbl_viz)
def trigger_preparation(self, jobspec): # make logger tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID), method_name='trigger_preparation') tmpLog.debug('start') # loop over all inputs inFileInfo = jobspec.get_input_file_attributes() gucInput = None for tmpFileSpec in jobspec.inFiles: # construct source and destination paths srcPath = mover_utils.construct_file_path( self.srcBasePath, inFileInfo[tmpFileSpec.lfn]['scope'], tmpFileSpec.lfn) dstPath = mover_utils.construct_file_path( self.dstBasePath, inFileInfo[tmpFileSpec.lfn]['scope'], tmpFileSpec.lfn) # local access path accPath = mover_utils.construct_file_path( self.localBasePath, inFileInfo[tmpFileSpec.lfn]['scope'], tmpFileSpec.lfn) if self.checkLocalPath: # check if already exits if os.path.exists(accPath): # calculate checksum checksum = core_utils.calc_adler32(accPath) checksum = 'ad:{0}'.format(checksum) if checksum == inFileInfo[tmpFileSpec.lfn]['checksum']: continue # make directories if needed if not os.path.isdir(os.path.dirname(accPath)): os.makedirs(os.path.dirname(accPath)) # make input for globus-url-copy if gucInput is None: gucInput = tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='_guc_in.tmp') gucInput.write("{0} {1}\n".format(srcPath, dstPath)) tmpFileSpec.attemptNr += 1 # nothing to transfer if gucInput is None: tmpLog.debug('done with no transfers') return True, '' # transfer tmpLog.debug('execute globus-url-copy') gucInput.close() args = ['globus-url-copy', '-f', gucInput.name, '-cd'] if self.gulOpts is not None: args += self.gulOpts.split() try: tmpLog.debug('execute: ' + ' '.join(args)) p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: stdout, stderr = p.communicate(timeout=self.timeout) except subprocess.TimeoutExpired: p.kill() stdout, stderr = p.communicate() tmpLog.warning('command timeout') return_code = p.returncode if stdout is not None: if not isinstance(stdout, str): stdout = stdout.decode() stdout = stdout.replace('\n', ' ') if stderr is not None: if not isinstance(stderr, str): stderr = stderr.decode() stderr = stderr.replace('\n', ' ') tmpLog.debug("stdout: %s" % stdout) tmpLog.debug("stderr: %s" % stderr) except Exception: core_utils.dump_error_message(tmpLog) return_code = 1 os.remove(gucInput.name) if return_code == 0: tmpLog.debug('succeeded') return True, '' else: errMsg = 'failed with {0}'.format(return_code) tmpLog.error(errMsg) # check attemptNr for tmpFileSpec in jobspec.inFiles: if tmpFileSpec.attemptNr >= self.maxAttempts: errMsg = 'gave up due to max attempts' tmpLog.error(errMsg) return (False, errMsg) return None, errMsg
try: import pty # pylint: disable=unused-import except ImportError: sys.exit('Unsupported operating system!') # version string __version__ = '0.3.3.post1' # install requires try: subprocess.check_call(['ps', 'axo', 'pid=,stat='], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) with subprocess.Popen(['yes'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) as pipe: subprocess.check_call(['pgrep', '-P', str(os.getpid())], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) pipe.terminate() pipe.kill() except subprocess.CalledProcessError: requirements = ['psutil'] else: requirements = None # README with open('README.rst', 'rb') as file: long_desc = file.read().decode('utf-8')
from easyprocess import EasyProcess import subprocess32 timeout = 3 print "#### use subprocess ####" # use subprocess # timeout can not set # stdout is display in realtime command = ['ping', 'localhost', '-t', '5'] child = subprocess.Popen(command) # s = child.communicate(timeout=timeout)[0] timeout can not set s = child.communicate()[0] print "#### use easyprocess ####" # use easyprocess # timeout can set # stdout is display when finished s = EasyProcess('ping localhost -t 5').call(timeout=timeout).stdout print(s) print "#### use subprocess32 ####" # use subprocess32 # timeout can set # stdout is display in realtime command = ['ping', 'localhost', '-t', '5'] child = subprocess32.Popen(command) try: s = child.communicate(timeout=timeout)[0] except subprocess32.TimeoutExpired: print "timeout"
def plot_raja_mmult(cali_file, baseFileName, cali_query, legend=True, title=True): ANNOTATIONS = ':'.join([ 'iteration', 'loop', 'size', 'depth', 'perm', 'mode', 'time.inclusive.duration' ]) CALI_QUERY = _get_cali_query(cali_query, ANNOTATIONS, cali_file) p = subprocess.Popen(CALI_QUERY, stdout=subprocess.PIPE) out, err = p.communicate() data = {} for line in out.split(): line = dict(map(lambda x: x.split('='), line.split(','))) if len(line) != 6: continue depth = line['depth'] mode = line['mode'] perm = line['perm'] size = int(line['size']) time = int(line['time.inclusive.duration']) if depth and perm and mode: if depth not in data: data[depth] = {} if perm not in data[depth]: data[depth][perm] = {} if mode not in data[depth][perm]: data[depth][perm][mode] = {} if size not in data[depth][perm][mode]: data[depth][perm][mode][size] = [] data[depth][perm][mode][size].append(time) colors = ['r', 'b', 'k', 'g', 'c', 'm'] markers = ('o', 'v', '8', 's', 'h', '*') namelist = (('OMP', 'RAJA w/ OpenMP'), ('Serial', 'RAJA Serial'), ('Agency', 'RAJA w/ Agency'), ('AgencyOMP', 'RAJA w/ Agency w/ OpenMP'), ('control', 'Control Serial'), ('rawOMP', 'Control OpenMP')) names = { key: (name, colors[i], markers[i]) for i, (key, name) in enumerate(namelist) } for depth, perm_dict in data.iteritems(): for perm, mode_dict in perm_dict.iteritems(): fig = plt.figure() legendNames = [] for i, (mode, datum) in enumerate(mode_dict.iteritems()): sizes = sorted(datum.keys()) xerr = [0] * len(sizes) means = [mean(datum[size]) for size in sizes] sems = [sem(datum[size]) for size in sizes] name, color, marker = names[mode] legendNames.append(name) plt.errorbar(sizes, means, xerr=xerr, yerr=sems, marker=marker, color=color) if legend: plt.legend(legendNames, loc='best', fontsize=12) if title: plt.title('Matrix Multiplication Performance', fontsize=22) plt.xlabel('Matrix Size', fontsize=16) plt.ylabel('Time Taken (ms)', fontsize=16) plt.yscale('log') plt.grid(b=True, which='major', color='k', linestyle='dotted') filename = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'figs', "{}_PERM_{}_Depth_{}.pdf".format(baseFileName, perm, depth)) plt.savefig(filename)
def word_to_pdf(in_file, in_format, out_file, pdfa=False, password=None, update_refs=False, tagged=False): tempdir = tempfile.mkdtemp() from_file = os.path.join(tempdir, "file." + in_format) to_file = os.path.join(tempdir, "file.pdf") shutil.copyfile(in_file, from_file) tries = 0 if pdfa: method = 'pdfa' elif tagged: method = 'tagged' else: method = 'default' while tries < 5: use_libreoffice = True if update_refs: if daconfig.get('convertapi secret', None) is not None: update_references(from_file) try: convertapi_to_pdf(from_file, to_file) result = 0 except: logmessage("Call to convertapi failed") result = 1 use_libreoffice = False else: subprocess_arguments = [ LIBREOFFICE_PATH, '--headless', '--invisible', 'macro:///Standard.Module1.ConvertToPdf(' + from_file + ',' + to_file + ',True,' + method + ')' ] elif daconfig.get('convertapi secret', None) is not None: try: convertapi_to_pdf(from_file, to_file) result = 0 except: logmessage("Call to convertapi failed") result = 1 use_libreoffice = False else: if method == 'default': subprocess_arguments = [ LIBREOFFICE_PATH, '--headless', '--convert-to', 'pdf', from_file ] else: subprocess_arguments = [ LIBREOFFICE_PATH, '--headless', '--invisible', 'macro:///Standard.Module1.ConvertToPdf(' + from_file + ',' + to_file + ',False,' + method + ')' ] if use_libreoffice: initialize_libreoffice() #logmessage("Trying libreoffice with " + repr(subprocess_arguments)) p = subprocess.Popen(subprocess_arguments, cwd=tempdir) result = p.wait() if os.path.isfile(to_file): break result = 1 tries += 1 time.sleep(2 + tries * random.random()) if use_libreoffice: logmessage("Retrying libreoffice with " + repr(subprocess_arguments)) else: logmessage("Retrying convertapi") continue if result == 0: if password: pdf_encrypt(to_file, password) shutil.copyfile(to_file, out_file) if tempdir is not None: shutil.rmtree(tempdir) if result != 0: return False return True
def plot_raja_reducer(cali_file, filename, cali_query): ANNOTATIONS = ':'.join([ 'iteration', 'loop', 'size', 'initialization', 'baseline', 'RajaSerial', 'OMP', 'AgencyParallel', 'AgencySerial', 'RawAgency', 'RawOMP', 'time.inclusive.duration' ]) CALI_QUERY = _get_cali_query(cali_query, ANNOTATIONS, cali_file) p = subprocess.Popen(CALI_QUERY, stdout=subprocess.PIPE) out, err = p.communicate() data = { 'init': {}, 'baseline': {}, 'OMP': {}, 'RajaSerial': {}, 'AgencyParallel': {}, 'AgencySerial': {}, 'RawAgency': {}, 'RawOMP': {} } for line in out.split(): line = dict(map(lambda x: x.split('='), line.split(','))) loop = None time = None mode = None size = None for key, value in line.iteritems(): if key == 'time.inclusive.duration': time = int(value) elif key == 'iteration': loop = int(value) elif key == 'size': size = int(value) else: mode = key if mode == 'initialization': data['init'][size] = time elif mode: if size not in data[mode]: data[mode][size] = [None] * 10 if size is not None and loop is not None: data[mode][size][loop] = time OMP = data['OMP'] control = data['baseline'] Serial = data['RajaSerial'] AgencyParallel = data['AgencyParallel'] AgencySerial = data['AgencySerial'] RawAgency = data['RawAgency'] RawOMP = data['RawOMP'] dat = [[size, c, omp, serial, agp, ags, ra, ro] for size in OMP for omp, serial, c, agp, ags, ra, ro in zip( OMP[size], Serial[size], control[size], AgencyParallel[size], AgencySerial[size], RawAgency[size], RawOMP[size])] dataframe = pd.DataFrame(data=dat, columns=[ 'Size', 'baseline', 'OMP', 'RajaSerial', 'AgencyParallel', 'AgencySerial', 'RawAgency', 'RawOMP' ]) d = [[ size, dataframe[dataframe['Size'] == size]['OMP'].mean(), stats.sem(dataframe[dataframe['Size'] == size]['OMP']), dataframe[dataframe['Size'] == size]['RajaSerial'].mean(), stats.sem(dataframe[dataframe['Size'] == size]['RajaSerial']), dataframe[dataframe['Size'] == size]['baseline'].mean(), stats.sem(dataframe[dataframe['Size'] == size]['baseline']), dataframe[dataframe['Size'] == size]['AgencyParallel'].mean(), stats.sem(dataframe[dataframe['Size'] == size]['AgencyParallel']), dataframe[dataframe['Size'] == size]['AgencySerial'].mean(), stats.sem(dataframe[dataframe['Size'] == size]['AgencySerial']), dataframe[dataframe['Size'] == size]['RawAgency'].mean(), stats.sem(dataframe[dataframe['Size'] == size]['RawAgency']), dataframe[dataframe['Size'] == size]['RawOMP'].mean(), stats.sem(dataframe[dataframe['Size'] == size]['RawOMP']) ] for size in sorted(OMP)] realDataframe = pd.DataFrame( data=d, columns=[ 'Size', 'OMPmean', 'OMPsem', 'RajaSerialmean', 'RajaSerialsem', 'baselinemean', 'baselinesem', 'AgencyParallelmean', 'AgencyParallelsem', 'AgencySerialmean', 'AgencySerialsem', 'RawAgencymean', 'RawAgencysem', 'RawOMPmean', 'RawOMPsem' ]) fig = plt.figure() sizes = sorted(OMP) legendNames = [] plt.errorbar(sizes, realDataframe['OMPmean'], color='k', xerr=[0] * len(sizes), yerr=realDataframe['OMPsem']) legendNames.append('OMP RAJA') plt.errorbar(sizes, realDataframe['RawOMPmean'], color='k', xerr=[0] * len(sizes), yerr=realDataframe['RawOMPsem'], linestyle='dashed') legendNames.append('Raw OMP') plt.errorbar(sizes, realDataframe['RajaSerialmean'], color='r', xerr=[0] * len(sizes), yerr=realDataframe['RajaSerialsem']) legendNames.append('Serial RAJA') plt.errorbar(sizes, realDataframe['baselinemean'], color='r', xerr=[0] * len(sizes), yerr=realDataframe['baselinesem'], linestyle='dashed') legendNames.append('Control') plt.errorbar(sizes, realDataframe['AgencySerialmean'], color='c', xerr=[0] * len(sizes), yerr=realDataframe['AgencySerialsem']) legendNames.append('Serial Agency RAJA') plt.errorbar(sizes, realDataframe['AgencyParallelmean'], color='b', xerr=[0] * len(sizes), yerr=realDataframe['AgencyParallelsem']) legendNames.append('Parallel Agency RAJA') plt.errorbar(sizes, realDataframe['RawAgencymean'], color='b', xerr=[0] * len(sizes), yerr=realDataframe['RawAgencysem'], linestyle='dashed') legendNames.append('Raw Agency') plt.legend(legendNames, loc='best', fontsize=8) plt.xlabel('Reduction Size', fontsize=16) plt.ylabel('Absolute Time', fontsize=16) plt.grid(b=True, which='major', color='k', linestyle='dotted') plt.title('Reducer Time Taken (32 cores)', fontsize=22) plt.yscale('log') plt.savefig(filename)
# Get the code totp = subprocess.check_output(['oathtool', '-b', '--totp', \ otpSecrets['otpsecrets'][args.label]]).rstrip('\n') # Is this a number? Print it. try: print '%s\t(%dsec)' % (totp, (30 - (datetime.now().second % 30))) # Try and put it on the clipboard unless disabled if 'use_clipboard' not in otpSecrets or otpSecrets[ 'use_clipboard'] != False: try: program = [ 'xclip', '-selection', 'clipboard' ] if platform.system() == 'Linux' else ['pbcopy'] process = subprocess.Popen(program, stdin=subprocess.PIPE) process.stdin.write(totp) process.stdin.close() except subprocess.CalledProcessError, err: print 'Couldn\'t put on the clipboard' # Wasn't parsed as int, but maybe the output is useful? except ValueError: print 'Output from oathtool doesn\'t seem to be valid' print 'Here\'s the output anyway:\n' print totp # Call to oathtool failed except subprocess.CalledProcessError, err: print err.output sys.exit(err.returncode) else: print 'Couldn\'t find label \'%s\' in the yaml. (Try the -l switch?)' % args.label
def start_job(self): self._logger.info("Starting Executable") job_error_msg = None job_template = self.job_template try: # create job UVE and log self.result_handler = JobResultHandler(self.job_template_id, self.job_execution_id, self.fabric_fq_name, self._logger, self.job_utils, self.job_log_utils, self.device_name, self.job_description, self.job_transaction_id, self.job_transaction_descr) msg = MsgBundle.getMessage(MsgBundle.START_JOB_MESSAGE, job_execution_id=self.job_execution_id, job_template_name=job_template.fq_name[ -1]) self._logger.debug(msg) timestamp = int(round(time.time() * 1000)) self.job_log_utils.send_job_log( job_template.fq_name, self.job_execution_id, self.fabric_fq_name, msg, JobStatus.STARTING.value, timestamp=timestamp, description=self.job_description, transaction_id=self.job_transaction_id, transaction_descr=self.job_transaction_descr) # validate job input if required by job_template input_schema input_schema = job_template.get_job_template_input_schema() if input_schema: self._validate_job_input(input_schema, self.job_data) executable_list = job_template.get_job_template_executables()\ .get_executable_info() for executable in executable_list: exec_path = executable.get_executable_path() executable.get_executable_args() job_input_args = self.gather_job_args() try: exec_process = subprocess32.Popen( [exec_path, "--job-input", json.dumps(job_input_args)], close_fds=True, cwd='/', stdout=subprocess32.PIPE, stderr=subprocess32.PIPE) self.job_file_write.write_to_file( self.job_execution_id, "job_summary", JobFileWrite.JOB_LOG, {"job_status": JobStatus.IN_PROGRESS.value}) msg = "Child process pid = " + str(exec_process.pid) self._logger.info(msg) (out, err) = exec_process.communicate( timeout=self.executable_timeout) self._logger.notice(str(out)) self._logger.notice(str(err)) timestamp = int(round(time.time() * 1000)) self.job_log_utils.send_job_log( job_template.fq_name, self.job_execution_id, self.fabric_fq_name, str(err), JobStatus.IN_PROGRESS.value, timestamp=timestamp, description=self.job_description, transaction_id=self.job_transaction_id, transaction_descr=self.job_transaction_descr) except subprocess32.TimeoutExpired as timeout_exp: if exec_process is not None: os.kill(exec_process.pid, 9) msg = MsgBundle.getMessage( MsgBundle.RUN_EXECUTABLE_PROCESS_TIMEOUT, exec_path=exec_path, exc_msg=repr(timeout_exp)) raise JobException(msg, self.job_execution_id) self._logger.info(exec_process.returncode) self._logger.info("Executable Completed") if exec_process.returncode != 0: job_status = JobStatus.FAILURE.value msg = MsgBundle.getMessage( MsgBundle.EXECUTABLE_RETURN_WITH_ERROR, exec_uri=exec_path) self._logger.error(msg) else: job_status = JobStatus.SUCCESS.value msg = MsgBundle.getMessage( MsgBundle.JOB_EXECUTION_COMPLETE, job_execution_id=self.job_execution_id, job_template_name=\ job_template.fq_name[-1]) self.job_file_write.write_to_file( self.job_execution_id, "job_summary", JobFileWrite.JOB_LOG, {"job_status": job_status}) self._logger.debug(msg) timestamp = int(round(time.time() * 1000)) self.job_log_utils.send_job_log( job_template.fq_name, self.job_execution_id, self.fabric_fq_name, msg, job_status, timestamp=timestamp, description=self.job_description, transaction_id=self.job_transaction_id, transaction_descr=self.job_transaction_descr) except JobException as exp: err_msg = "Job Exception recieved: %s " % repr(exp) self._logger.error(err_msg) self._logger.error("%s" % traceback.format_exc()) self.result_handler.update_job_status(JobStatus.FAILURE, err_msg) if job_template: self.result_handler.create_job_summary_log( job_template.fq_name) job_error_msg = err_msg except Exception as exp: err_msg = "Error while executing job %s " % repr(exp) self._logger.error(err_msg) self._logger.error("%s" % traceback.format_exc()) self.result_handler.update_job_status(JobStatus.FAILURE, err_msg) self.result_handler.create_job_summary_log(job_template.fq_name) job_error_msg = err_msg finally: # need to wait for the last job log and uve update to complete # via sandesh and then close sandesh connection sandesh_util = SandeshUtils(self._logger) sandesh_util.close_sandesh_connection() self._logger.info("Closed Sandesh connection") if job_error_msg is not None: sys.exit(job_error_msg)
def cutadaptit_pairs(data, sample): """ Applies trim & filters to pairs, including adapter detection. If we have barcode information then we use it to trim reversecut+bcode+adapter from reverse read, if not then we have to apply a more general cut to make sure we remove the barcode, this uses wildcards and so will have more false positives that trim a little extra from the ends of reads. Should we add a warning about this when filter_adapters=2 and no barcodes? """ LOGGER.debug("Entering cutadaptit_pairs - {}".format(sample.name)) sname = sample.name ## applied to read pairs #trim_r1 = str(data.paramsdict["edit_cutsites"][0]) #trim_r2 = str(data.paramsdict["edit_cutsites"][1]) finput_r1 = sample.files.concat[0][0] finput_r2 = sample.files.concat[0][1] ## Get adapter sequences. This is very important. For the forward adapter ## we don't care all that much about getting the sequence just before the ## Illumina adapter, b/c it will either be random (in RAD), or the reverse ## cut site of cut1 or cut2 (gbs or ddrad). Either way, we can still trim it ## off later in step7 with trim overhang if we want. And it should be invar- ## iable unless the cut site has an ambiguous char. The reverse adapter is ## super important, however b/c it can contain the inline barcode and ## revcomp cut site. We def want to trim out the barcode, and ideally the ## cut site too to be safe. Problem is we don't always know the barcode if ## users demultiplexed their data elsewhere. So, if barcode is missing we ## do a very fuzzy match before the adapter and trim it out. if not data.barcodes: ## try linking barcodes again in case user just added a barcodes path ## after receiving the warning try: data._link_barcodes() except Exception as inst: LOGGER.warning(" error adding barcodes info: %s", inst) if data.barcodes: try: adapter1 = fullcomp(data.paramsdict["restriction_overhang"][1])[::-1]+\ data._hackersonly["p3_adapter"] adapter2 = fullcomp(data.paramsdict["restriction_overhang"][0])[::-1]+\ fullcomp(data.barcodes[sample.name])[::-1]+\ data._hackersonly["p5_adapter"] ## <- should this be revcomp? except KeyError as inst: msg = """ Sample name does not exist in the barcode file. The name in the barcode file for each sample must exactly equal the raw file name for the sample minus `_R1`. So for example a sample called WatDo_PipPrep_R1_100.fq.gz must be referenced in the barcode file as WatDo_PipPrep_100. The name in your barcode file for this sample must match: {} """.format(sample.name) LOGGER.error(msg) raise IPyradWarningExit(msg) else: print(NO_BARS_GBS_WARNING) #adapter1 = fullcomp(data.paramsdict["restriction_overhang"][1])[::-1]+\ # data._hackersonly["p3_adapter"] #adapter2 = "XXX" adapter1 = data._hackersonly["p3_adapter"] adapter2 = fullcomp(data._hackersonly["p5_adapter"]) ## parse trim_reads trim5r1 = trim5r2 = trim3r1 = trim3r2 = [] if data.paramsdict.get("trim_reads"): trimlen = data.paramsdict.get("trim_reads") ## trim 5' end if trimlen[0]: trim5r1 = ["-u", str(trimlen[0])] if trimlen[1] < 0: trim3r1 = ["-u", str(trimlen[1])] if trimlen[1] > 0: trim3r1 = ["--length", str(trimlen[1])] ## legacy support for trimlen = 0,0 default if len(trimlen) > 2: if trimlen[2]: trim5r2 = ["-U", str(trimlen[2])] if len(trimlen) > 3: if trimlen[3]: if trimlen[3] < 0: trim3r2 = ["-U", str(trimlen[3])] if trimlen[3] > 0: trim3r2 = ["--length", str(trimlen[3])] else: ## legacy support trimlen = data.paramsdict.get("edit_cutsites") trim5r1 = ["-u", str(trimlen[0])] trim5r2 = ["-U", str(trimlen[1])] ## testing new 'trim_reads' setting cmdf1 = ["cutadapt"] if trim5r1: cmdf1 += trim5r1 if trim3r1: cmdf1 += trim3r1 if trim5r2: cmdf1 += trim5r2 if trim3r2: cmdf1 += trim3r2 cmdf1 += ["--trim-n", "--max-n", str(data.paramsdict["max_low_qual_bases"]), "--minimum-length", str(data.paramsdict["filter_min_trim_len"]), "-o", OPJ(data.dirs.edits, sname+".trimmed_R1_.fastq.gz"), "-p", OPJ(data.dirs.edits, sname+".trimmed_R2_.fastq.gz"), finput_r1, finput_r2] ## additional args if int(data.paramsdict["filter_adapters"]) < 2: ## add a dummy adapter to let cutadapt know whe are not using legacy-mode cmdf1.insert(1, "XXX") cmdf1.insert(1, "-A") if int(data.paramsdict["filter_adapters"]): cmdf1.insert(1, "20,20") cmdf1.insert(1, "-q") cmdf1.insert(1, str(data.paramsdict["phred_Qscore_offset"])) cmdf1.insert(1, "--quality-base") if int(data.paramsdict["filter_adapters"]) > 1: ## first enter extra cuts zcut1 = data._hackersonly["p3_adapters_extra"][::-1] zcut2 = data._hackersonly["p5_adapters_extra"][::-1] for ecut1, ecut2 in zip(zcut1, zcut2): cmdf1.insert(1, ecut1) cmdf1.insert(1, "-a") cmdf1.insert(1, ecut2) cmdf1.insert(1, "-A") ## then put the main cut first cmdf1.insert(1, adapter1) cmdf1.insert(1, '-a') cmdf1.insert(1, adapter2) cmdf1.insert(1, '-A') ## do modifications to read1 and write to tmp file LOGGER.debug(" ".join(cmdf1)) #sys.exit() try: proc1 = sps.Popen(cmdf1, stderr=sps.STDOUT, stdout=sps.PIPE, close_fds=True) res1 = proc1.communicate()[0] except KeyboardInterrupt: proc1.kill() LOGGER.info("this is where I want it to interrupt") raise KeyboardInterrupt() ## raise errors if found if proc1.returncode: raise IPyradWarningExit(" error [returncode={}]: {}\n{}".format(proc1.returncode, " ".join(cmdf1), res1)) LOGGER.debug("Exiting cutadaptit_pairs - {}".format(sname)) ## return results string to be parsed outside of engine return res1