class CppSim(ProcessWorkerThread): def handle_eval(self, record): self.process = Popen( ['./sumfun_ext', array2str(record.params[0])], stdout=PIPE) val = np.nan # Continuously check for new outputs from the subprocess while True: output = self.process.stdout.readline() if output == '' and self.process.poll( ) is not None: # No new output break if output: # New intermediate output try: val = float(output.strip()) # Try to parse output if val > 350: # Terminate if too large self.process.terminate() self.finish_success(record, 350) return except ValueError: # If the output is nonsense we terminate logging.warning("Incorrect output") self.process.terminate() self.finish_cancelled(record) return rc = self.process.poll() # Check the return code if rc < 0 or np.isnan(val): logging.info("WARNING: Incorrect output or crashed evaluation") self.finish_cancelled(record) else: self.finish_success(record, val)
def run_cmd(cmd): # c = [] # for cm in cmd[:-1]: # if ' ' in cm: # c.append('"' + cm + '"') # else: # c.append(cm) c = cmd[:-1] timeout = cmd[-1] display("Executing command: %s" % " ".join(c)) current_time = time.time() if timeout: process = Popen(c) while time.time() < current_time + timeout and process.poll() is None: time.sleep(5) if process.poll() is None: display_error( "Timeout of %s reached. Aborting thread for command: %s" % (timeout, " ".join(c)) ) process.terminate() else: Popen(c).wait() return cmd
def run_playbook(playbook, inventory, *args, **kwargs): env = ansible_env(os.environ.copy()) cmd = ['ansible-playbook', '-i', inventory, playbook] + list(args) if verbosity(): cmd += ['-' + ('v' * verbosity())] show_timestamp = False if 'timestamp' in kwargs: show_timestamp = kwargs['timestamp'] del kwargs['timestamp'] output = print if show_timestamp: output = timestamp logger.info('running %s', ' '.join(cmd)) logger.debug('env: %r', env) process = Popen(cmd, env=env, stdout=PIPE, bufsize=1, **kwargs) for line in iter(process.stdout.readline, b''): output(line[:-1]) # empty output buffers process.poll() return process.returncode
def run_cmd_noout(cmd_data): cmd = cmd_data[0] output = cmd_data[1] c = cmd[:-1] timeout = cmd[-1] display("Executing command: %s" % " ".join(c)) current_time = time.time() f = open(output, 'w') if timeout: process = Popen(c, stdout=f, stderr=STDOUT) while time.time() < current_time + timeout and process.poll() is None: time.sleep(5) if process.poll() is None: display_error( "Timeout of %s reached. Aborting thread for command: %s" % (timeout, " ".join(c)) ) process.terminate() else: Popen(c, stdout=f, stderr=STDOUT).wait() f.close() return cmd_data
class DummySim(ProcessWorkerThread): def handle_eval(self, record): self.process = Popen(['./sumfun_ext', array2str(record.params[0])], stdout=PIPE) val = np.nan # Continuously check for new outputs from the subprocess while True: output = self.process.stdout.readline() if output == '' and self.process.poll() is not None: # No new output break if output: # New intermediate output try: val = float(output.strip()) # Try to parse output if val > 350: # Terminate if too large self.process.terminate() self.finish_success(record, 350) return except ValueError: # If the output is nonsense we terminate logging.warning("Incorrect output") self.process.terminate() self.finish_failure(record) return rc = self.process.poll() # Check the return code if rc < 0 or np.isnan(val): logging.warning("Incorrect output or crashed evaluation") self.finish_failure(record) else: self.finish_success(record, val)
def run(pro, *args, **kwargs): """ Run vagrant within a project :param pro: .project.Project :param args: list[string] :param kwargs: dict[string,string] :return: """ with cd(pro.folder()): # fix invalid exports for vagrant NFS().fix_anomalies() new_env = ansible_env(os.environ.copy()) new_env['PATH'] = os.pathsep.join([ new_env['PATH'], os.path.join(aeriscloud_path, 'venv/bin') ]) new_env['VAGRANT_DOTFILE_PATH'] = pro.vagrant_dir() new_env['VAGRANT_CWD'] = pro.vagrant_working_dir() new_env['VAGRANT_DISKS_PATH'] = os.path.join(data_dir(), 'disks') # We might want to remove that or bump the verbosity level even more if verbosity() >= 4: new_env['VAGRANT_LOG'] = 'info' new_env['AERISCLOUD_PATH'] = aeriscloud_path new_env['AERISCLOUD_ORGANIZATIONS_DIR'] = os.path.join(data_dir(), 'organizations') org = default_organization() if org: new_env['AERISCLOUD_DEFAULT_ORGANIZATION'] = org organization_name = pro.organization() if organization_name: organization = Organization(organization_name) else: organization = Organization(org) basebox_url = organization.basebox_url() if basebox_url: new_env['VAGRANT_SERVER_URL'] = basebox_url args = ['vagrant'] + list(args) logger.debug('running: %s\nenv: %r', ' '.join(args), new_env) # support for the vagrant prompt if args[1] == 'destroy': return call(args, env=new_env, **kwargs) else: process = Popen(args, env=new_env, stdout=PIPE, bufsize=1, **kwargs) for line in iter(process.stdout.readline, b''): timestamp(line[:-1]) # empty output buffers process.poll() return process.returncode
class CppSim(ProcessWorkerThread): def handle_eval(self, record): val = np.nan # Continuously check for new outputs from the subprocess self.process = Popen(['./sumfun_ext', array2str(record.params[0])], stdout=PIPE, bufsize=1, universal_newlines=True) for line in self.process.stdout: try: val = float(line.strip()) # Try to parse output if val > 350: # Terminate if too large self.process.terminate() self.finish_success(record, 350) return except ValueError: # If the output is nonsense we terminate logging.warning("Incorrect output") self.process.terminate() self.finish_cancelled(record) return self.process.wait() rc = self.process.poll() # Check the return code if rc < 0 or np.isnan(val): logging.info("WARNING: Incorrect output or crashed evaluation") self.finish_cancelled(record) else: self.finish_success(record, val)
def runcmd(cmd, input=None, stringio=True, **kwargs): if six.PY2: from subprocess32 import Popen, PIPE else: from subprocess import Popen, PIPE timeout = kwargs.pop('timeout', None) if input: kwargs['stdin'] = PIPE if not 'bufsize' in kwargs: kwargs['bufsize']= -1 if isinstance(cmd, six.string_types): import shlex cmd = shlex.split(cmd) elif kwargs.get('shell'): from six.moves import shlex_quote cmd = [shlex_quote(arg) for arg in cmd] process = Popen(cmd,universal_newlines=stringio,stdout=PIPE,stderr=PIPE,**kwargs) try: output, error = process.communicate(input=input, timeout=timeout) except TimeoutExpired: process.kill() output, error = process.communicate() raise TimeoutExpired(process.args, timeout, output=output) retcode = process.poll() if retcode: raise SubProcessError(retcode, process.args, output=output, error=error) return output, error
def _run_packer_cmd(self, cmd): """ Triggers the Packer command line :param cmd: command arguments :return: The result of the command execution """ result = "" packer_env = os.environ.copy() if not os.path.isdir(PACKER_LOGDIR): os.makedirs(PACKER_LOGDIR) packer_env['TMPDIR'] = PACKER_LOGDIR if self._assumed_role: packer_env['AWS_ACCESS_KEY_ID'] = self._aws_credentials[ 'aws_access_key'] packer_env['AWS_SECRET_ACCESS_KEY'] = self._aws_credentials[ 'aws_secret_key'] packer_env['AWS_SESSION_TOKEN'] = self._aws_credentials['token'] packer_env['AWS_SECURITY_TOKEN'] = self._aws_credentials['token'] process = Popen(cmd, stdout=PIPE, env=packer_env) while True: output = process.stdout.readline() if output == '' and process.poll() is not None: break if output: out_tab = output.strip().split(',') if len(out_tab) > 3: ts = out_tab[0] target = out_tab[1] msg_type = out_tab[2] data = out_tab[3] if msg_type == "ui" and len(out_tab) > 4: log("{0}".format(out_tab[4]), self._log_file) elif msg_type == "artifact": if len(out_tab) > 4 and out_tab[4] == "id": result = out_tab[5] log("AMI: {0}".format(result), self._log_file) else: log("{0}: {1}".format(msg_type, data), self._log_file) rc = process.poll() return rc, result
def run_compiler(self, language, filename, executable_name): args = ["g++" if language else "gcc", "-static", "-w", "-O2", filename, "-o", executable_name] self.log += ['Running: ' + ' '.join(args)] proc = Popen(args, cwd=self.base_dir, stdin=PIPE, stdout=PIPE, stderr=PIPE) output = proc.communicate(timeout=self.COMPILE_TIMEOUT) self.log += [str(output[1])] if proc.poll() is None: try: self.log += ['Compile timeout.'] proc.kill() except Exception: pass self.log += ["Compiler returns %d." % proc.returncode] if proc.returncode: raise CompileErrorException()
def check_output(*popenargs, **kwargs): """ Re-implement check_output from subprocess32, but with a timeout that kills child processes. See https://github.com/google/python-subprocess32/blob/master/subprocess32.py#L606 """ timeout = kwargs.pop('timeout', None) if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it will be overridden.') process = Popen(stdout=PIPE, preexec_fn=os.setsid, *popenargs, **kwargs) try: output = process.communicate(timeout=timeout)[0] except TimeoutExpired as error: os.killpg(process.pid, signal.SIGINT) raise error retcode = process.poll() if retcode: raise CalledProcessError(retcode, process.args, output=output) return output
def check_output(*popenargs, **kwargs): r"""Run command with arguments and return its output as a byte string. If the exit code was non-zero it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute and output in the output attribute. The arguments are the same as for the Popen constructor. Example: >>> check_output(["ls", "-l", "/dev/null"]) 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' The stdout argument is not allowed as it is used internally. To capture standard error in the result, use stderr=STDOUT. >>> check_output(["/bin/sh", "-c", ... "ls -l non_existent_file ; exit 0"], ... stderr=STDOUT) 'ls: non_existent_file: No such file or directory\n' """ timeout = kwargs.pop('timeout', None) if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it will be overridden.') if _kill_processes.is_set(): raise TerminateSignaled() process = Popen(stdout=PIPE, *popenargs, **kwargs) _processes.append(process) try: output, unused_err = process.communicate(timeout=timeout) _processes.remove(process) except TimeoutExpired: _processes.remove(process) process.kill() output, unused_err = process.communicate() raise TimeoutExpired(process.args, timeout, output=output) retcode = process.poll() if retcode: raise CalledProcessError(retcode, process.args, output=output) return output
def run_compiler(self, language, filename, executable_name): args = [ "g++" if language else "gcc", "-static", "-w", "-O2", filename, "-o", executable_name ] self.log += ['Running: ' + ' '.join(args)] proc = Popen(args, cwd=self.base_dir, stdin=PIPE, stdout=PIPE, stderr=PIPE) output = proc.communicate(timeout=self.COMPILE_TIMEOUT) self.log += [str(output[1])] if proc.poll() is None: try: self.log += ['Compile timeout.'] proc.kill() except Exception: pass self.log += ["Compiler returns %d." % proc.returncode] if proc.returncode: raise CompileErrorException()
def run2(command, check=True, timeout=None, *args, **kwargs): ''' Run a command. If check=True (the default), then if return code is not zero or there is stderr output, raise CalledProcessError. Return any output in the exception. If timeout (in seconds) is set and command times out, raise TimeoutError. ''' ''' Parts from subprocess32.check_output(). ''' raise Exception('Deprecated. Use the sh module.') # use subprocess32 for timeout from subprocess32 import Popen, CalledProcessError, TimeoutExpired process = Popen(command, stdout=stdout, stderr=stderr, *args, **kwargs) try: process.wait(timeout=timeout) except TimeoutExpired: print('TimeoutExpired') #DEBUG #print('stdout: %s, (%d)' % (str(stdout), len(str(stdout)))) #DEBUG #print('stderr: %s, (%d)' % (str(stderr), len(str(stderr)))) #DEBUG try: process.kill() process.wait() finally: print('after kill/wait') #DEBUG #print('stdout: %s, (%d)' % (str(stdout), len(str(stdout)))) #DEBUG #print('stderr: %s, (%d)' % (str(stderr), len(str(stderr)))) #DEBUG raise TimeoutExpired(process.args, timeout) if check: retcode = process.poll() if retcode: raise CalledProcessError(retcode, process.args)
except (KeyboardInterrupt): print(" ") print("Terminating Program...") heat_process.kill() heat_process.wait() hum_process.kill() hum_process.wait() fan_process.kill() fan_process.wait() light_camera_process.kill() light_camera_process.wait() sys.exit() except: pass #poll subprocesses if applicable and relaunch/update actuators poll_heat = heat_process.poll() #heat if poll_heat is not None: heat_process = Popen(['python', 'heatingElement.py', str(tempPID_out)], stdout=PIPE, stdin=PIPE, stderr=PIPE) poll_hum = hum_process.poll() #hum if poll_hum is not None: hum_process = Popen(['python', 'humidityElement.py', str(humPID_out)], stdout=PIPE, stdin=PIPE, stderr=PIPE) poll_fan = fan_process.poll() #fan if poll_fan is not None: fan_process = Popen(['python', 'fanElement.py', str(fanPD_out)], stdout=PIPE, stdin=PIPE, stderr=PIPE) poll_light_camera = light_camera_process.poll() #light if poll_light_camera is not None: light_camera_process = Popen(['python', 'lightingCameraElement.py', str(targetL), str(LtimeOn), str(LtimeOff), str(lightCameraInterval)], stdout=PIPE, stdin=PIPE, stderr=PIPE)
fan_process.kill() fan_process.wait() light_process.kill() light_process.wait() camera_process.kill() camera_process.wait() water_process.kill() water_process.wait() sys.exit() except Exception as e: print(e) pass #poll subprocesses if applicable and relaunch/update actuators poll_heat = heat_process.poll() #heat if poll_heat is not None: heat_process = Popen(['python3', 'heatingElement.py', str(tempPID_out)], stdout=PIPE, stdin=PIPE, stderr=PIPE) poll_hum = hum_process.poll() #hum if poll_hum is not None: hum_process = Popen(['python3', 'humidityElement.py', str(humPID_out)], stdout=PIPE, stdin=PIPE, stderr=PIPE) poll_fan = fan_process.poll() #fan if poll_fan is not None: fan_process = Popen(['python3', 'fanElement.py', str(fanPD_out)], stdout=PIPE, stdin=PIPE, stderr=PIPE) poll_light = light_process.poll() #light if poll_light is not None: light_process = Popen(['python3', 'lightingElement.py', str(targetL), str(LtimeOn), str(LtimeOff), str(lightInterval)], stdout=PIPE, stdin=PIPE, stderr=PIPE)