class CppSim(ProcessWorkerThread):
def handle_eval(self, record):
self.process = Popen(
['./sumfun_ext', array2str(record.params[0])], stdout=PIPE)
val = np.nan
# Continuously check for new outputs from the subprocess
while True:
output = self.process.stdout.readline()
if output == '' and self.process.poll(
) is not None: # No new output
break
if output: # New intermediate output
try:
val = float(output.strip()) # Try to parse output
if val > 350: # Terminate if too large
self.process.terminate()
self.finish_success(record, 350)
return
except ValueError: # If the output is nonsense we terminate
logging.warning("Incorrect output")
self.process.terminate()
self.finish_cancelled(record)
return
rc = self.process.poll() # Check the return code
if rc < 0 or np.isnan(val):
logging.info("WARNING: Incorrect output or crashed evaluation")
self.finish_cancelled(record)
else:
self.finish_success(record, val)
def run_cmd(cmd):
# c = []
# for cm in cmd[:-1]:
# if ' ' in cm:
# c.append('"' + cm + '"')
# else:
# c.append(cm)
c = cmd[:-1]
timeout = cmd[-1]
display("Executing command: %s" % " ".join(c))
current_time = time.time()
if timeout:
process = Popen(c)
while time.time() < current_time + timeout and process.poll() is None:
time.sleep(5)
if process.poll() is None:
display_error(
"Timeout of %s reached. Aborting thread for command: %s"
% (timeout, " ".join(c))
)
process.terminate()
else:
Popen(c).wait()
return cmd
def run_playbook(playbook, inventory, *args, **kwargs):
env = ansible_env(os.environ.copy())
cmd = ['ansible-playbook', '-i', inventory, playbook] + list(args)
if verbosity():
cmd += ['-' + ('v' * verbosity())]
show_timestamp = False
if 'timestamp' in kwargs:
show_timestamp = kwargs['timestamp']
del kwargs['timestamp']
output = print
if show_timestamp:
output = timestamp
logger.info('running %s', ' '.join(cmd))
logger.debug('env: %r', env)
process = Popen(cmd, env=env, stdout=PIPE,
bufsize=1, **kwargs)
for line in iter(process.stdout.readline, b''):
output(line[:-1])
# empty output buffers
process.poll()
return process.returncode
def run_cmd_noout(cmd_data):
cmd = cmd_data[0]
output = cmd_data[1]
c = cmd[:-1]
timeout = cmd[-1]
display("Executing command: %s" % " ".join(c))
current_time = time.time()
f = open(output, 'w')
if timeout:
process = Popen(c, stdout=f, stderr=STDOUT)
while time.time() < current_time + timeout and process.poll() is None:
time.sleep(5)
if process.poll() is None:
display_error(
"Timeout of %s reached. Aborting thread for command: %s"
% (timeout, " ".join(c))
)
process.terminate()
else:
Popen(c, stdout=f, stderr=STDOUT).wait()
f.close()
return cmd_data
class DummySim(ProcessWorkerThread):
def handle_eval(self, record):
self.process = Popen(['./sumfun_ext', array2str(record.params[0])],
stdout=PIPE)
val = np.nan
# Continuously check for new outputs from the subprocess
while True:
output = self.process.stdout.readline()
if output == '' and self.process.poll() is not None: # No new output
break
if output: # New intermediate output
try:
val = float(output.strip()) # Try to parse output
if val > 350: # Terminate if too large
self.process.terminate()
self.finish_success(record, 350)
return
except ValueError: # If the output is nonsense we terminate
logging.warning("Incorrect output")
self.process.terminate()
self.finish_failure(record)
return
rc = self.process.poll() # Check the return code
if rc < 0 or np.isnan(val):
logging.warning("Incorrect output or crashed evaluation")
self.finish_failure(record)
else:
self.finish_success(record, val)
def run(pro, *args, **kwargs):
"""
Run vagrant within a project
:param pro: .project.Project
:param args: list[string]
:param kwargs: dict[string,string]
:return:
"""
with cd(pro.folder()):
# fix invalid exports for vagrant
NFS().fix_anomalies()
new_env = ansible_env(os.environ.copy())
new_env['PATH'] = os.pathsep.join([
new_env['PATH'],
os.path.join(aeriscloud_path, 'venv/bin')
])
new_env['VAGRANT_DOTFILE_PATH'] = pro.vagrant_dir()
new_env['VAGRANT_CWD'] = pro.vagrant_working_dir()
new_env['VAGRANT_DISKS_PATH'] = os.path.join(data_dir(), 'disks')
# We might want to remove that or bump the verbosity level even more
if verbosity() >= 4:
new_env['VAGRANT_LOG'] = 'info'
new_env['AERISCLOUD_PATH'] = aeriscloud_path
new_env['AERISCLOUD_ORGANIZATIONS_DIR'] = os.path.join(data_dir(),
'organizations')
org = default_organization()
if org:
new_env['AERISCLOUD_DEFAULT_ORGANIZATION'] = org
organization_name = pro.organization()
if organization_name:
organization = Organization(organization_name)
else:
organization = Organization(org)
basebox_url = organization.basebox_url()
if basebox_url:
new_env['VAGRANT_SERVER_URL'] = basebox_url
args = ['vagrant'] + list(args)
logger.debug('running: %s\nenv: %r', ' '.join(args), new_env)
# support for the vagrant prompt
if args[1] == 'destroy':
return call(args, env=new_env, **kwargs)
else:
process = Popen(args, env=new_env, stdout=PIPE,
bufsize=1, **kwargs)
for line in iter(process.stdout.readline, b''):
timestamp(line[:-1])
# empty output buffers
process.poll()
return process.returncode
class CppSim(ProcessWorkerThread):
def handle_eval(self, record):
val = np.nan
# Continuously check for new outputs from the subprocess
self.process = Popen(['./sumfun_ext', array2str(record.params[0])], stdout=PIPE, bufsize=1, universal_newlines=True)
for line in self.process.stdout:
try:
val = float(line.strip()) # Try to parse output
if val > 350: # Terminate if too large
self.process.terminate()
self.finish_success(record, 350)
return
except ValueError: # If the output is nonsense we terminate
logging.warning("Incorrect output")
self.process.terminate()
self.finish_cancelled(record)
return
self.process.wait()
rc = self.process.poll() # Check the return code
if rc < 0 or np.isnan(val):
logging.info("WARNING: Incorrect output or crashed evaluation")
self.finish_cancelled(record)
else:
self.finish_success(record, val)
def runcmd(cmd, input=None, stringio=True, **kwargs):
if six.PY2:
from subprocess32 import Popen, PIPE
else:
from subprocess import Popen, PIPE
timeout = kwargs.pop('timeout', None)
if input: kwargs['stdin'] = PIPE
if not 'bufsize' in kwargs: kwargs['bufsize']= -1
if isinstance(cmd, six.string_types):
import shlex
cmd = shlex.split(cmd)
elif kwargs.get('shell'):
from six.moves import shlex_quote
cmd = [shlex_quote(arg) for arg in cmd]
process = Popen(cmd,universal_newlines=stringio,stdout=PIPE,stderr=PIPE,**kwargs)
try:
output, error = process.communicate(input=input, timeout=timeout)
except TimeoutExpired:
process.kill()
output, error = process.communicate()
raise TimeoutExpired(process.args, timeout, output=output)
retcode = process.poll()
if retcode:
raise SubProcessError(retcode, process.args, output=output, error=error)
return output, error
def _run_packer_cmd(self, cmd):
"""
Triggers the Packer command line
:param cmd: command arguments
:return: The result of the command execution
"""
result = ""
packer_env = os.environ.copy()
if not os.path.isdir(PACKER_LOGDIR):
os.makedirs(PACKER_LOGDIR)
packer_env['TMPDIR'] = PACKER_LOGDIR
if self._assumed_role:
packer_env['AWS_ACCESS_KEY_ID'] = self._aws_credentials[
'aws_access_key']
packer_env['AWS_SECRET_ACCESS_KEY'] = self._aws_credentials[
'aws_secret_key']
packer_env['AWS_SESSION_TOKEN'] = self._aws_credentials['token']
packer_env['AWS_SECURITY_TOKEN'] = self._aws_credentials['token']
process = Popen(cmd, stdout=PIPE, env=packer_env)
while True:
output = process.stdout.readline()
if output == '' and process.poll() is not None:
break
if output:
out_tab = output.strip().split(',')
if len(out_tab) > 3:
ts = out_tab[0]
target = out_tab[1]
msg_type = out_tab[2]
data = out_tab[3]
if msg_type == "ui" and len(out_tab) > 4:
log("{0}".format(out_tab[4]), self._log_file)
elif msg_type == "artifact":
if len(out_tab) > 4 and out_tab[4] == "id":
result = out_tab[5]
log("AMI: {0}".format(result), self._log_file)
else:
log("{0}: {1}".format(msg_type, data), self._log_file)
rc = process.poll()
return rc, result
def run_compiler(self, language, filename, executable_name):
args = ["g++" if language else "gcc", "-static", "-w", "-O2", filename, "-o",
executable_name]
self.log += ['Running: ' + ' '.join(args)]
proc = Popen(args,
cwd=self.base_dir, stdin=PIPE, stdout=PIPE, stderr=PIPE)
output = proc.communicate(timeout=self.COMPILE_TIMEOUT)
self.log += [str(output[1])]
if proc.poll() is None:
try:
self.log += ['Compile timeout.']
proc.kill()
except Exception:
pass
self.log += ["Compiler returns %d." % proc.returncode]
if proc.returncode:
raise CompileErrorException()
def check_output(*popenargs, **kwargs):
"""
Re-implement check_output from subprocess32, but with a timeout that kills
child processes.
See https://github.com/google/python-subprocess32/blob/master/subprocess32.py#L606
"""
timeout = kwargs.pop('timeout', None)
if 'stdout' in kwargs:
raise ValueError('stdout argument not allowed, it will be overridden.')
process = Popen(stdout=PIPE, preexec_fn=os.setsid, *popenargs, **kwargs)
try:
output = process.communicate(timeout=timeout)[0]
except TimeoutExpired as error:
os.killpg(process.pid, signal.SIGINT)
raise error
retcode = process.poll()
if retcode:
raise CalledProcessError(retcode, process.args, output=output)
return output
def check_output(*popenargs, **kwargs):
r"""Run command with arguments and return its output as a byte string.
If the exit code was non-zero it raises a CalledProcessError. The
CalledProcessError object will have the return code in the returncode
attribute and output in the output attribute.
The arguments are the same as for the Popen constructor. Example:
>>> check_output(["ls", "-l", "/dev/null"])
'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
The stdout argument is not allowed as it is used internally.
To capture standard error in the result, use stderr=STDOUT.
>>> check_output(["/bin/sh", "-c",
... "ls -l non_existent_file ; exit 0"],
... stderr=STDOUT)
'ls: non_existent_file: No such file or directory\n'
"""
timeout = kwargs.pop('timeout', None)
if 'stdout' in kwargs:
raise ValueError('stdout argument not allowed, it will be overridden.')
if _kill_processes.is_set():
raise TerminateSignaled()
process = Popen(stdout=PIPE, *popenargs, **kwargs)
_processes.append(process)
try:
output, unused_err = process.communicate(timeout=timeout)
_processes.remove(process)
except TimeoutExpired:
_processes.remove(process)
process.kill()
output, unused_err = process.communicate()
raise TimeoutExpired(process.args, timeout, output=output)
retcode = process.poll()
if retcode:
raise CalledProcessError(retcode, process.args, output=output)
return output
def run_compiler(self, language, filename, executable_name):
args = [
"g++" if language else "gcc", "-static", "-w", "-O2", filename,
"-o", executable_name
]
self.log += ['Running: ' + ' '.join(args)]
proc = Popen(args,
cwd=self.base_dir,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
output = proc.communicate(timeout=self.COMPILE_TIMEOUT)
self.log += [str(output[1])]
if proc.poll() is None:
try:
self.log += ['Compile timeout.']
proc.kill()
except Exception:
pass
self.log += ["Compiler returns %d." % proc.returncode]
if proc.returncode:
raise CompileErrorException()
def run2(command, check=True, timeout=None, *args, **kwargs):
''' Run a command.
If check=True (the default),
then if return code is not zero or there is stderr output,
raise CalledProcessError. Return any output in the exception.
If timeout (in seconds) is set and command times out, raise TimeoutError. '''
''' Parts from subprocess32.check_output(). '''
raise Exception('Deprecated. Use the sh module.')
# use subprocess32 for timeout
from subprocess32 import Popen, CalledProcessError, TimeoutExpired
process = Popen(command, stdout=stdout, stderr=stderr, *args, **kwargs)
try:
process.wait(timeout=timeout)
except TimeoutExpired:
print('TimeoutExpired') #DEBUG
#print('stdout: %s, (%d)' % (str(stdout), len(str(stdout)))) #DEBUG
#print('stderr: %s, (%d)' % (str(stderr), len(str(stderr)))) #DEBUG
try:
process.kill()
process.wait()
finally:
print('after kill/wait') #DEBUG
#print('stdout: %s, (%d)' % (str(stdout), len(str(stdout)))) #DEBUG
#print('stderr: %s, (%d)' % (str(stderr), len(str(stderr)))) #DEBUG
raise TimeoutExpired(process.args, timeout)
if check:
retcode = process.poll()
if retcode:
raise CalledProcessError(retcode, process.args)
except (KeyboardInterrupt):
print(" ")
print("Terminating Program...")
heat_process.kill()
heat_process.wait()
hum_process.kill()
hum_process.wait()
fan_process.kill()
fan_process.wait()
light_camera_process.kill()
light_camera_process.wait()
sys.exit()
except:
pass
#poll subprocesses if applicable and relaunch/update actuators
poll_heat = heat_process.poll() #heat
if poll_heat is not None:
heat_process = Popen(['python', 'heatingElement.py', str(tempPID_out)], stdout=PIPE, stdin=PIPE, stderr=PIPE)
poll_hum = hum_process.poll() #hum
if poll_hum is not None:
hum_process = Popen(['python', 'humidityElement.py', str(humPID_out)], stdout=PIPE, stdin=PIPE, stderr=PIPE)
poll_fan = fan_process.poll() #fan
if poll_fan is not None:
fan_process = Popen(['python', 'fanElement.py', str(fanPD_out)], stdout=PIPE, stdin=PIPE, stderr=PIPE)
poll_light_camera = light_camera_process.poll() #light
if poll_light_camera is not None:
light_camera_process = Popen(['python', 'lightingCameraElement.py', str(targetL), str(LtimeOn), str(LtimeOff), str(lightCameraInterval)], stdout=PIPE, stdin=PIPE, stderr=PIPE)
fan_process.kill()
fan_process.wait()
light_process.kill()
light_process.wait()
camera_process.kill()
camera_process.wait()
water_process.kill()
water_process.wait()
sys.exit()
except Exception as e:
print(e)
pass
#poll subprocesses if applicable and relaunch/update actuators
poll_heat = heat_process.poll() #heat
if poll_heat is not None:
heat_process = Popen(['python3', 'heatingElement.py', str(tempPID_out)], stdout=PIPE, stdin=PIPE, stderr=PIPE)
poll_hum = hum_process.poll() #hum
if poll_hum is not None:
hum_process = Popen(['python3', 'humidityElement.py', str(humPID_out)], stdout=PIPE, stdin=PIPE, stderr=PIPE)
poll_fan = fan_process.poll() #fan
if poll_fan is not None:
fan_process = Popen(['python3', 'fanElement.py', str(fanPD_out)], stdout=PIPE, stdin=PIPE, stderr=PIPE)
poll_light = light_process.poll() #light
if poll_light is not None:
light_process = Popen(['python3', 'lightingElement.py', str(targetL), str(LtimeOn), str(LtimeOff), str(lightInterval)], stdout=PIPE, stdin=PIPE, stderr=PIPE)
