def lets_run_a_test(name):
sitl_args = [
"dronekit-sitl",
"copter-3.3-rc5",
"-I0",
"-S",
"--model",
"quad",
"--home=-35.363261,149.165230,584,353",
]
speedup = os.environ.get("TEST_SPEEDUP", "1")
rate = os.environ.get("TEST_RATE", "200")
sitl_args += ["--speedup", str(speedup), "-r", str(rate)]
# Change CPU core affinity.
# TODO change affinity on osx/linux
if sys.platform == "win32":
# 0x14 = 0b1110 = all cores except cpu 1
sitl = Popen(
["start", "/affinity", "14", "/realtime", "/b", "/wait"] + sitl_args, shell=True, stdout=PIPE, stderr=PIPE
)
else:
sitl = Popen(sitl_args, stdout=PIPE, stderr=PIPE)
bg.append(sitl)
while sitl.poll() == None:
line = sitl.stdout.readline()
if "Waiting for connection" in line:
break
if sitl.poll() != None and sitl.returncode != 0:
print("[runner] ...aborting with SITL error code " + str(sitl.returncode))
sys.stdout.flush()
sys.stderr.flush()
sys.exit(sitl.returncode)
newenv = os.environ.copy()
newenv["PYTHONUNBUFFERED"] = "1"
if sys.platform == "win32":
out_fd = 1
err_fd = 2
else:
out_fd = os.dup(1)
err_fd = os.dup(2)
(out_fd, out_h) = wrap_fd(out_fd)
(err_fd, err_h) = wrap_fd(err_fd)
newenv["TEST_WRITE_OUT"] = out_fd
newenv["TEST_WRITE_ERR"] = err_fd
newenv["TEST_NAME"] = name
print("[runner] " + name, file=sys.stderr)
sys.stdout.flush()
sys.stderr.flush()
mavproxy_verbose = os.environ.get("TEST_VERBOSE", "0") != "0"
timeout = 5 * 60
try:
if sys.platform == "win32":
try:
# Try to find installed Python MAVProxy module.
import imp
imp.find_module("MAVProxy")
mavprog = [sys.executable, "-m", "MAVProxy.mavproxy"]
except:
# Uses mavproxy.exe instead.
mavprog = ["mavproxy"]
else:
mavprog = [sys.executable, "-m", "MAVProxy.mavproxy"]
p = Popen(
mavprog + ["--logfile=" + tempfile.mkstemp()[1], "--master=tcp:127.0.0.1:5760"],
cwd=testpath,
env=newenv,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE if not mavproxy_verbose else None,
)
bg.append(p)
# TODO this sleep is only for us to waiting until
# all parameters to be received; would prefer to
# move this to testlib.py and happen asap
while p.poll() == None:
line = p.stdout.readline()
if mavproxy_verbose:
sys.stdout.write(line)
sys.stdout.flush()
if "parameters" in line:
break
time.sleep(3)
# NOTE these are *very inappropriate settings*
# to make on a real vehicle. They are leveraged
# exclusively for simulation. Take heed!!!
p.stdin.write("param set ARMING_CHECK 0\n")
p.stdin.write("param set FS_THR_ENABLE 0\n")
p.stdin.write("param set FS_GCS_ENABLE 0\n")
p.stdin.write("param set EKF_CHECK_THRESH 0\n")
p.stdin.write("module load droneapi.module.api\n")
p.stdin.write("api start testlib.py\n")
p.stdin.flush()
wait_timeout(p, timeout, mavproxy_verbose)
except RuntimeError:
kill(p.pid)
p.returncode = 143
print("Error: Timeout after " + str(timeout) + " seconds.")
bg.remove(p)
if sys.platform == "win32":
out_h.Close()
err_h.Close()
kill(sitl.pid)
bg.remove(sitl)
if p.returncode != 0:
print("[runner] ...failed with dronekit error code " + str(p.returncode))
else:
print("[runner] ...success.")
sys.stdout.flush()
sys.stderr.flush()
time.sleep(5)
return p.returncode
def _execute_anybodycon(macro,
logfile,
anybodycon_path=None,
timeout=3600,
keep_macrofile=False,
env=None):
""" Launches the AnyBodyConsole applicaiton with the specified macro
saving the result to logfile """
if anybodycon_path is None:
anybodycon_path = get_anybodycon_path()
if not os.path.isfile(anybodycon_path):
raise IOError("Can not find anybodycon.exe: " + anybodycon_path)
macro_filename = os.path.splitext(logfile.name)[0] + '.anymcr'
with open(macro_filename, 'w+b') as macro_file:
macro_file.write('\n'.join(macro).encode('UTF-8'))
macro_file.flush()
anybodycmd = [os.path.realpath(anybodycon_path),
'--macro=', macro_file.name, '/ni']
if sys.platform.startswith("win"):
# Don't display the Windows GPF dialog if the invoked program dies.
# See comp.os.ms-windows.programmer.win32
# How to suppress crash notification dialog?, Jan 14,2004 -
# Raymond Chen's response [1]
SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN
ctypes.windll.kernel32.SetErrorMode(SEM_NOGPFAULTERRORBOX)
subprocess_flags = 0x8000000 # win32con.CREATE_NO_WINDOW?
else:
subprocess_flags = 0
try:
# Check global module flag to avoid starting processes after
# the user cancelled the processes
timeout_time = time.clock() + timeout
proc = Popen(anybodycmd,
stdout=logfile,
stderr=logfile,
creationflags=subprocess_flags,
env=env)
_subprocess_container.add(proc.pid)
while proc.poll() is None:
if time.clock() > timeout_time:
proc.terminate()
proc.communicate()
logfile.seek(0, 2)
logfile.write('\nERROR: AnyPyTools : Timeout after {:d} sec.'.format(int(timeout)))
proc.returncode = 0
break
time.sleep(0.05)
_subprocess_container.remove(proc.pid)
retcode = ctypes.c_int32(proc.returncode).value
if retcode == _KILLED_BY_ANYPYTOOLS:
logfile.write('\nAnybodycon.exe was interrupted by AnyPyTools')
elif retcode:
logfile.write('\nERROR: AnyPyTools : anybodycon.exe exited unexpectedly.'
' Return code: ' + str(proc.returncode))
if not keep_macrofile:
silentremove(macro_file.name)
finally:
logfile.seek(0)
return retcode
def calculateKaKs(indir, outdir, method, axt):
'''Calculates substition rates.'''
with open(axt, "r") as infile:
filename = axt.split("/")[-1]
# Create output file
outfile = (outdir + filename.split(".")[0] + ".kaks")
cmd = "bin/KaKs_Calculator -i " + axt + " -o " + outfile + " -m " + method
ck = Popen(split(cmd), stdout=DEVNULL)
ck.wait()
if ck.returncode() == 0:
return True
def calculateKaKs(indir, outdir, method, axt):
'''Calculates substition rates.'''
with open(axt, "r") as infile:
filename = axt.split("/")[-1]
# Create output file
outfile = (outdir + filename.split(".")[0] + ".kaks")
cmd = "bin/KaKs_Calculator -i "+axt+" -o "+outfile+" -m "+method
ck = Popen(split(cmd), stdout = DEVNULL)
ck.wait()
if ck.returncode() == 0:
return True
def run_cmd(cmd, my_cwd=None):
'''
Description:
Run a command given by a dictionary, check for stderr output,
return code.
To check the return code catch SystemExit then examine:
ret['subproc'].returncode.
Input:
cmd - a list containing the command to execute.
e.g.: cmd = ['ls', '/tmp']
Returns:
ret - a dictionary upon return contains the keys:
'subproc', 'err', 'out'
ret['subproc'].returncode = subprocess return code
ret['err'] = command errors string.
ret['out'] = command out list.
Example:
cmd = ['ls', '/tmp']
ret = run_cmd(cmd)
ret.keys()
['subproc', 'err', 'out']
ret['subproc'].returncode
0
ret['err']
''
ret['out']
'''
pfail = SubprocReturn()
# Return dictionary to contain keys: 'cmd', 'subproc', 'err', 'out'
ret = {'subproc': None, 'err': '', 'out': ''}
try:
ret['subproc'] = Popen(cmd, cwd=my_cwd, stdout=PIPE, stderr=PIPE)
# unable to find command will result in an OSError
except OSError, err:
if not ret['subproc']:
ret['subproc'] = pfail
ret['subproc'].returncode = 127 # command not found
ret['err'] = str(err)
return ret
def __close(process: subprocess.Popen,
stderr_capture_thread: StreamCaptureThread[bytes],
stderr_encoding: str, tmp_input: FilesystemIPC) -> None:
'''Shut down the process if it is still running. Raise a SolverSubprocessError if the process exited with an error.'''
if process.poll() is None:
# Still running? Kill the subprocess
process.terminate()
try:
process.wait(
timeout=0.001
) # type: ignore (mypy does not know about `timeout`)
except subprocess.TimeoutExpired: # type: ignore (mypy does not know about `TimeoutExpired`)
# Kill unconditionally after a short timeout.
# A potential problem with SIGKILL: we might not get all the error messages on stderr (if the child process is killed before it has a chance to write an error message)
process.kill()
process.wait()
assert process.poll() is not None # subprocess has stopped
# Various outcomes were observed that do not signify a real error condition as far as py-dlvhex is concerned:
# * dlvhex2 simply terminates by itself between the is_running() check and the terminate() call
# * dlvhex2 executes its SIGTERM handler and exits with code 2, and a message on stderr ("dlvhex2 [...] got termination signal 15")
# * dlvhex2 doesn't have its SIGTERM handler active and the default handler exits with code -15
# * dlvhex2 hangs after receiving SIGTERM (maybe when it's blocking on stdout? we're not reading from it anymore at this point), so we send SIGKILL after the timeout and it exits with -9
# * dlvhex2 receives SIGTERM and crashes with "Assertion failed: (!res), function ~mutex, file /usr/local/include/boost/thread/pthread/mutex.hpp, line 111", and exit code -6 (SIGABRT)
if process.returncode in (2, -signal.SIGTERM, -signal.SIGKILL,
-signal.SIGABRT):
# In the cases mentioned above (and only if we are responsible for termination, i.e. after calling terminate() from this function), don't throw an exception
process.returncode = 0
# Note: only close stdin after the process has been terminated, otherwise dlvhex will start outputting everything at once
process.stdin.close()
process.stdout.close()
stderr_capture_thread.join() # ensure cleanup of stderr
# Remove the temporary input pipe/file
# Note: To clean up tmp_input in a reliable way, we must be sure dlvhex2 has already read all the data (or, has at the very least opened the file).
# Because of this, the earliest point where we are able to clean up tmp_input would be when dlvhex2 starts outputting the first answer set.
tmp_input.cleanup()
if process.returncode != 0:
err = SolverSubprocessError(
process.returncode,
str(stderr_capture_thread.data, encoding=stderr_encoding))
process.returncode = 0 # make sure we only raise an error once
raise err
def exec_shell_command(arguments, env=None):
if env is None:
env = {}
merge_env = environ.copy()
merge_env.update(env)
p = Popen(arguments, env=merge_env)
try:
p.wait()
except KeyboardInterrupt:
p.kill()
p.wait()
p.returncode = 1
return p.returncode
def exec_shell_command(arguments, env=None):
if env is None:
env = {}
merge_env = environ.copy()
merge_env.update(env)
p = Popen(arguments, env=merge_env)
try:
p.wait()
except KeyboardInterrupt:
p.kill()
p.wait()
p.returncode = 1
return p.returncode
def remote_sync_cmd(host, cmd, log):
if host and cmd:
proc = Popen(["ssh", "root@" + host, "bash -s --"],
stdin=PIPE,
stdout=log,
stderr=STDOUT,
universal_newlines=True)
proc.stdin.write(cmd)
proc.stdin.close()
sleep(1)
proc.poll()
else:
proc = Dummy()
proc.returncode = 0
return proc
def run(self, input_filepath, student_id="local", with_timestamp=True, cwd="./"):
"""Run the executable with input.
The output will be temporarily placed in specific location,
and the path will be returned for the validation.
"""
if not os.path.isfile(cwd + self.executable):
return 1, "no_executable_to_run"
output_filepath = os.path.join(
self.temp_output_dir, get_filename(input_filepath)
)
if with_timestamp:
output_filepath += student_id + "_" + str(int(time.time()))
output_filepath += self._ans_ext
cmd = self.run_command
cmd = re.sub(r"{input}", input_filepath, cmd)
cmd = re.sub(r"{output}", output_filepath, cmd)
process = Popen(
cmd,
stdout=PIPE,
stderr=PIPE,
shell=True,
executable="bash",
cwd=cwd,
start_new_session=True,
)
try:
_, err = process.communicate(timeout=float(self.timeout))
except TimeoutExpired:
os.killpg(os.getpgid(process.pid), signal.SIGTERM)
# Ref: https://stackoverflow.com/a/44705997
self.error_handler.handle(
f"TLE at {get_filename(input_filepath)}; kill `{cmd}`",
student_id=student_id,
)
process.returncode = 124
return process.returncode, output_filepath
except KeyboardInterrupt:
os.killpg(os.getpgid(process.pid), signal.SIGTERM)
raise KeyboardInterrupt from None
if process.returncode != 0:
self.error_handler.handle(
"Failed in run stage. "
+ str(err, encoding="utf8")
+ "Please check `your program` first.",
student_id=student_id,
)
return process.returncode, output_filepath
def run_python_function(w, timeout=600):
try:
url, func = w['base']['location'].split("::")
except Exception as e:
raise Exception("can not split", w['base']['location'], e)
pars = ",".join(["%s=\"%s\""%(k,v) for k,v in w['inputs'].items()])
time_constrain = None
for k,v in w['inputs'].items():
if 'timestamp' in k:
time_constrain = float(v)
if time_constrain is not None:
if time_constrain < time.time() - 3600*3:
print(f"time-constrained goal obsolete: {time_constrain - time.time()} < {- 3600*3}!")
result = dict(stdout='')
status = 'obsolete'
return dict(result=result, status=status)
if re.match("https://raw.githubusercontent.com/volodymyrss/oda_test_kit/+[0-9a-z]+?/test_[a-z0-9]+?.py", url):
print("found valid url", url)
else:
raise UnsupportedCallType("invalid url: %s!"%url)
if re.match("test_[a-z0-9]+?", func):
print("found valid func:", func)
else:
raise Exception("invalid func: %s!"%func)
urls = [
url.replace("https://raw.githubusercontent.com/volodymyrss/oda_test_kit/",
"https://gitlab.astro.unige.ch/savchenk/osa_test_kit/raw/"
),
url,
]
r = None
for url in urls:
print("fetching url option %s ..."%url)
r = requests.get(url)
if r.status_code == 200:
break
else:
print("unable to reach url %s: %s"%(url, r))
if r is None:
raise Exception("all urls failed!")
c = r.text
c += "\n\n%s(%s)"%(func, pars)
print("calling python with:\n", "\n>".join(c.split("\n")))
stdout = ""
p = Popen(["python"],
stdin=PIPE,
stdout=PIPE,
stderr=STDOUT,
env={**os.environ, "PYTHONUNBUFFERED":"1"},
bufsize=0)
p.stdin.write(c.encode())
p.stdin.close()
def enqueue_output(out, queue):
for line in iter(out.readline, b''):
queue.put(line)
out.close()
q = Queue()
t = Thread(target=enqueue_output, args=(p.stdout, q))
t.daemon = True
t.start()
time_spent = 0
while True:
try:
l = q.get_nowait() # or q.get(timeout=.1)
except Empty:
print('no output', time_spent, 's since start')
if p.poll() is not None:
print('\033[32mterminated!\033[0m')
break
time.sleep(1)
time_spent += 1
if time_spent > timeout:
print('\033[31mtimeout exceeded, killing\033[0m')
p.kill()
p.returncode = -1
break
else: # got line
print("> ", l.decode().strip())
stdout += l.decode()
time.sleep(0.01)
print("exited as ", p.returncode)
if p.returncode == 0:
result = dict(stdout=stdout)
status = 'success'
print("\033[32mSUCCESS!\033[0m")
else:
result = dict(stdout=stdout, exception=p.returncode)
status = 'failed'
print("\033[31mFAILED!\033[0m")
return dict(result=result, status=status)
def _run_latexmk(self):
# Do not call latexmk if this is a pass-through build
if self.latexmk_passthrough:
# Determine if interim build has failed from --latexmk-passthrough-fail
if self.args.latexmk_passthrough_fail:
self.latexmk_returncode = 1
return
# Make sure that the latexmkrc file exists in the expected location
self.latexmkrc_abs_path = os.path.join(self.buildPDF_dir_path,'support','latexmk','latexmkrc')
if not os.path.isfile(self.latexmkrc_abs_path):
matches = []
for root, dirnames, filenames in os.walk(self.buildPDF_dir_path):
for filename in fnmatch.filter(filenames, 'latexmkrc'):
matches.append(os.path.join(root, filename))
if len(matches) > 1:
print_error('Multiple latexmkrc file matches found on path!\n {0}'.format(matches))
elif len(matches) < 1:
print_error('latexmkrc file not found anywhere on search path!\nExpected NASA-LaTeX-Docs location: nasa-latex-docs/support/latexmk/latexmkrc')
else:
self.latexmkrc_abs_path = matches[0]
print_status("Building from TeX Root ",self.input_abs_path,quiet=self._quiet)
# Go to directory where input TeX file is located
os.chdir(self.input_dir_path)
# Create the tmp/ dir if it does not exist
if not os.path.exists(self.ENV['TMPDIR']):
os.makedirs(self.ENV['TMPDIR'])
# Find all folders in current input_dir_path that contain a .tex file (required for \include{} command)
dirs_with_tex = set(folder for folder, subfolders, files in os.walk(self.input_dir_path) for file_ in files if os.path.splitext(file_)[1] == '.tex')
# Create these directories in tmp/ so that correct .aux files can be written there
for dir_with_tex in dirs_with_tex:
dir_with_tex = dir_with_tex.replace(self.input_dir_path+os.sep,'').strip()
if dir_with_tex and not os.path.exists(os.path.join(self.ENV['TMPDIR'],dir_with_tex)):
if self.input_bare+'_standalone' not in dir_with_tex:
os.makedirs(os.path.join(self.ENV['TMPDIR'],dir_with_tex))
# Add blank .bbl file
bbl_file = os.path.join(self.ENV['TMPDIR'],self.input_bare+'.bbl')
if not os.path.isfile(bbl_file):
open(bbl_file, 'a').close()
try:
if self.args.force:
latexmk = Popen(['latexmk',self.input_bare,'-g','-r',self.latexmkrc_abs_path], env=self.ENV,stdout=self.stdout,stderr=self.stderr)
else:
latexmk = Popen(['latexmk',self.input_bare,'-r',self.latexmkrc_abs_path], env=self.ENV,stdout=self.stdout,stderr=self.stderr)
latexmk.wait()
except KeyboardInterrupt:
latexmk.kill()
print_warn("Keyboard Interrupt: Exiting continuous preview watch mode",pre='\n')
# Delete the tmp/ directory if user specifies
if self.args.clean and latexmk.returncode == 0:
shutil.rmtree(self.ENV['TMPDIR'])
sys.exit(latexmk.returncode)
try:
if latexmk.returncode == 0:
with open(os.path.join(self.ENV['TMPDIR'],'build.out'), 'r') as f:
latexmk.returncode = int(f.readline().strip())
except:
pass
self.latexmk_returncode = latexmk.returncode
def execute_anybodycon(macro,
logfile=None,
anybodycon_path=None,
timeout=3600,
keep_macrofile=False,
env=None):
"""Launch a single AnyBodyConsole applicaiton.
This is a low level function to start a AnyBody Console process
with a given list macros.
Parameters
----------
macro : list of str
List of macros strings to pass to the AnyBody Console Application
logfile : file like object, optional
An open file like object to write to pipe the output of AnyBody
into. (Defaults to None, in which case it will use sys.stdout)
anybodycon_path : str, optional
Path to the AnyBodyConsole application. Default to None, in which
case the default installed AnyBody installation will be looked up
in the Windows registry.
timeout : int, optional
Timeout before the process is killed autmotically. Defaults to
3600 seconds (1 hour).
keep_macrofile : bool, optional
Set to True to prevent the temporary macro file from beeing deleted.
(Defaults to False)
env: dict
Environment varaibles which are passed to the started AnyBody console
application.
Returns
-------
int
The return code from the AnyBody Console application.
"""
if logfile is None:
logfile = sys.stdout
try:
macro_filename = os.path.splitext(logfile.name)[0] + '.anymcr'
except AttributeError:
macro_filename = 'macrofile.anymcr'
if anybodycon_path is None:
anybodycon_path = get_anybodycon_path()
if macro and macro[-1] != 'exit':
macro.append('exit')
if not os.path.isfile(anybodycon_path):
raise IOError("Can not find anybodycon.exe: " + anybodycon_path)
with open(macro_filename, 'w+b') as macro_file:
macro_file.write('\n'.join(macro).encode('UTF-8'))
macro_file.flush()
anybodycmd = [
os.path.realpath(anybodycon_path), '--macro=', macro_file.name, '/ni'
]
if sys.platform.startswith("win"):
# Don't display the Windows GPF dialog if the invoked program dies.
# See comp.os.ms-windows.programmer.win32
# How to suppress crash notification dialog?, Jan 14,2004 -
# Raymond Chen's response [1]
SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN
ctypes.windll.kernel32.SetErrorMode(SEM_NOGPFAULTERRORBOX)
subprocess_flags = 0x8000000 # win32con.CREATE_NO_WINDOW?
else:
subprocess_flags = 0
# Check global module flag to avoid starting processes after
# the user cancelled the processes
timeout_time = time.clock() + timeout
proc = Popen(anybodycmd,
stdout=logfile,
stderr=logfile,
creationflags=subprocess_flags,
env=env)
_subprocess_container.add(proc.pid)
while proc.poll() is None:
if time.clock() > timeout_time:
proc.terminate()
proc.communicate()
try:
logfile.seek(0, os.SEEK_END)
except io.UnsupportedOperation:
pass
logfile.write(
'\nERROR: AnyPyTools : Timeout after {:d} sec.'.format(
int(timeout)))
proc.returncode = 0
break
time.sleep(0.05)
_subprocess_container.remove(proc.pid)
retcode = ctypes.c_int32(proc.returncode).value
if retcode == _KILLED_BY_ANYPYTOOLS:
logfile.write('\nAnybodycon.exe was interrupted by AnyPyTools')
elif retcode == _NO_LICENSES_AVAILABLE:
logfile.write('\nERROR: anybodycon.exe existed unexpectedly. '
'Return code: ' + str(_NO_LICENSES_AVAILABLE) +
' : No license available.')
elif retcode:
logfile.write(
'\nERROR: AnyPyTools : anybodycon.exe exited unexpectedly.'
' Return code: ' + str(retcode))
if not keep_macrofile:
silentremove(macro_file.name)
return retcode
def lets_run_a_test(name):
sitl_args = ['python', '-m', 'dronekit.sitl', 'copter-3.3-rc5', '-I0', '-S', '--model', 'quad', '--home=-35.363261,149.165230,584,353']
speedup = os.environ.get('TEST_SPEEDUP', '1')
rate = os.environ.get('TEST_RATE', '200')
sitl_args += ['--speedup', str(speedup), '-r', str(rate)]
newenv = os.environ.copy()
newenv['PYTHONUNBUFFERED'] = '1'
# Change CPU core affinity.
# TODO change affinity on osx/linux
if sys.platform == 'win32':
# 0x14 = 0b1110 = all cores except cpu 1
sitl = Popen(['start', '/affinity', '14', '/realtime', '/b', '/wait'] + sitl_args, shell=True, stdout=PIPE, stderr=PIPE, env=newenv)
else:
sitl = Popen(sitl_args, stdout=PIPE, stderr=PIPE, env=newenv)
bg.append(sitl)
while sitl.poll() == None:
line = sitl.stdout.readline()
if 'Waiting for connection' in line:
break
if sitl.poll() != None and sitl.returncode != 0:
print('[runner] ...aborting with SITL error code ' + str(sitl.returncode))
sys.stdout.flush()
sys.stderr.flush()
sys.exit(sitl.returncode)
if sys.platform == 'win32':
out_fd = 1
err_fd = 2
else:
out_fd = os.dup(1)
err_fd = os.dup(2)
(out_fd, out_h) = wrap_fd(out_fd)
(err_fd, err_h) = wrap_fd(err_fd)
newenv['TEST_WRITE_OUT'] = out_fd
newenv['TEST_WRITE_ERR'] = err_fd
newenv['TEST_NAME'] = name
print('[runner] ' + name, file=sys.stderr)
sys.stdout.flush()
sys.stderr.flush()
mavproxy_verbose = os.environ.get('TEST_VERBOSE', '0') != '0'
timeout = 5*60
try:
if sys.platform == 'win32':
try:
# Try to find installed Python MAVProxy module.
import imp
imp.find_module('MAVProxy')
mavprog = [sys.executable, '-m', 'MAVProxy.mavproxy']
except:
# Uses mavproxy.exe instead.
mavprog = ['mavproxy']
else:
mavprog = [sys.executable, '-m', 'MAVProxy.mavproxy']
p = Popen(mavprog + ['--logfile=' + tempfile.mkstemp()[1], '--master=tcp:127.0.0.1:5760'],
cwd=testpath, env=newenv, stdin=PIPE, stdout=PIPE,
stderr=PIPE if not mavproxy_verbose else None)
bg.append(p)
# TODO this sleep is only for us to waiting until
# all parameters to be received; would prefer to
# move this to testlib.py and happen asap
while p.poll() == None:
line = p.stdout.readline()
if mavproxy_verbose:
sys.stdout.write(line)
sys.stdout.flush()
if 'parameters' in line:
break
time.sleep(3)
# NOTE these are *very inappropriate settings*
# to make on a real vehicle. They are leveraged
# exclusively for simulation. Take heed!!!
p.stdin.write('set moddebug 3\n')
p.stdin.write('param set ARMING_CHECK 0\n')
p.stdin.write('param set FS_THR_ENABLE 0\n')
p.stdin.write('param set FS_GCS_ENABLE 0\n')
p.stdin.write('param set EKF_CHECK_THRESH 0\n')
p.stdin.write('module load droneapi.module.api\n')
p.stdin.write('api start testlib.py\n')
p.stdin.flush()
wait_timeout(p, timeout, mavproxy_verbose)
except RuntimeError:
kill(p.pid)
p.returncode = 143
print('Error: Timeout after ' + str(timeout) + ' seconds.')
bg.remove(p)
if sys.platform == 'win32':
out_h.Close()
err_h.Close()
kill(sitl.pid)
bg.remove(sitl)
if p.returncode != 0:
print('[runner] ...failed with dronekit error code ' + str(p.returncode))
else:
print('[runner] ...success.')
sys.stdout.flush()
sys.stderr.flush()
time.sleep(5)
return p.returncode
class DockerOnion(object):
"""
The DockerOnion object is where GhostWriter runs docker images
"""
def __init__(self, base, project, containers_path, is_cli, log_handler):
self.base = base
self.containers_path = containers_path
self.project = project
self.log_handler = log_handler
self.base.log("[GhostWriter][Docker]", "__init__",
"is_cli={}".format(is_cli))
def build(self):
self.build_docker = Popen(["docker", "build", ".", "-t", "website"],
cwd=self.containers_path,
start_new_session=True)
self.base.log("[GhostWriter][Docker]", "Docker Build",
"pid={}".format(self.build_docker.pid))
self.build_docker.wait()
def start(self):
# Here we should check that Popen has finished before calling all the other run functions
self.build()
self.docker = Popen([
"docker", "run", "-v", "{}/public:/var/www/html".format(
self.project.folder), "--name", "website", "-t", "-d",
"website"
],
cwd=self.containers_path,
start_new_session=True)
self.base.log("[GhostWriter][Docker]", "Docker Start",
"pid={}".format(self.docker.pid))
self.docker.wait()
self.run_tor()
self.run_web_server()
time.sleep(.5000)
self.get_onionservice_address()
def run_tor(self):
self.tor = Popen(
["docker", "exec", "-t", "-d", "--user=root", "website", "tor"],
cwd=self.containers_path,
start_new_session=True)
self.base.log("[GhostWriter][Docker]", "Docker run Tor",
"pid={}".format(self.tor.pid))
self.tor.wait()
def run_web_server(self):
self.nginx = Popen(
["docker", "exec", "-t", "-d", "--user=root", "website", "nginx"],
cwd=self.containers_path,
start_new_session=True)
self.base.log("[GhostWriter][Docker]", "Docker run Nginx",
"pid={}".format(self.nginx.pid))
self.nginx.wait()
def get_onionservice_address(self):
self.onion = Popen([
"docker", "exec", "-t", "--user=root", "website",
"sh -c \'cat onion_web_service/hostname\'"
],
cwd=self.containers_path,
start_new_session=True,
stdout=self.log_handler,
stderr=self.log_handler)
self.base.log("[GhostWriter][Docker]",
"Docker get onion service address",
"pid={}".format(self.onion.pid))
self.onion.wait()
def status(self):
outs = None
errs = None
try:
outs, errs = self.docker.communicate(timeout=3)
except Exception as e:
self.docker.kill()
outs, errs = self.docker.communicate()
return outs, errs
def check_output(self):
return self.docker.stdout.readline()
def stop(self):
self.docker.terminate()
self.base.log("[GhostWriter][Web]", "Lektor Stop",
"return_code={}".format(self.docker.returncode()))
def execute_anybodycon(
macro,
logfile=None,
anybodycon_path=None,
timeout=3600,
keep_macrofile=False,
env=None,
priority=BELOW_NORMAL_PRIORITY_CLASS,
debug_mode=0,
):
"""Launch a single AnyBodyConsole applicaiton.
This is a low level function to start a AnyBody Console process
with a given list macros.
Parameters
----------
macro : list of str
List of macros strings to pass to the AnyBody Console Application
logfile : file like object, optional
An open file like object to write to pipe the output of AnyBody
into. (Defaults to None, in which case it will use sys.stdout)
anybodycon_path : str, optional
Path to the AnyBodyConsole application. Default to None, in which
case the default installed AnyBody installation will be looked up
in the Windows registry.
timeout : int, optional
Timeout before the process is killed autmotically. Defaults to
3600 seconds (1 hour).
keep_macrofile : bool, optional
Set to True to prevent the temporary macro file from beeing deleted.
(Defaults to False)
env: dict
Environment varaibles which are passed to the started AnyBody console
application.
priority : int, optional
The priority of the subprocesses. This can be on of the following:
``anypytools.IDLE_PRIORITY_CLASS``, ``anypytools.BELOW_NORMAL_PRIORITY_CLASS``,
``anypytools.NORMAL_PRIORITY_CLASS``, ``anypytools.HIGH_PRIORITY_CLASS``
Default is BELOW_NORMAL_PRIORITY_CLASS.
debug_mode : int
The AMS debug mode to use. Defaults to 0 which is disabled. 1 correspond to
crashdump enabled
Returns
-------
int
The return code from the AnyBody Console application.
"""
if logfile is None:
logfile = sys.stdout
try:
macro_filename = os.path.splitext(logfile.name)[0] + ".anymcr"
except AttributeError:
macro_filename = "macrofile.anymcr"
if anybodycon_path is None:
anybodycon_path = get_anybodycon_path()
if macro and macro[-1] != "exit":
macro.append("exit")
if not os.path.isfile(anybodycon_path):
raise IOError("Can not find anybodycon.exe: " + anybodycon_path)
with open(macro_filename, "w+b") as macro_file:
macro_file.write("\n".join(macro).encode("UTF-8"))
macro_file.flush()
anybodycmd = [
os.path.realpath(anybodycon_path),
"--macro=",
macro_file.name,
"/ni",
"/deb",
str(debug_mode),
]
if sys.platform.startswith("win"):
# Don't display the Windows GPF dialog if the invoked program dies.
# See comp.os.ms-windows.programmer.win32
# How to suppress crash notification dialog?, Jan 14,2004 -
# Raymond Chen's response [1]
SEM_NOGPFAULTERRORBOX = 0x0002 # From MSDN
ctypes.windll.kernel32.SetErrorMode(SEM_NOGPFAULTERRORBOX)
subprocess_flags = 0x8000000 # win32con.CREATE_NO_WINDOW?
else:
subprocess_flags = 0
subprocess_flags |= priority
# Check global module flag to avoid starting processes after
# the user cancelled the processes
timeout_time = time.process_time() + timeout
proc = Popen(
anybodycmd,
stdout=logfile,
stderr=logfile,
creationflags=subprocess_flags,
env=env,
)
_subprocess_container.add(proc.pid)
while proc.poll() is None:
if time.process_time() > timeout_time:
proc.terminate()
proc.communicate()
try:
logfile.seek(0, os.SEEK_END)
except io.UnsupportedOperation:
pass
logfile.write(
"\nERROR: AnyPyTools : Timeout after {:d} sec.".format(int(timeout))
)
proc.returncode = 0
break
time.sleep(0.05)
_subprocess_container.remove(proc.pid)
retcode = ctypes.c_int32(proc.returncode).value
if retcode == _KILLED_BY_ANYPYTOOLS:
logfile.write("\nAnybodycon.exe was interrupted by AnyPyTools")
elif retcode == _NO_LICENSES_AVAILABLE:
logfile.write(
"\nERROR: anybodycon.exe existed unexpectedly. "
"Return code: " + str(_NO_LICENSES_AVAILABLE) + " : No license available."
)
elif retcode:
logfile.write(
"\nERROR: AnyPyTools : anybodycon.exe exited unexpectedly."
" Return code: " + str(retcode)
)
if not keep_macrofile:
silentremove(macro_file.name)
return retcode
def main(): #{
# RE-INSTANTIATE GLOBALS
global cmd_list
# TRY THE FOLLOWING:
try: #{
#######################
# BEGIN INFINITE LOOP #
#######################
while 1 == 1: #{
# GET CURRENT TIME
ts_now = pd.Timestamp.now()
logging.info("CURRENT TIME:\t\t" + str(ts_now))
##############################################
# COMPARE THE TWO
###############################################
# if the input time_stamp is greater than the current one right now..
if (test_ts > ts_now): #{
# that means it hasnt happened yet...
logging.info("\n\n\t" + str(test_ts) +
" HASN'T HAPPENED YET !! \n(it is: " +
str(ts_now) + ")\n")
# WAIT ANOTHER 60 SECONDS to check again...
countdown(60)
#}
###############################################
# << HAS HAPPENED >>
else: #{
# that means it **HAS** happened...
logging.info("\n\n\t" + str(test_ts) +
" **HAS** HAPPENED !! \n(it is: " + str(ts_now) +
")\n")
# CREATER COUNTER
x = 0
for command in cmd_list: #{
logging.info("COMMAND # " + str(x + 1) + "! !")
# RUN PROCESS
proc = Popen(str(command))
logging.info(str(command))
# SLEEP FOR A MINUTE
countdown(60)
proc_status = proc.returncode()
logging.info("RETURNCODE:\t" + str(proc_status))
# INCREMENT COUNTER
x + 1
#}
break
#}
#}
#####################
# END INFINITE LOOP #
#####################
#}
except SystemExit: #{
print("noob")
#}
except: #{
errorMessage = str(sys.exc_info()[0]) + "\n\t\t"
errorMessage = errorMessage + str(sys.exc_info()[1]) + "\n\t\t"
errorMessage = errorMessage + str(sys.exc_info()[2]) + "\n"
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
typeE = str("TYPE : " + str(exc_type))
fileE = str("FILE : " + str(fname))
lineE = str("LINE : " + str(exc_tb.tb_lineno))
messageE = str("MESG : " + "\n" + str(errorMessage) + "\n")
logging.error("\n" + typeE + "\n" + fileE + "\n" + lineE + "\n" +
messageE)
#}
finally: #{
# CALL AGAIN MAKING IT RECURSIVE
main()
