def safe_check_subprocess(code):
"""
<Purpose>
Runs safe_check() in a subprocess. This is done because the AST safe_check()
creates uses a large amount of RAM. By running safe_check() in a subprocess
we can guarantee that the memory will be reclaimed when the process ends.
<Arguments>
code: See safe_check.
<Exceptions>
As with safe_check.
<Return>
See safe_check.
"""
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR, "safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen([sys.executable, path_to_safe_check],stdin=subprocess.PIPE, stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while nonportable.getruntime() - starttime < EVALUTATION_TIMEOUT:
# Did the process finish running?
if proc.poll() != None:
break;
time.sleep(0.02)
else:
# Kill the timed-out process
try:
harshexit.portablekill(proc.pid)
except:
pass
raise Exception, "Evaluation of code safety exceeded timeout threshold \
("+str(nonportable.getruntime() - starttime)+" seconds)"
# Read the output and close the pipe
output = proc.stdout.read()
proc.stdout.close()
# Check the output, None is success, else it is a failure
if output == "None":
return True
# If there is no output, this is a fatal error condition
elif output == "":
raise Exception, "Fatal error while evaluating code safety!"
else:
# Raise the error from the output
raise exception_hierarchy.SafeException, output
def main():
"""
<Purpose>
Kills all the seattle programs that are running.
<Arguments>
None.
<Exceptions>
None.
<Side Effects>
Kills all the seattle programs that are running.
<Returns>
None.
"""
for lockname in locklist:
lockstate = runonce.getprocesslock(lockname)
# If lockstate is a process pid, then we need to terminate it. Otherwise,
# that lock is not being held by a program that needs to be terminated.
if not lockstate == True and not lockstate == False:
# We got the pid, we can stop the process
harshexit.portablekill(lockstate)
# Now acquire the lock for ourselves, looping until we
# actually get it.
retrievedlock = runonce.getprocesslock(lockname)
while retrievedlock != True:
harshexit.portablekill(retrievedlock)
retrievedlock = runonce.getprocesslock(lockname)
def _stopfile_exit(exitcode, pid):
# On Windows, we are in the Repy process, so we can just use harshexit
if harshexit.ostype in ["Windows", "WindowsCE"]:
# Harshexit will store the appriopriate status for us
harshexit.harshexit(exitcode)
else: # On NIX we are on the external process
try:
if exitcode == 44:
# Write out status information, repy was Stopped
statusstorage.write_status("Stopped")
else:
# Status terminated
statusstorage.write_status("Terminated")
except:
pass
# Disable the other status thread, in case the resource thread detects we've killed repy
statusstorage.init(None)
# Kill repy
harshexit.portablekill(pid)
# Fix Derek proposed, this should solve the problem of
# the monitor exiting before the repy process.
time.sleep(1)
# Exit
harshexit.harshexit(78)
def _stopfile_exit(exitcode, pid):
# On Windows, we are in the Repy process, so we can just use harshexit
if harshexit.ostype in ["Windows", "WindowsCE"]:
# Harshexit will store the appriopriate status for us
harshexit.harshexit(exitcode)
else: # On NIX we are on the external process
try:
if exitcode == 44:
# Write out status information, repy was Stopped
statusstorage.write_status("Stopped")
else:
# Status terminated
statusstorage.write_status("Terminated")
except:
pass
# Disable the other status thread, in case the resource thread detects we've killed repy
statusstorage.init(None)
# Kill repy
harshexit.portablekill(pid)
# Fix Derek proposed, this should solve the problem of
# the monitor exiting before the repy process.
time.sleep(1)
# Exit
harshexit.harshexit(78)
def main():
"""
<Purpose>
Kills all the seattle programs that are running.
<Arguments>
None.
<Exceptions>
None.
<Side Effects>
Kills all the seattle programs that are running.
<Returns>
None.
"""
for lockname in locklist:
lockstate = runonce.getprocesslock(lockname)
# If lockstate is a process pid, then we need to terminate it. Otherwise,
# that lock is not being held by a program that needs to be terminated.
if not lockstate == True and not lockstate == False:
# We got the pid, we can stop the process
harshexit.portablekill(lockstate)
# Now acquire the lock for ourselves, looping until we
# actually get it.
retrievedlock = runonce.getprocesslock(lockname)
while retrievedlock != True:
harshexit.portablekill(retrievedlock)
retrievedlock = runonce.getprocesslock(lockname)
def safe_check(code):
"""Check the code to be safe."""
return True
# NOTE: This code will not work in Windows Mobile due to the reliance on subprocess
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR,
"safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen(["python", path_to_safe_check],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
status = None
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while status == None and (nonportable.getruntime() - starttime <
EVALUTATION_TIMEOUT):
status = proc.poll()
time.sleep(0.02)
else:
# Check if the process is still running
if status == None:
# Try to terminate the external process
try:
harshexit.portablekill(proc.pid)
except:
pass
# Raise an exception
raise Exception, "Evaluation of code safety exceeded timeout threshold (" + str(
nonportable.getruntime() - starttime) + " seconds)"
# Read the output and close the pipe
output = proc.stdout.read()
proc.stdout.close()
# Check the output, None is success, else it is a failure
if output == "None":
return True
# If there is no output, this is a fatal error condition
elif output == "":
raise Exception, "Fatal error while evaluating code safety!"
else:
# Raise the error from the output
raise exception_hierarchy.SafeException, output
def stop_nm():
# Stop the NM
gotlock = runonce.getprocesslock("seattlenodemanager")
if gotlock == True:
# No NM running? This is an error
logstream.write("FAILURE: Successfully acquired the NM process lock! The NM should be running!\n")
else:
if gotlock:
# Kill the NM
harshexit.portablekill(gotlock)
# Allow the sockets to cleanup and the locks to be cleaned
time.sleep(3)
def stop_nm():
# Stop the NM
gotlock = runonce.getprocesslock("seattlenodemanager")
if gotlock == True:
# No NM running? This is an error
logstream.write("FAILURE: Successfully acquired the NM process lock! The NM should be running!\n")
else:
if gotlock:
# Kill the NM
harshexit.portablekill(gotlock)
# Allow the sockets to cleanup and the locks to be cleaned
time.sleep(3)
def safe_check(code):
"""Check the code to be safe."""
return True
# NOTE: This code will not work in Windows Mobile due to the reliance on subprocess
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR, "safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen(["python", path_to_safe_check],stdin=subprocess.PIPE, stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
status = None
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while status == None and (nonportable.getruntime() - starttime < EVALUTATION_TIMEOUT):
status = proc.poll()
time.sleep(0.02)
else:
# Check if the process is still running
if status == None:
# Try to terminate the external process
try:
harshexit.portablekill(proc.pid)
except:
pass
# Raise an exception
raise Exception, "Evaluation of code safety exceeded timeout threshold ("+str(nonportable.getruntime() - starttime)+" seconds)"
# Read the output and close the pipe
output = proc.stdout.read()
proc.stdout.close()
# Check the output, None is success, else it is a failure
if output == "None":
return True
# If there is no output, this is a fatal error condition
elif output == "":
raise Exception, "Fatal error while evaluating code safety!"
else:
# Raise the error from the output
raise exception_hierarchy.SafeException, output
def restart_client(filenamelist):
"""
<Purpose>
Restarts the node manager.
<Arguments>
filenamelist - Currently not used, but is included for possible future use.
<Exceptions>
None
<Side Effects>
The current node manager is killed, and a new one is started.
<Returns>
None.
"""
# kill nmmain if it is currently running
retval = runonce.getprocesslock('seattlenodemanager')
if retval == True:
safe_log(
"[restart_client] Obtained the lock 'seattlenodemanager', it wasn't running."
)
# I got the lock, it wasn't running...
# we want to start a new one, so lets release
runonce.releaseprocesslock('seattlenodemanager')
elif retval == False:
# Someone has the lock, but I can't do anything...
safe_log(
"[restart_client] The lock 'seattlenodemanager' is held by an unknown process. Will try to start it anyways."
)
else:
safe_log("[restart_client] Stopping the nodemanager.")
# I know the process ID! Let's stop the process...
harshexit.portablekill(retval)
safe_log("[restart_client] Starting the nodemanager.")
# run the node manager. I rely on it to do the smart thing (handle multiple
# instances, etc.)
nm_restart_command_args_list = ["python", "nmmain.py"]
if run_nodemanager_in_foreground:
nm_restart_command_args_list.append('--foreground')
junkprocessobject = portable_popen.Popen(nm_restart_command_args_list)
def _internal_error(message):
try:
print >> sys.stderr, message
sys.stderr.flush()
except:
pass
# Stop the nmstatusinterface, we don't want any more status updates
nmstatusinterface.stop()
# Kill repy
harshexit.portablekill(childpid)
try:
# Write out status information, repy was Stopped
statusstorage.write_status("Terminated")
except:
pass
def stoptest(socket,pid):
"""
<Purpose>
Checks the socket for any incoming user messages while the test is running.
If the user sends 'cancel' to us, we will kill the running test.
<Arguments>
socket: A TCP socket to read from
pid: The PID of the process to kill.
"""
try:
mesg = socket.recv(8)
print "Incoming message:",mesg
if "cancel" in mesg:
harshexit.portablekill(pid)
except:
pass
def restart_client(filenamelist):
"""
<Purpose>
Restarts the node manager.
<Arguments>
filenamelist - Currently not used, but is included for possible future use.
<Exceptions>
None
<Side Effects>
The current node manager is killed, and a new one is started.
<Returns>
None.
"""
# kill nmmain if it is currently running
retval = runonce.getprocesslock("seattlenodemanager")
if retval == True:
safe_log("[restart_client] Obtained the lock 'seattlenodemanager', it wasn't running.")
# I got the lock, it wasn't running...
# we want to start a new one, so lets release
runonce.releaseprocesslock("seattlenodemanager")
elif retval == False:
# Someone has the lock, but I can't do anything...
safe_log(
"[restart_client] The lock 'seattlenodemanager' is held by an unknown process. Will try to start it anyways."
)
else:
safe_log("[restart_client] Stopping the nodemanager.")
# I know the process ID! Let's stop the process...
harshexit.portablekill(retval)
safe_log("[restart_client] Starting the nodemanager.")
# run the node manager. I rely on it to do the smart thing (handle multiple
# instances, etc.)
nm_restart_command_args_list = ["python", "nmmain.py"]
if run_nodemanager_in_foreground:
nm_restart_command_args_list.append("--foreground")
junkprocessobject = portable_popen.Popen(nm_restart_command_args_list)
def safe_check_subprocess(code):
"""
<Purpose>
Runs safe_check() in a subprocess. This is done because the AST
safe_check() uses a large amount of RAM. By running safe_check() in a
subprocess we can guarantee that the memory will be reclaimed when the
process ends.
<Arguments>
code: See safe_check.
<Exceptions>
As with safe_check.
<Return>
See safe_check.
"""
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR,
"safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen([sys.executable, path_to_safe_check],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while nonportable.getruntime() - starttime < EVALUTATION_TIMEOUT:
# Did the process finish running?
if proc.poll() != None:
break
time.sleep(0.02)
else:
# Kill the timed-out process
try:
harshexit.portablekill(proc.pid)
except:
pass
raise Exception, "Evaluation of code safety exceeded timeout threshold \
(" + str(nonportable.getruntime() -
starttime) + " seconds)"
# Read the output and close the pipe
rawoutput = proc.stdout.read()
proc.stdout.close()
# Interim fix for SeattleTestbed/attic#1080:
# Get rid of stray debugging output on Android of the form
# `dlopen libpython2.6.so` and `dlopen /system/lib/libc.so`,
# yet preserve all of the other output (including empty lines).
if IS_ANDROID:
output = ""
for line in rawoutput.split("\n"):
# Preserve empty lines
if line == "":
output += "\n"
continue
# Suppress debug messages we know can turn up
wordlist = line.split()
if wordlist[0] == "dlopen":
if wordlist[-1] == "/system/lib/libc.so":
continue
if wordlist[-1].startswith("libpython") and \
wordlist[-1].endswith(".so"):
# We expect "libpython" + version number + ".so".
# The version number should be a string convertible to float.
# If it's not, raise an exception.
try:
versionstring = (wordlist[-1].replace("libpython",
"")).replace(
".so", "")
junk = float(versionstring)
except TypeError, ValueError:
raise Exception("Unexpected debug output '" + line +
"' while evaluating code safety!")
else:
output += line + "\n"
# Strip off the last newline character we added
output = output[0:-1]
# without closing stdin, stdout, stderr, nmmain.py won't execute on XP
nmproc.stdin.close()
nmproc.stdout.close()
nmproc.stderr.close()
try:
# We'll want to kill this once we are told to do so... (sys.stdin.close() is
# the signal)
sys.stdin.read()
except KeyboardInterrupt:
# the user interrupted us, let's clean up!
print
print 'Cleaning up the subprocess'
# We need the PID. Since the node manager daemonizes, this isn't the pid of
# nmproc. We can get the PID from the log
nodemanagerlogfo = open('v2' + os.sep + 'nodemanager.old')
firstline = nodemanagerlogfo.readline()
# an entry looks like this:
# 1292537407.82:PID-27493:[INFO]: Running nodemanager in test mode
# let's get the PID out as a number...
pidportion = firstline.split(':')[1]
assert (pidportion.startswith('PID-'))
nmpid = int(pidportion[4:])
# let's terminate the node manager...
harshexit.portablekill(nmpid)
def main():
repy_args = [sys.executable, 'repy.py',
'restrictions.default',
'test_killp_writetodisk.py']
if nonportable.ostype == 'Darwin' or nonportable.ostype == 'Linux':
process = subprocess.Popen(repy_args,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
pid = process.pid
# Give it a few seconds to start.
time.sleep(4)
# Find the orphaned child's PID.
# Run <ps> and print the first field (PID) of the line with second
# field (PPID) equal to the parent pid.
if nonportable.ostype == 'Darwin' or nonportable.osrealtype == 'FreeBSD':
command = "ps -aO ppid | awk '{if ($2==" + str(pid) + ") {print $1}}'"
elif nonportable.ostype == 'Linux':
command = "ps -ef | awk '{if ($3==" + str(pid) + ") {print $2}}'"
(out, error) = subprocess.Popen(command,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
shell = True).communicate()
child_pid = out.strip()
# Windows
elif nonportable.ostype == 'Windows' or nonportable.ostype == 'WindowsCE':
# This is much easier because we don't worry about the path or have
# children to worry about.
process = subprocess.Popen(repy_args,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
pid = process.pid
# Wait while the process starts.
time.sleep(4)
else:
print "Error: Unknown OS type '" + nonportable.ostype + "'!"
sys.exit(1)
# Kill the repy resource monitor.
harshexit.portablekill(pid)
time.sleep(1)
# See ticket #413 and #421
# This is a workaround for the possibility that the repy child was sleeping
if nonportable.ostype == 'Darwin' or nonportable.ostype == 'Linux':
# Send SIGCONT to the child if the process still exists, it should wake up,
# get a read error when checking the pipe, and then exit...
try: os.kill(int(child_pid), signal.SIGCONT)
except OSError: pass # The child has likely exited
# Wait for the signal to take effect
time.sleep(1)
# Make sure the file size is not changing
firstsize = os.path.getsize("junk_test.out")
time.sleep(1)
secondsize = os.path.getsize("junk_test.out")
if firstsize != secondsize: print 'FAIL'
def run_test(arguments, tmpdir, socket):
"""
<Purpose>
Runs the actual test while streaming the results back to the user.
<Arguments>
arguments: The command to executed, will be prepended with 'python '
tmpdir: The temporary directory which we should execute in
socket: A TCP socket to stream the results back on.
<Side Effects>
Launches a new process.
<Returns>
Nothing.
"""
global INITIAL_CONFIG
enable_stderr = INITIAL_CONFIG["stderr"]
path = INITIAL_CONFIG["path"]
use_shell = INITIAL_CONFIG["shell"]
# Inform the user what is happening
if not enable_stderr:
socket.send("NOTE: stderr will be deferred until program exits.\n")
# Go into the temporary directory
originaldir = os.getcwd()
os.chdir(tmpdir)
try:
# Start time
start = time.time()
# Launch the process
proc = subprocess.Popen(path+" "+arguments,shell=use_shell,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
pid = proc.pid
# Setup a thread to check for an interrupt request
settimer(0, stoptest,(socket,pid))
# Read the data from stdout and send it, timeout after 10 minutes
while proc.poll() == None and time.time() - start < 600:
# Enable stderr by default, disable on some systems (Windows)
# On windows select cannot be used, defer stderr
if enable_stderr:
try:
# Check for ready sockets
(rdy,wrt,excep) = select.select([proc.stdout, proc.stderr],[],[],SAMPLE_RATE)
except Exception, e:
rdy = []
# Get data from each ready handle
data = ""
for handle in rdy:
data += handle.read(CHUNK_SIZE)
# If there is any data, write it out
if data != "":
socket.send(data)
else:
# Only check stderr until the then
data = proc.stdout.read(CHUNK_SIZE)
if data != "": socket.send(data)
time.sleep(SAMPLE_RATE)
# Done, flush any remaining data
data = proc.stdout.read()
data += proc.stderr.read()
if data != "":
socket.send(data)
proc.stdout.close()
proc.stderr.close()
# Kill the process if it is still running
if proc.poll() == None:
harshexit.portablekill(pid)
def safe_check_subprocess(code):
"""
<Purpose>
Runs safe_check() in a subprocess. This is done because the AST safe_check()
creates uses a large amount of RAM. By running safe_check() in a subprocess
we can guarantee that the memory will be reclaimed when the process ends.
<Arguments>
code: See safe_check.
<Exceptions>
As with safe_check.
<Return>
See safe_check.
"""
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR,
"safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen([sys.executable, path_to_safe_check],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while nonportable.getruntime() - starttime < EVALUTATION_TIMEOUT:
# Did the process finish running?
if proc.poll() != None:
break
time.sleep(0.02)
else:
# Kill the timed-out process
try:
harshexit.portablekill(proc.pid)
except:
pass
raise Exception, "Evaluation of code safety exceeded timeout threshold \
(" + str(nonportable.getruntime() -
starttime) + " seconds)"
# Read the output and close the pipe
output = proc.stdout.read()
proc.stdout.close()
# Check the output, None is success, else it is a failure
if output == "None":
return True
# If there is no output, this is a fatal error condition
elif output == "":
raise Exception, "Fatal error while evaluating code safety!"
else:
# Raise the error from the output
raise exception_hierarchy.SafeException, output
def safe_check_subprocess(code):
"""
<Purpose>
Runs safe_check() in a subprocess. This is done because the AST safe_check()
creates uses a large amount of RAM. By running safe_check() in a subprocess
we can guarantee that the memory will be reclaimed when the process ends.
<Arguments>
code: See safe_check.
<Exceptions>
As with safe_check.
<Return>
See safe_check.
"""
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR, "safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen([sys.executable, path_to_safe_check],stdin=subprocess.PIPE, stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while nonportable.getruntime() - starttime < EVALUTATION_TIMEOUT:
# Did the process finish running?
if proc.poll() != None:
break;
time.sleep(0.02)
else:
# Kill the timed-out process
try:
harshexit.portablekill(proc.pid)
except:
pass
raise Exception, "Evaluation of code safety exceeded timeout threshold \
("+str(nonportable.getruntime() - starttime)+" seconds)"
# Read the output and close the pipe
rawoutput = proc.stdout.read()
proc.stdout.close()
# Interim fix for #1080: Get rid of stray debugging output on Android
# of the form "dlopen libpython2.6.so" and "dlopen /system/lib/libc.so",
# yet preserve all of the other output (including empty lines).
if IS_ANDROID:
output = ""
for line in rawoutput.split("\n"):
# Preserve empty lines
if line == "":
output += "\n"
continue
# Suppress debug messages we know can turn up
wordlist = line.split()
if wordlist[0]=="dlopen":
if wordlist[-1]=="/system/lib/libc.so":
continue
if wordlist[-1].startswith("libpython") and \
wordlist[-1].endswith(".so"):
# We expect "libpython" + version number + ".so".
# The version number should be a string convertible to float.
# If it's not, raise an exception.
try:
versionstring = (wordlist[-1].replace("libpython",
"")).replace(".so", "")
junk = float(versionstring)
except TypeError, ValueError:
raise Exception("Unexpected debug output '" + line +
"' while evaluating code safety!")
else:
output += line + "\n"
# Strip off the last newline character we added
output = output[0:-1]
def do_oddballtests():
global passcount
global failcount
global endput
# oddball "stop" tests...
logstream.write("Running test %-50s [" % "Stop Test 1")
logstream.flush()
(testout, testerr) = exec_repy_script("stop_testsleep.py", "restrictions.default", {'stop':'nonexist', 'status':'foo'})
if testout == '' and testerr == '':
passcount = passcount + 1
logstream.write(" PASS ]\n")
else:
failcount = failcount + 1
endput = endput+"Stop Test 1\noutput or errput! out:"+testout+"err:"+ testerr+"\n\n"
logstream.write("FAILED]\n")
# oddball "stop" test2...
logstream.write("Running test %-50s [" % "Stop Test 2")
logstream.flush()
(testout, testerr) = exec_repy_script("stop_testsleep.py", "restrictions.default", {'stop':'repy.py', 'status':'foo'})
if (not mobileNoSubprocess) and testout == '' and testerr != '':
passcount = passcount + 1
logstream.write(" PASS ]\n")
elif mobileNoSubprocess and testout.find('Traceback') == -1:
passcount = passcount + 1
logstream.write(" PASS ]\n")
else:
failcount = failcount + 1
logstream.write("FAILED]\n")
endput = endput+"Stop Test 2\noutput or no errput! out:"+testout+"err:"+ testerr+"\n\n"
# oddball "stop" test3...
logstream.write("Running test %-50s [" % "Stop Test 3")
logstream.flush()
# clean up the stop file if it already exists...
if os.path.exists('junk_test.out'):
os.remove('junk_test.out')
(testout, testerr) = exec_repy_script('stop_testsleepwrite.py', "restrictions.default", {'stop':'junk_test.out','status':'foo'})
if testout == '' and testerr == '':
passcount = passcount + 1
logstream.write(" PASS ]\n")
else:
failcount = failcount + 1
logstream.write("FAILED]\n")
endput = endput+"Stop Test 3\noutput or errput! out:"+testout+"err:"+ testerr+"\n\n"
# "stop" test3, however with an absolute path to the stop file
logstream.write("Running test %-50s [" % "Absolute Path Stop Test")
logstream.flush()
# clean up the stop file if it already exists...
if os.path.exists('junk_test.out'):
os.remove('junk_test.out')
currentdirectory = os.getcwd()
stopfilename = os.path.join(currentdirectory,'junk_test.out')
# This is needed when a space is in the directory where the tests are run
stopfilename = '"'+stopfilename+'"'
(testout, testerr) = exec_repy_script('stop_testsleepwrite.py', "restrictions.default", {'stop':stopfilename,'status':'foo'})
if testout == '' and testerr == '':
passcount = passcount + 1
logstream.write(" PASS ]\n")
else:
failcount = failcount + 1
logstream.write("FAILED]\n")
endput = endput+"Absolute Path Stop Test\noutput or errput! out:"+testout+"err:"+ testerr+"\n\n"
# Test running repy from a sub directory
logstream.write("Running test %-50s [" % "Sub-directory test")
logstream.flush()
# Make a temporary directory
if not os.path.exists("subdirtest"):
os.mkdir("subdirtest")
(testout, testerr) = exec_repy_script("../n_testinit.py", "../restrictions.default", {'cwd':'subdirtest'})
if testout != '' and testerr == '':
passcount = passcount + 1
logstream.write(" PASS ]\n")
else:
failcount = failcount + 1
endput = endput+"Sub-directory test\nno output or errput! out:"+testout+"err:"+ testerr+"\n\n"
logstream.write("FAILED]\n")
# oddball killing the parent test...
logstream.write("Running test %-50s [" % "Kill Repy resource monitor.")
logstream.flush()
# Mac or Linux...
if nonportable.ostype == 'Darwin' or nonportable.ostype == 'Linux':
# Get the location of python
locationproc = subprocess.Popen("which python",shell=True,stdout=subprocess.PIPE)
locationproc.wait()
location = locationproc.stdout.read().strip()
locationproc.stdout.close()
# Start the test
p = subprocess.Popen((location+" repy.py restrictions.default killp_writetodisk.py").split(),stdout=subprocess.PIPE, stderr=subprocess.PIPE)
pid = p.pid
# give it a few seconds to start...
time.sleep(4)
# find the orphaned child's PID... Different on Mac / Linux because of ps
# options...
if nonportable.ostype == 'Darwin' or nonportable.osrealtype == 'FreeBSD':
# run ps and print the 1st field (PID) of the line with 2nd field (PPID)
# equal to the parent pid...
childpidprocess = subprocess.Popen("ps -aO ppid | awk '{if ($2=="+str(pid)+") {print $1}}'",stdout=subprocess.PIPE, shell=True)
childpidstring = childpidprocess.stdout.read().strip()
childpidprocess.stdout.close()
elif nonportable.ostype == 'Linux':
# run ps and print the 2nd field (PID) of the line with 3rd field (PPID)
# equal to the parent pid...
childpidprocess = subprocess.Popen("ps -ef | awk '{if ($3=="+str(pid)+") {print $2}}'",stdout=subprocess.PIPE, shell=True)
childpidstring = childpidprocess.stdout.read().strip()
childpidprocess.stdout.close()
else:
print "Internal Error re-examining OS type '"+nonportable.ostype+"'!"
sys.exit(1)
# Windows
elif nonportable.ostype == 'Windows' or nonportable.ostype == 'WindowsCE':
# this is much easier because we don't worry about the path or have
# children to worry about.
p = subprocess.Popen("python repy.py restrictions.default killp_writetodisk.py".split(),stdout=subprocess.PIPE,stderr=subprocess.PIPE)
pid = p.pid
# Wait while the process starts
time.sleep(4)
else:
print "Error: Unknown OS type '"+nonportable.ostype+"'!"
sys.exit(1)
# Kill the repy resource monitor
harshexit.portablekill(pid)
time.sleep(1)
# See ticket #413 and #421
# This is a workaround for the possibility that the repy child was sleeping
if nonportable.ostype == 'Darwin' or nonportable.ostype == 'Linux':
# Send SIGCONT to the child if the process still exists, it should wake up,
# get a read error when checking the pipe, and then exit...
try:
os.kill(int(childpidstring), signal.SIGCONT)
except OSError:
# the child has likely exited
pass
except ValueError:
# This happens when the child PID is ''
pass
# Wait for the signal to take effect
time.sleep(1)
try:
# Make sure the file size is not changing
firstsize = os.path.getsize("junk_test.out")
time.sleep(1)
secondsize = os.path.getsize("junk_test.out")
except (OSError, IOError):
failcount = failcount + 1
logstream.write("FAILED]\n")
endput = endput+"Killing Repy's resource monitor did not stop repy (size error)!\n\n"
else:
if firstsize == secondsize:
passcount = passcount + 1
logstream.write(" PASS ]\n")
else:
failcount = failcount + 1
logstream.write("FAILED]\n")
endput = endput+"Killing Repy's resource monitor did not stop repy!\n\n"
# Close the pipes
p.stdout.close()
p.stderr.close()
# Make sure repy.py works when invoked with options in a various order (to
# test that adding getopt worked.)
logstream.write("Running test %-50s [" % "repy.py takes args any order")
test_process = subprocess.Popen(["python", "repy.py", "--simple", "--logfile", "log.log", "restrictions.loose", "testoptions.py"])
test_process.wait()
# Check that log was created.
if os.path.exists("log.log.old"):
os.remove("log.log.old")
test_process = subprocess.Popen(["python", "repy.py", "--logfile", "log.log", "--simple", "restrictions.loose", "testoptions.py"])
test_process.wait()
# Check that log was created (again).
if os.path.exists("log.log.old"):
passcount += 1
logstream.write(" PASS ]\n")
os.remove("log.log.old")
else:
failcount += 1
logstream.write("FAILED]\n")
endput = endput+"Passing arguments in the opposite order failed!\n\n"
else:
failcount += 1
logstream.write("FAILED]\n")
endput = endput+"Test for passing arguments in the opposite order failed!\n\n"
logstream.flush()
def safe_check(code):
"""Check the code to be safe."""
# NOTE: This code will not work in Windows Mobile due to the reliance on subprocess
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR,
"safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen([sys.executable, path_to_safe_check],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
status = None
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while status == None and (nonportable.getruntime() - starttime <
EVALUTATION_TIMEOUT):
status = proc.poll()
time.sleep(0.02)
else:
# Check if the process is still running
if status == None:
# Try to terminate the external process
try:
harshexit.portablekill(proc.pid)
except:
pass
# Raise an exception
raise Exception, "Evaluation of code safety exceeded timeout threshold (" + str(
nonportable.getruntime() - starttime) + " seconds)"
# Read the output and close the pipe
rawoutput = proc.stdout.read()
proc.stdout.close()
# Interim fix for #1080: Get rid of stray debugging output on Android
# of the form "dlopen libpython2.6.so" and "dlopen /system/lib/libc.so",
# yet preserve all of the other output (including empty lines).
output = ""
for line in rawoutput.split("\n"):
# Preserve empty lines
if line == "":
output += "\n"
continue
# Suppress debug messages we know can turn up
wordlist = line.split()
if wordlist[0] == "dlopen":
if wordlist[-1] == "/system/lib/libc.so":
continue
if wordlist[-1].startswith("libpython") and \
wordlist[-1].endswith(".so"):
# We expect "libpython" + version number + ".so".
# The version number should be a string convertible to float.
# If it's not, raise an exception.
try:
versionstring = (wordlist[-1].replace("libpython",
"")).replace(
".so", "")
junk = float(versionstring)
except TypeError, ValueError:
raise Exception("Unexpected debug output '" + line +
"' while evaluating code safety!")
else:
output += line + "\n"
# Wait for the process to terminate
starttime = nonportable.getruntime()
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while nonportable.getruntime() - starttime < EVALUTATION_TIMEOUT:
# Did the process finish running?
if (IS_ANDROID and os.waitpid(procpid, os.WNOHANG) != (0, 0)) or \
(not IS_ANDROID and proc.poll() != None):
break
time.sleep(0.02)
else:
# Kill the timed-out process
try:
harshexit.portablekill(procpid)
except:
pass
raise Exception, "Evaluation of code safety exceeded timeout threshold \
(" + str(nonportable.getruntime() -
starttime) + " seconds)"
if IS_ANDROID:
# Should return ("safe_check", "None")
msg = nonportable.read_message_from_pipe(readhandle)
if type(msg) == tuple and len(msg) == 2 and msg[0] == "safe_check":
rawoutput = msg[1]
else:
rawoutput = ""
# without closing stdin, stdout, stderr, nmmain.py won't execute on XP
nmproc.stdin.close()
nmproc.stdout.close()
nmproc.stderr.close()
try:
# We'll want to kill this once we are told to do so... (sys.stdin.close() is
# the signal)
sys.stdin.read()
except KeyboardInterrupt:
# the user interrupted us, let's clean up!
print
print 'Cleaning up the subprocess'
# We need the PID. Since the node manager daemonizes, this isn't the pid of
# nmproc. We can get the PID from the log
nodemanagerlogfo = open('v2'+os.sep+'nodemanager.old')
firstline = nodemanagerlogfo.readline()
# an entry looks like this:
# 1292537407.82:PID-27493:[INFO]: Running nodemanager in test mode
# let's get the PID out as a number...
pidportion = firstline.split(':')[1]
assert(pidportion.startswith('PID-'))
nmpid = int(pidportion[4:])
# let's terminate the node manager...
harshexit.portablekill(nmpid)
def do_oddballtests():
global passcount
global failcount
global endput
# oddball "stop" tests...
logstream.write("Running test %-50s [" % "Stop Test 1")
logstream.flush()
(testout, testerr) = exec_repy_script("stop_testsleep.py",
"restrictions.default", {
'stop': 'nonexist',
'status': 'foo'
})
if testout == '' and testerr == '':
passcount = passcount + 1
logstream.write(" PASS ]\n")
else:
failcount = failcount + 1
endput = endput + "Stop Test 1\noutput or errput! out:" + testout + "err:" + testerr + "\n\n"
logstream.write("FAILED]\n")
# oddball "stop" test2...
logstream.write("Running test %-50s [" % "Stop Test 2")
logstream.flush()
(testout, testerr) = exec_repy_script("stop_testsleep.py",
"restrictions.default", {
'stop': 'repy.py',
'status': 'foo'
})
if (not mobileNoSubprocess) and testout == '' and testerr != '':
passcount = passcount + 1
logstream.write(" PASS ]\n")
elif mobileNoSubprocess and testout.find('Traceback') == -1:
passcount = passcount + 1
logstream.write(" PASS ]\n")
else:
failcount = failcount + 1
logstream.write("FAILED]\n")
endput = endput + "Stop Test 2\noutput or no errput! out:" + testout + "err:" + testerr + "\n\n"
# oddball "stop" test3...
logstream.write("Running test %-50s [" % "Stop Test 3")
logstream.flush()
# clean up the stop file if it already exists...
if os.path.exists('junk_test.out'):
os.remove('junk_test.out')
(testout, testerr) = exec_repy_script('stop_testsleepwrite.py',
"restrictions.default", {
'stop': 'junk_test.out',
'status': 'foo'
})
if testout == '' and testerr == '':
passcount = passcount + 1
logstream.write(" PASS ]\n")
else:
failcount = failcount + 1
logstream.write("FAILED]\n")
endput = endput + "Stop Test 3\noutput or errput! out:" + testout + "err:" + testerr + "\n\n"
# "stop" test3, however with an absolute path to the stop file
logstream.write("Running test %-50s [" % "Absolute Path Stop Test")
logstream.flush()
# clean up the stop file if it already exists...
if os.path.exists('junk_test.out'):
os.remove('junk_test.out')
currentdirectory = os.getcwd()
stopfilename = os.path.join(currentdirectory, 'junk_test.out')
# This is needed when a space is in the directory where the tests are run
stopfilename = '"' + stopfilename + '"'
(testout, testerr) = exec_repy_script('stop_testsleepwrite.py',
"restrictions.default", {
'stop': stopfilename,
'status': 'foo'
})
if testout == '' and testerr == '':
passcount = passcount + 1
logstream.write(" PASS ]\n")
else:
failcount = failcount + 1
logstream.write("FAILED]\n")
endput = endput + "Absolute Path Stop Test\noutput or errput! out:" + testout + "err:" + testerr + "\n\n"
# Test running repy from a sub directory
logstream.write("Running test %-50s [" % "Sub-directory test")
logstream.flush()
# Make a temporary directory
if not os.path.exists("subdirtest"):
os.mkdir("subdirtest")
(testout, testerr) = exec_repy_script("../n_testinit.py",
"../restrictions.default",
{'cwd': 'subdirtest'})
if testout != '' and testerr == '':
passcount = passcount + 1
logstream.write(" PASS ]\n")
else:
failcount = failcount + 1
endput = endput + "Sub-directory test\nno output or errput! out:" + testout + "err:" + testerr + "\n\n"
logstream.write("FAILED]\n")
# oddball killing the parent test...
logstream.write("Running test %-50s [" % "Kill Repy resource monitor.")
logstream.flush()
# Mac or Linux...
if nonportable.ostype == 'Darwin' or nonportable.ostype == 'Linux':
# Get the location of python
locationproc = subprocess.Popen("which python",
shell=True,
stdout=subprocess.PIPE)
locationproc.wait()
location = locationproc.stdout.read().strip()
locationproc.stdout.close()
# Start the test
p = subprocess.Popen(
(location +
" repy.py restrictions.default killp_writetodisk.py").split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
pid = p.pid
# give it a few seconds to start...
time.sleep(4)
# find the orphaned child's PID... Different on Mac / Linux because of ps
# options...
if nonportable.ostype == 'Darwin' or nonportable.osrealtype == 'FreeBSD':
# run ps and print the 1st field (PID) of the line with 2nd field (PPID)
# equal to the parent pid...
childpidprocess = subprocess.Popen("ps -aO ppid | awk '{if ($2==" +
str(pid) + ") {print $1}}'",
stdout=subprocess.PIPE,
shell=True)
childpidstring = childpidprocess.stdout.read().strip()
childpidprocess.stdout.close()
elif nonportable.ostype == 'Linux':
# run ps and print the 2nd field (PID) of the line with 3rd field (PPID)
# equal to the parent pid...
childpidprocess = subprocess.Popen("ps -ef | awk '{if ($3==" +
str(pid) + ") {print $2}}'",
stdout=subprocess.PIPE,
shell=True)
childpidstring = childpidprocess.stdout.read().strip()
childpidprocess.stdout.close()
else:
print "Internal Error re-examining OS type '" + nonportable.ostype + "'!"
sys.exit(1)
# Windows
elif nonportable.ostype == 'Windows' or nonportable.ostype == 'WindowsCE':
# this is much easier because we don't worry about the path or have
# children to worry about.
p = subprocess.Popen(
"python repy.py restrictions.default killp_writetodisk.py".split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
pid = p.pid
# Wait while the process starts
time.sleep(4)
else:
print "Error: Unknown OS type '" + nonportable.ostype + "'!"
sys.exit(1)
# Kill the repy resource monitor
harshexit.portablekill(pid)
time.sleep(1)
# See ticket #413 and #421
# This is a workaround for the possibility that the repy child was sleeping
if nonportable.ostype == 'Darwin' or nonportable.ostype == 'Linux':
# Send SIGCONT to the child if the process still exists, it should wake up,
# get a read error when checking the pipe, and then exit...
try:
os.kill(int(childpidstring), signal.SIGCONT)
except OSError:
# the child has likely exited
pass
except ValueError:
# This happens when the child PID is ''
pass
# Wait for the signal to take effect
time.sleep(1)
try:
# Make sure the file size is not changing
firstsize = os.path.getsize("junk_test.out")
time.sleep(1)
secondsize = os.path.getsize("junk_test.out")
except (OSError, IOError):
failcount = failcount + 1
logstream.write("FAILED]\n")
endput = endput + "Killing Repy's resource monitor did not stop repy (size error)!\n\n"
else:
if firstsize == secondsize:
passcount = passcount + 1
logstream.write(" PASS ]\n")
else:
failcount = failcount + 1
logstream.write("FAILED]\n")
endput = endput + "Killing Repy's resource monitor did not stop repy!\n\n"
# Close the pipes
p.stdout.close()
p.stderr.close()
# Make sure repy.py works when invoked with options in a various order (to
# test that adding getopt worked.)
logstream.write("Running test %-50s [" % "repy.py takes args any order")
test_process = subprocess.Popen([
"python", "repy.py", "--simple", "--logfile", "log.log",
"restrictions.loose", "testoptions.py"
])
test_process.wait()
# Check that log was created.
if os.path.exists("log.log.old"):
os.remove("log.log.old")
test_process = subprocess.Popen([
"python", "repy.py", "--logfile", "log.log", "--simple",
"restrictions.loose", "testoptions.py"
])
test_process.wait()
# Check that log was created (again).
if os.path.exists("log.log.old"):
passcount += 1
logstream.write(" PASS ]\n")
os.remove("log.log.old")
else:
failcount += 1
logstream.write("FAILED]\n")
endput = endput + "Passing arguments in the opposite order failed!\n\n"
else:
failcount += 1
logstream.write("FAILED]\n")
endput = endput + "Test for passing arguments in the opposite order failed!\n\n"
logstream.flush()
raise e
else:
kill_repy = True
error_msg = repr(
exp) + " Monitor death! Impolitely killing repy process!"
monitor_exit_code = 98
finally:
if (error_msg):
print >> sys.stderr, error_msg
# Repy/Montior proccesses both _exit, so the thread should be stopped anyway
# nmstatusinterface.stop()
if (kill_repy):
harshexit.portablekill(repypid)
# XXX LP: Is this actually doeing something???
try:
statusstorage.write_status("Terminated")
except:
pass
# The monitor process (child) should always exit this way on android
# because we don't want the child to return back to Java if repy (parent)
# is not alive anymore, which it should not be at this point
harshexit.harshexit(monitor_exit_code)
def resource_monitor(repypid, pipe_handle):
"""
def main():
repy_args = [
sys.executable, 'repy.py', 'restrictions.default',
'test_killp_writetodisk.py'
]
if nonportable.ostype == 'Darwin' or nonportable.ostype == 'Linux':
process = subprocess.Popen(repy_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
pid = process.pid
# Give it a few seconds to start.
time.sleep(4)
# Find the orphaned child's PID.
# Run <ps> and print the first field (PID) of the line with second
# field (PPID) equal to the parent pid.
if nonportable.ostype == 'Darwin' or nonportable.osrealtype == 'FreeBSD':
command = "ps -aO ppid | awk '{if ($2==" + str(
pid) + ") {print $1}}'"
elif nonportable.ostype == 'Linux':
command = "ps -ef | awk '{if ($3==" + str(pid) + ") {print $2}}'"
(out, error) = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True).communicate()
child_pid = out.strip()
# Windows
elif nonportable.ostype == 'Windows' or nonportable.ostype == 'WindowsCE':
# This is much easier because we don't worry about the path or have
# children to worry about.
process = subprocess.Popen(repy_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
pid = process.pid
# Wait while the process starts.
time.sleep(4)
else:
print "Error: Unknown OS type '" + nonportable.ostype + "'!"
sys.exit(1)
# Kill the repy resource monitor.
harshexit.portablekill(pid)
time.sleep(1)
# See ticket #413 and #421
# This is a workaround for the possibility that the repy child was sleeping
if nonportable.ostype == 'Darwin' or nonportable.ostype == 'Linux':
# Send SIGCONT to the child if the process still exists, it should wake up,
# get a read error when checking the pipe, and then exit...
try:
os.kill(int(child_pid), signal.SIGCONT)
except OSError:
pass # The child has likely exited
# Wait for the signal to take effect
time.sleep(1)
# Make sure the file size is not changing
firstsize = os.path.getsize("junk_test.out")
time.sleep(1)
secondsize = os.path.getsize("junk_test.out")
if firstsize != secondsize: print 'FAIL'
def safe_check(code):
"""Check the code to be safe."""
# NOTE: This code will not work in Windows Mobile due to the reliance on subprocess
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR, "safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen([sys.executable, path_to_safe_check],stdin=subprocess.PIPE, stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
status = None
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while status == None and (nonportable.getruntime() - starttime < EVALUTATION_TIMEOUT):
status = proc.poll()
time.sleep(0.02)
else:
# Check if the process is still running
if status == None:
# Try to terminate the external process
try:
harshexit.portablekill(proc.pid)
except:
pass
# Raise an exception
raise Exception, "Evaluation of code safety exceeded timeout threshold ("+str(nonportable.getruntime() - starttime)+" seconds)"
# Read the output and close the pipe
rawoutput = proc.stdout.read()
proc.stdout.close()
# Interim fix for #1080: Get rid of stray debugging output on Android
# of the form "dlopen libpython2.6.so" and "dlopen /system/lib/libc.so",
# yet preserve all of the other output (including empty lines).
output = ""
for line in rawoutput.split("\n"):
# Preserve empty lines
if line == "":
output += "\n"
continue
# Suppress debug messages we know can turn up
wordlist = line.split()
if wordlist[0]=="dlopen":
if wordlist[-1]=="/system/lib/libc.so":
continue
if wordlist[-1].startswith("libpython") and \
wordlist[-1].endswith(".so"):
# We expect "libpython" + version number + ".so".
# The version number should be a string convertible to float.
# If it's not, raise an exception.
try:
versionstring = (wordlist[-1].replace("libpython",
"")).replace(".so", "")
junk = float(versionstring)
except TypeError, ValueError:
raise Exception("Unexpected debug output '" + line +
"' while evaluating code safety!")
else:
output += line + "\n"
