def should_start_waitable_thread(threadid, threadname):
# first time! Let's init!
if threadid not in thread_starttime:
thread_waittime[threadid] = minwaittime
thread_starttime[threadid] = 0.0
# If asking about advert thread and node_reset_config specifies to reset it,
# then return True
if threadid == 'advert' and node_reset_config['reset_advert']:
# Before returning, turn off the reset flag
node_reset_config['reset_advert'] = False
return True
# If it has been started, and the elapsed time is too short, always return
# False to say it shouldn't be restarted
if thread_starttime[threadid] and nonportable.getruntime() - thread_starttime[threadid] < thread_waittime[threadid]:
return False
for thread in threading.enumerate():
if threadname in str(thread):
# running now. If it's run for a reasonable time, let's reduce the
# wait time...
if nonportable.getruntime() - thread_starttime[threadid] > reasonableruntime:
thread_waittime[threadid] = max(minwaittime, thread_waittime[threadid]-decreaseamount)
return False
else:
return True
def lookup_timedout():
"""
<Purpose>
Waits for lookup_done_event and notifies the folks on the
notify_list (global var) of the lookup timeout.
<Arguments>
None.
<Exceptions>
None.
<Side Effects>
Sends an email to the notify_list folks
<Returns>
None.
"""
integrationtestlib.log("in lookup_timedout()")
notify_msg = "Centralized lookup failed -- lookup_timedout() fired after 30 sec."
# wait for the event to be set, timeout after 30 minutes
wait_time = 1800
tstamp_before_wait = nonportable.getruntime()
lookup_done_event.wait(wait_time)
tstamp_after_wait = nonportable.getruntime()
t_waited = tstamp_after_wait - tstamp_before_wait
if abs(wait_time - t_waited) < 5:
notify_msg += " And lookup stalled for over 30 minutes (max timeout value)."
else:
notify_msg += " And lookup stalled for " + str(t_waited) + " seconds"
integrationtestlib.notify(notify_msg)
return
def lookup_timedout():
"""
<Purpose>
Waits for lookup_done_event and notifies the folks on the
notify_list (global var) of the lookup timeout.
<Arguments>
None.
<Exceptions>
None.
<Side Effects>
Sends an email to the notify_list folks
<Returns>
None.
"""
integrationtestlib.log("in lookup_timedout()")
notify_msg = "Centralized lookup failed -- lookup_timedout() fired after 30 sec."
# wait for the event to be set, timeout after 30 minutes
wait_time = 1800
tstamp_before_wait = nonportable.getruntime()
lookup_done_event.wait(wait_time)
tstamp_after_wait = nonportable.getruntime()
t_waited = tstamp_after_wait - tstamp_before_wait
if abs(wait_time - t_waited) < 5:
notify_msg += " And lookup stalled for over 30 minutes (max timeout value)."
else:
notify_msg += " And lookup stalled for " + str(t_waited) + " seconds"
integrationtestlib.notify(notify_msg)
return
def should_start_waitable_thread(threadid, threadname):
# first time! Let's init!
if threadid not in thread_starttime:
thread_waittime[threadid] = minwaittime
thread_starttime[threadid] = 0.0
# If asking about advert thread and node_reset_config specifies to reset it,
# then return True
if threadid == 'advert' and node_reset_config['reset_advert']:
# Before returning, turn off the reset flag
node_reset_config['reset_advert'] = False
return True
# If it has been started, and the elapsed time is too short, always return
# False to say it shouldn't be restarted
if thread_starttime[threadid] and nonportable.getruntime(
) - thread_starttime[threadid] < thread_waittime[threadid]:
return False
for thread in threading.enumerate():
if threadname in str(thread):
# running now. If it's run for a reasonable time, let's reduce the
# wait time...
if nonportable.getruntime(
) - thread_starttime[threadid] > reasonableruntime:
thread_waittime[threadid] = max(
minwaittime, thread_waittime[threadid] - decreaseamount)
return False
else:
return True
def safe_check_subprocess(code):
"""
<Purpose>
Runs safe_check() in a subprocess. This is done because the AST safe_check()
creates uses a large amount of RAM. By running safe_check() in a subprocess
we can guarantee that the memory will be reclaimed when the process ends.
<Arguments>
code: See safe_check.
<Exceptions>
As with safe_check.
<Return>
See safe_check.
"""
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR, "safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen([sys.executable, path_to_safe_check],stdin=subprocess.PIPE, stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while nonportable.getruntime() - starttime < EVALUTATION_TIMEOUT:
# Did the process finish running?
if proc.poll() != None:
break;
time.sleep(0.02)
else:
# Kill the timed-out process
try:
harshexit.portablekill(proc.pid)
except:
pass
raise Exception, "Evaluation of code safety exceeded timeout threshold \
("+str(nonportable.getruntime() - starttime)+" seconds)"
# Read the output and close the pipe
output = proc.stdout.read()
proc.stdout.close()
# Check the output, None is success, else it is a failure
if output == "None":
return True
# If there is no output, this is a fatal error condition
elif output == "":
raise Exception, "Fatal error while evaluating code safety!"
else:
# Raise the error from the output
raise exception_hierarchy.SafeException, output
def safe_check(code):
"""Check the code to be safe."""
return True
# NOTE: This code will not work in Windows Mobile due to the reliance on subprocess
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR,
"safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen(["python", path_to_safe_check],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
status = None
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while status == None and (nonportable.getruntime() - starttime <
EVALUTATION_TIMEOUT):
status = proc.poll()
time.sleep(0.02)
else:
# Check if the process is still running
if status == None:
# Try to terminate the external process
try:
harshexit.portablekill(proc.pid)
except:
pass
# Raise an exception
raise Exception, "Evaluation of code safety exceeded timeout threshold (" + str(
nonportable.getruntime() - starttime) + " seconds)"
# Read the output and close the pipe
output = proc.stdout.read()
proc.stdout.close()
# Check the output, None is success, else it is a failure
if output == "None":
return True
# If there is no output, this is a fatal error condition
elif output == "":
raise Exception, "Fatal error while evaluating code safety!"
else:
# Raise the error from the output
raise exception_hierarchy.SafeException, output
def safe_check(code):
"""Check the code to be safe."""
return True
# NOTE: This code will not work in Windows Mobile due to the reliance on subprocess
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR, "safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen(["python", path_to_safe_check],stdin=subprocess.PIPE, stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
status = None
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while status == None and (nonportable.getruntime() - starttime < EVALUTATION_TIMEOUT):
status = proc.poll()
time.sleep(0.02)
else:
# Check if the process is still running
if status == None:
# Try to terminate the external process
try:
harshexit.portablekill(proc.pid)
except:
pass
# Raise an exception
raise Exception, "Evaluation of code safety exceeded timeout threshold ("+str(nonportable.getruntime() - starttime)+" seconds)"
# Read the output and close the pipe
output = proc.stdout.read()
proc.stdout.close()
# Check the output, None is success, else it is a failure
if output == "None":
return True
# If there is no output, this is a fatal error condition
elif output == "":
raise Exception, "Fatal error while evaluating code safety!"
else:
# Raise the error from the output
raise exception_hierarchy.SafeException, output
def measure_write(write_file_obj, blocksize, totalbytes, use_sync=False):
"""
<Purpose>
Attempts to measure the disk write rate by writing totalbytes bytes to a
temporary file (performing a flush each time blocksize many bytes have been
written), timing how long it took, and dividing num_bytes by the time
to get the write rate.
<Arguments>
write_file - The file to be written to. This should be an already opened
file handle that was opened with write access.
blocksize - The amount of data in bytes to write before a flush is performed.
totalbytes - The total number of bytes that should be written for the test.
use_sync - Set to True if sync should be used to make sure the data is
actually written to disk. Should not be set to True on
Windows because sync does not exist there. Defaults to False.
<Side Effects>
Creates a file of size totalbytes.
<Exceptions>
Exceptions could be thrown if there is a problem opening/writing the file.
ZeroDivisionError if the drive is to fast for the accuracy of the clock (for
a fast drive in combination with a time that provided poor granularity.
<Return>
A tuple (rate, fn) where rate is the measured write rate, and fn is the
name of the file created. It is up to the caller to ensure that this
file is deleted. We do not delete it here because it will likely be
useful in doing the read rate measurments.
"""
start_time = nonportable.getruntime()
for trial in range(0, totalbytes, blocksize):
write_file_obj.write(' ' * blocksize)
#write_file_obj.flush()
#if use_sync:
# # Only use sync if it is requested. See comment at import for explanation.
# libc.sync()
write_file_obj.flush()
end_time = nonportable.getruntime()
return (totalbytes)/(end_time - start_time)
def measure_write(write_file_obj, blocksize, totalbytes, use_sync=False):
"""
<Purpose>
Attempts to measure the disk write rate by writing totalbytes bytes to a
temporary file (performing a flush each time blocksize many bytes have been
written), timing how long it took, and dividing num_bytes by the time
to get the write rate.
<Arguments>
write_file - The file to be written to. This should be an already opened
file handle that was opened with write access.
blocksize - The amount of data in bytes to write before a flush is performed.
totalbytes - The total number of bytes that should be written for the test.
use_sync - Set to True if sync should be used to make sure the data is
actually written to disk. Should not be set to True on
Windows because sync does not exist there. Defaults to False.
<Side Effects>
Creates a file of size totalbytes.
<Exceptions>
Exceptions could be thrown if there is a problem opening/writing the file.
ZeroDivisionError if the drive is to fast for the accuracy of the clock (for
a fast drive in combination with a time that provided poor granularity.
<Return>
A tuple (rate, fn) where rate is the measured write rate, and fn is the
name of the file created. It is up to the caller to ensure that this
file is deleted. We do not delete it here because it will likely be
useful in doing the read rate measurments.
"""
start_time = nonportable.getruntime()
for trial in range(0, totalbytes, blocksize):
write_file_obj.write(' ' * blocksize)
#write_file_obj.flush()
#if use_sync:
# # Only use sync if it is requested. See comment at import for explanation.
# libc.sync()
write_file_obj.flush()
end_time = nonportable.getruntime()
return (totalbytes) / (end_time - start_time)
def do_sleep(seconds):
# Using getruntime() in lieu of time.time() because we want elapsed time
# regardless of the oddities of NTP
start = nonportable.getruntime()
sleeptime = seconds
# return no earlier than the finish time
finish = start + seconds
while sleeptime > 0.0:
time.sleep(sleeptime)
now = nonportable.getruntime()
# If sleeptime > 0.0 then I woke up early...
sleeptime = finish - now
def _update_resource_consumption_table(resource, resource_allowed_dict, consumed_resource_dict):
thetime = nonportable.getruntime()
# I'm going to reduce all renewable resources by the appropriate amount given
# the amount of elapsed time.
elapsedtime = thetime - consumed_resource_dict['renewable_update_time'][resource]
consumed_resource_dict['renewable_update_time'][resource] = thetime
if elapsedtime < 0:
# A negative number (likely a NTP reset). Let's just ignore it.
return
# Remove the charge
reduction = elapsedtime * resource_allowed_dict[resource]
if reduction > consumed_resource_dict[resource]:
# It would reduce it below zero (so put it at zero)
consumed_resource_dict[resource] = 0.0
else:
# Subtract some for elapsed time...
consumed_resource_dict[resource] = consumed_resource_dict[resource] - reduction
def _update_resource_consumption_table(resource, resource_allowed_dict,
consumed_resource_dict):
thetime = nonportable.getruntime()
# I'm going to reduce all renewable resources by the appropriate amount given
# the amount of elapsed time.
elapsedtime = thetime - consumed_resource_dict['renewable_update_time'][
resource]
consumed_resource_dict['renewable_update_time'][resource] = thetime
if elapsedtime < 0:
# A negative number (likely a NTP reset). Let's just ignore it.
return
# Remove the charge
reduction = elapsedtime * resource_allowed_dict[resource]
if reduction > consumed_resource_dict[resource]:
# It would reduce it below zero (so put it at zero)
consumed_resource_dict[resource] = 0.0
else:
# Subtract some for elapsed time...
consumed_resource_dict[
resource] = consumed_resource_dict[resource] - reduction
def should_start_waitable_thread(threadid, threadname):
# first time! Let's init!
if threadid not in thread_starttime:
thread_waittime[threadid] = minwaittime
thread_starttime[threadid] = 0.0
# If it has been started, and the elapsed time is too short, always return
# False to say it shouldn't be restarted
if thread_starttime[threadid] and nonportable.getruntime() - thread_starttime[threadid] < thread_waittime[threadid]:
return False
for thread in threading.enumerate():
if threadname in str(thread):
# running now. If it's run for a reasonable time, let's reduce the
# wait time...
if nonportable.getruntime() - thread_starttime[threadid] > reasonableruntime:
thread_waittime[threadid] = max(minwaittime, thread_waittime[threadid]-decreaseamount)
return False
else:
return True
def measure_read(read_file_obj, blocksize):
"""
<Purpose>
Attempts to measure the disk read rate by reading blocksize bytes from a
temp file, timing how long it took, and dividing blocksize by the time
to get the read rate. Note that at this time, read rate is far too fast
because it reads what was just written. It should be ok to just take the
value given by the write test and use it for both read and write rate.
<Arguments>
read_file_obj - The file object that is to be read from for the read test.
This file object is not closed by this function.
blocksize - The number of bytes that should be read to determine the
read rate.
<Side Effects>
None
<Exceptions>
Exceptions could be thrown if there is a problem opening/reading the file.
ZeroDivisionError if the drive is to fast for the accuracy of the clock (for
a fast drive in combination with a time that provided poor granularity.
<Return>
A tuple (rate, blocksize) where rate is the measured read rate, and
blocksize is the number of bytes actually read. It will be no more than
what was actually asked for, but it could be less if the given file was
too short. The read rate will have been calculated using the returned
blocksize.
"""
# Time how long it takes to read in blocksize.
start_time = nonportable.getruntime()
junk_data = read_file_obj.read(blocksize)
end_time = nonportable.getruntime()
blocksize = len(junk_data)
return (blocksize / (end_time - start_time), blocksize)
def measure_read(read_file_obj, blocksize):
"""
<Purpose>
Attempts to measure the disk read rate by reading blocksize bytes from a
temp file, timing how long it took, and dividing blocksize by the time
to get the read rate. Note that at this time, read rate is far too fast
because it reads what was just written. It should be ok to just take the
value given by the write test and use it for both read and write rate.
<Arguments>
read_file_obj - The file object that is to be read from for the read test.
This file object is not closed by this function.
blocksize - The number of bytes that should be read to determine the
read rate.
<Side Effects>
None
<Exceptions>
Exceptions could be thrown if there is a problem opening/reading the file.
ZeroDivisionError if the drive is to fast for the accuracy of the clock (for
a fast drive in combination with a time that provided poor granularity.
<Return>
A tuple (rate, blocksize) where rate is the measured read rate, and
blocksize is the number of bytes actually read. It will be no more than
what was actually asked for, but it could be less if the given file was
too short. The read rate will have been calculated using the returned
blocksize.
"""
# Time how long it takes to read in blocksize.
start_time = nonportable.getruntime()
junk_data = read_file_obj.read(blocksize)
end_time = nonportable.getruntime()
blocksize = len(junk_data)
return (blocksize/(end_time - start_time), blocksize)
def stopvessel(vesselname, exitparams=(44, '')):
if vesselname not in vesseldict:
raise BadRequest, "No such vessel"
# this is broken out to prevent a race between checking the status and
# reporting the error
currentstatus = vesseldict[vesselname]['status']
# It must be started for us to stop it...
if currentstatus != 'Started':
raise BadRequest("Cannot stop vessel with status '" + currentstatus +
"'")
# Armon: Create the stop file, using a .tmp extension
fileo = open(vesseldict[vesselname]['stopfilename'] + ".tmp", "w")
# Write out the stop string, Format: EINT;EMESG
fileo.write(str(exitparams[0]) + ";" + exitparams[1])
# Close the object
fileo.close()
# Rename the tmp file to the actual stopfile, this should be detected by repy now
os.rename(vesseldict[vesselname]['stopfilename'] + ".tmp",
vesseldict[vesselname]['stopfilename'])
starttime = nonportable.getruntime()
# wait for up to 10 seconds for it to stop (else return an error)
while nonportable.getruntime() - starttime < 10:
if vesseldict[vesselname]['status'] != 'Started':
break
# sleep while busy waiting...
time.sleep(.5)
else:
return "May not have stopped in a timely manner\nWarning"
return vesseldict[vesselname]['status'] + "\nSuccess"
def sleep(seconds):
"""
<Purpose>
Allow the current event to pause execution (similar to time.sleep()).
This function will not return early for any reason
<Arguments>
seconds:
The number of seconds to sleep. This can be a floating point value
<Exceptions>
RepyArgumentException if seconds is not an int/long/float.
<Side Effects>
None.
<Returns>
None.
"""
# Check seconds to ensure it is a valid type.
if type(seconds) not in [float, int]:
raise RepyArgumentError("Invalid type " + str(type(seconds)))
# Using getruntime() in lieu of time.time() because we want elapsed time
# regardless of the oddities of NTP
start = nonportable.getruntime()
sleeptime = seconds
# Return no earlier than the finish time
finish = start + seconds
while sleeptime > 0.0:
time.sleep(sleeptime)
# If sleeptime > 0.0 then I woke up early...
sleeptime = finish - nonportable.getruntime()
def stopvessel(vesselname,exitparams=(44, '')):
if vesselname not in vesseldict:
raise BadRequest, "No such vessel"
# this is broken out to prevent a race between checking the status and
# reporting the error
currentstatus = vesseldict[vesselname]['status']
# It must be started for us to stop it...
if currentstatus != 'Started':
raise BadRequest("Cannot stop vessel with status '"+currentstatus+"'")
# Armon: Create the stop file, using a .tmp extension
fileo = open(vesseldict[vesselname]['stopfilename']+".tmp","w")
# Write out the stop string, Format: EINT;EMESG
fileo.write(str(exitparams[0]) + ";" + exitparams[1])
# Close the object
fileo.close()
# Rename the tmp file to the actual stopfile, this should be detected by repy now
os.rename(vesseldict[vesselname]['stopfilename']+".tmp", vesseldict[vesselname]['stopfilename'])
starttime = nonportable.getruntime()
# wait for up to 10 seconds for it to stop (else return an error)
while nonportable.getruntime()-starttime < 10:
if vesseldict[vesselname]['status'] != 'Started':
break
# sleep while busy waiting...
time.sleep(.5)
else:
return "May not have stopped in a timely manner\nWarning"
return vesseldict[vesselname]['status']+"\nSuccess"
def sleep(seconds):
"""
<Purpose>
Allow the current event to pause execution (similar to time.sleep()).
This function will not return early for any reason
<Arguments>
seconds:
The number of seconds to sleep. This can be a floating point value
<Exceptions>
RepyArgumentException if seconds is not an int/long/float.
<Side Effects>
None.
<Returns>
None.
"""
# Check seconds to ensure it is a valid type.
if type(seconds) not in [long, float, int]:
raise RepyArgumentError("Invalid type " + str(type(seconds)))
# Using getruntime() in lieu of time.time() because we want elapsed time
# regardless of the oddities of NTP
start = nonportable.getruntime()
sleeptime = seconds
# Return no earlier than the finish time
finish = start + seconds
while sleeptime > 0.0:
time.sleep(sleeptime)
# If sleeptime > 0.0 then I woke up early...
sleeptime = finish - nonportable.getruntime()
def calculate_sleeptimes():
'''
<Purpose>
calculate amount of sleep time for each process
'''
global pid_list
global process_cpu_info
## calculate ecalapsed time of the last "period"
global last_runtime
global cpu_limit
this_runtime = nonportable.getruntime()
# print "Debug - this_runtime: ", this_runtime
# if this_runtime == 0.0:
# print "Debug - this_runtime == 0.0 "
# return
elapsedtime = this_runtime - last_runtime
# print "Debug - last_runtime: ", last_runtime
# print "Debug - this_runtime: ", this_runtime
# print "\n"
# print "Debug1 - elapsedtime: ", elapsedtime
if elapsedtime < 0.0:
print "Error: elapsed time < 0"
last_runtime = this_runtime
for pid in pid_list:
last_user_time = process_cpu_info[pid][0]
if ostype == 'Windows':
this_user_time = windows_api.get_process_cpu_time(pid)
elif ostype == 'Linux':
this_user_time = linux_api.get_process_cpu_time(pid)
else:
raise UnsupportedSystemException, "Unsupported system type: '"+osrealtype+"' (alias: "+ostype+")"
# percent used is the amount of change divided by the time...
percentused = (this_user_time - last_user_time) / elapsedtime
# print "Debug1 - pid:",pid, "percentused: ",percentused
# Calculate amount of time to sleep for
# print "Debug - cpu_limit: ",cpu_limit[pid]
sleeptime = nanny.calculate_cpu_sleep_interval(cpu_limit[pid], percentused,elapsedtime)
# print "Debug1 - pid:",pid,"sleeptime: ",sleeptime
process_cpu_info[pid][1] = sleeptime
process_cpu_info[pid][0] = this_user_time
def getruntime():
"""
<Purpose>
Return the amount of time the program has been running. This is in
wall clock time. This is guaranteed to be monotonic.
<Arguments>
None
<Exceptions>
None.
<Side Effects>
None
<Returns>
The elapsed time as float
"""
return nonportable.getruntime()
def getruntime():
"""
<Purpose>
Return the amount of time the program has been running. This is in
wall clock time. This is guaranteed to be monotonic.
<Arguments>
None
<Exceptions>
None.
<Side Effects>
None
<Returns>
The elapsed time as float
"""
return nonportable.getruntime()
def getruntime():
"""
<Purpose>
Return the amount of time the program has been running. This is in
wall clock time. This function is not guaranteed to always return
increasing values due to NTP, etc.
<Arguments>
None
<Exceptions>
None.
<Side Effects>
None
<Remarks>
Accurate granularity not guaranteed past 1 second.
<Returns>
The elapsed time as float
"""
restrictions.assertisallowed('getruntime')
return nonportable.getruntime()
def getruntime():
"""
<Purpose>
Return the amount of time the program has been running. This is in
wall clock time. This function is not guaranteed to always return
increasing values due to NTP, etc.
<Arguments>
None
<Exceptions>
None.
<Side Effects>
None
<Remarks>
Accurate granularity not guaranteed past 1 second.
<Returns>
The elapsed time as float
"""
restrictions.assertisallowed('getruntime')
return nonportable.getruntime()
def safe_check(code):
"""Check the code to be safe."""
# NOTE: This code will not work in Windows Mobile due to the reliance on subprocess
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR,
"safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen([sys.executable, path_to_safe_check],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
status = None
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while status == None and (nonportable.getruntime() - starttime <
EVALUTATION_TIMEOUT):
status = proc.poll()
time.sleep(0.02)
else:
# Check if the process is still running
if status == None:
# Try to terminate the external process
try:
harshexit.portablekill(proc.pid)
except:
pass
# Raise an exception
raise Exception, "Evaluation of code safety exceeded timeout threshold (" + str(
nonportable.getruntime() - starttime) + " seconds)"
# Read the output and close the pipe
rawoutput = proc.stdout.read()
proc.stdout.close()
# Interim fix for #1080: Get rid of stray debugging output on Android
# of the form "dlopen libpython2.6.so" and "dlopen /system/lib/libc.so",
# yet preserve all of the other output (including empty lines).
output = ""
for line in rawoutput.split("\n"):
# Preserve empty lines
if line == "":
output += "\n"
continue
# Suppress debug messages we know can turn up
wordlist = line.split()
if wordlist[0] == "dlopen":
if wordlist[-1] == "/system/lib/libc.so":
continue
if wordlist[-1].startswith("libpython") and \
wordlist[-1].endswith(".so"):
# We expect "libpython" + version number + ".so".
# The version number should be a string convertible to float.
# If it's not, raise an exception.
try:
versionstring = (wordlist[-1].replace("libpython",
"")).replace(
".so", "")
junk = float(versionstring)
except TypeError, ValueError:
raise Exception("Unexpected debug output '" + line +
"' while evaluating code safety!")
else:
output += line + "\n"
# Start advert thread...
start_advert_thread(vesseldict, myname, configuration['publickey'])
# Start status thread...
start_status_thread(vesseldict, configuration['pollfrequency'])
# we should be all set up now.
servicelogger.log("[INFO]:Started")
# I will count my iterations through the loop so that I can log a message
# periodically. This makes it clear I am alive.
times_through_the_loop = 0
# Setup the initial time for checking Affix status.
last_check_affix_time = nonportable.getruntime()
# BUG: Need to exit all when we're being upgraded
while True:
# E.K Previous there was a check to ensure that the accepter
# thread was started. There is no way to actually check this
# and this code was never executed, so i removed it completely
myname = node_reset_config['name']
if not is_worker_thread_started():
servicelogger.log("[WARN]:At " + str(time.time()) +
" restarting worker...")
start_worker_thread(configuration['pollfrequency'])
start_advert_thread(vesseldict, myname, configuration['publickey'])
# Start status thread...
start_status_thread(vesseldict,configuration['pollfrequency'])
# we should be all set up now.
servicelogger.log("[INFO]:Started")
# I will count my iterations through the loop so that I can log a message
# periodically. This makes it clear I am alive.
times_through_the_loop = 0
# Setup the initial time for checking Affix status.
last_check_affix_time = nonportable.getruntime()
# BUG: Need to exit all when we're being upgraded
while True:
# E.K Previous there was a check to ensure that the accepter
# thread was started. There is no way to actually check this
# and this code was never executed, so i removed it completely
myname = node_reset_config['name']
if not is_worker_thread_started():
servicelogger.log("[WARN]:At " + str(time.time()) + " restarting worker...")
start_worker_thread(configuration['pollfrequency'])
def startvessel_ex(vesselname, prog_platform, argstring):
# Convert the programming platform to lowercase to make
# it case insensitive.
prog_platform = prog_platform.lower()
if vesselname not in vesseldict:
raise BadRequest, "No such vessel"
if vesseldict[vesselname]['status'] == 'Started':
raise BadRequest("Vessel has already been started")
if prog_platform not in prog_platform_dir.keys():
raise BadRequest("Programming language platform is not supported.")
# remove any prior stop file so that we can start
if os.path.exists(vesseldict[vesselname]['stopfilename']):
os.remove(vesseldict[vesselname]['stopfilename'])
for char in argstring:
if char not in allowedchars:
raise BadRequest("Character '"+char+"' not allowed in arguments")
# I'm going to capture the status and timestamp and then check the see if
# the timestamp is updated...
oldstatus, oldtimestamp = statusstorage.read_status(vesseldict[vesselname]['statusfilename'])
# Armon: this is required to fetch the networkrestrictions information from the configuration
configuration = persist.restore_object("nodeman.cfg")
# Armon: Generate the IP/Iface preferences if they exist
ip_iface_preference_flags = []
ip_iface_preference_str = "" # Needed for Win Mobile
# Only add the flags if everything necessary exists
if 'networkrestrictions' in configuration and 'repy_restricted' in configuration['networkrestrictions'] \
and configuration['networkrestrictions']['repy_restricted'] and 'repy_user_preference' in configuration['networkrestrictions']:
# Generate the preference list
for (is_ip, value) in configuration['networkrestrictions']['repy_user_preference']:
# Append the correct flag
if is_ip:
ip_iface_preference_flags.append("--ip")
ip_iface_preference_str += "--ip "
else:
ip_iface_preference_flags.append("--iface")
ip_iface_preference_str += "--iface "
# Append the value
ip_iface_preference_flags.append(value)
ip_iface_preference_str += "'" + value + "' "
# Check for the --nootherips flag
if 'repy_nootherips' in configuration['networkrestrictions'] and configuration['networkrestrictions']['repy_nootherips']:
# Append the flag
ip_iface_preference_flags.append("--nootherips")
ip_iface_preference_str += "--nootherips "
# Find the location where the sandbox files is located. Location of repyV1, repyV2 etc.
prog_platform_location = os.path.join(prog_platform_dir[prog_platform], "repy.py")
# I use absolute paths so that repy can still find the files after it
# changes directories...
# Conrad: switched this to sequence-style Popen invocation so that spaces
# in files work. Switched it back to absolute paths.
command = [sys.executable, prog_platform_location] + ip_iface_preference_flags + [
"--logfile", os.path.abspath(vesseldict[vesselname]['logfilename']),
"--stop", os.path.abspath(vesseldict[vesselname]['stopfilename']),
"--status", os.path.abspath(vesseldict[vesselname]['statusfilename']),
"--cwd", os.path.abspath(vesselname),
"--servicelog",
"--execinfo",
os.path.abspath(vesseldict[vesselname]['resourcefilename'])] + argstring.split()
portable_popen.Popen(command)
starttime = nonportable.getruntime()
# wait for 10 seconds for it to start (else return an error)
while nonportable.getruntime()-starttime < 10:
newstatus, newtimestamp = statusstorage.read_status(vesseldict[vesselname]['statusfilename'])
# Great! The timestamp was updated... The new status is the result of
# our work. Let's tell the user what happened...
if newtimestamp != oldtimestamp and newstatus != None:
break
# sleep while busy waiting...
time.sleep(.5)
else:
return "Did not start in a timely manner\nWarning"
# We need to update the status in the table because the status thread might
# not notice this before our next request... (else occasional failures on XP)
nmstatusmonitor.update_status(vesseldict, vesselname, newstatus, newtimestamp)
return newstatus+"\nSuccess"
def started_waitable_thread(threadid):
thread_starttime[threadid] = nonportable.getruntime()
thread_waittime[threadid] = min(maxwaittime,
thread_waittime[threadid]**wait_exponent)
def safe_check_subprocess(code):
"""
<Purpose>
Runs safe_check() in a subprocess. This is done because the AST
safe_check() uses a large amount of RAM. By running safe_check() in a
subprocess we can guarantee that the memory will be reclaimed when the
process ends.
<Arguments>
code: See safe_check.
<Exceptions>
As with safe_check.
<Return>
See safe_check.
"""
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR,
"safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen([sys.executable, path_to_safe_check],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while nonportable.getruntime() - starttime < EVALUTATION_TIMEOUT:
# Did the process finish running?
if proc.poll() != None:
break
time.sleep(0.02)
else:
# Kill the timed-out process
try:
harshexit.portablekill(proc.pid)
except:
pass
raise Exception, "Evaluation of code safety exceeded timeout threshold \
(" + str(nonportable.getruntime() -
starttime) + " seconds)"
# Read the output and close the pipe
rawoutput = proc.stdout.read()
proc.stdout.close()
# Interim fix for SeattleTestbed/attic#1080:
# Get rid of stray debugging output on Android of the form
# `dlopen libpython2.6.so` and `dlopen /system/lib/libc.so`,
# yet preserve all of the other output (including empty lines).
if IS_ANDROID:
output = ""
for line in rawoutput.split("\n"):
# Preserve empty lines
if line == "":
output += "\n"
continue
# Suppress debug messages we know can turn up
wordlist = line.split()
if wordlist[0] == "dlopen":
if wordlist[-1] == "/system/lib/libc.so":
continue
if wordlist[-1].startswith("libpython") and \
wordlist[-1].endswith(".so"):
# We expect "libpython" + version number + ".so".
# The version number should be a string convertible to float.
# If it's not, raise an exception.
try:
versionstring = (wordlist[-1].replace("libpython",
"")).replace(
".so", "")
junk = float(versionstring)
except TypeError, ValueError:
raise Exception("Unexpected debug output '" + line +
"' while evaluating code safety!")
else:
output += line + "\n"
# Strip off the last newline character we added
output = output[0:-1]
def safe_check_subprocess(code):
"""
<Purpose>
Runs safe_check() in a subprocess. This is done because the AST safe_check()
creates uses a large amount of RAM. By running safe_check() in a subprocess
we can guarantee that the memory will be reclaimed when the process ends.
<Arguments>
code: See safe_check.
<Exceptions>
As with safe_check.
<Return>
See safe_check.
"""
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR, "safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen([sys.executable, path_to_safe_check],stdin=subprocess.PIPE, stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while nonportable.getruntime() - starttime < EVALUTATION_TIMEOUT:
# Did the process finish running?
if proc.poll() != None:
break;
time.sleep(0.02)
else:
# Kill the timed-out process
try:
harshexit.portablekill(proc.pid)
except:
pass
raise Exception, "Evaluation of code safety exceeded timeout threshold \
("+str(nonportable.getruntime() - starttime)+" seconds)"
# Read the output and close the pipe
rawoutput = proc.stdout.read()
proc.stdout.close()
# Interim fix for #1080: Get rid of stray debugging output on Android
# of the form "dlopen libpython2.6.so" and "dlopen /system/lib/libc.so",
# yet preserve all of the other output (including empty lines).
if IS_ANDROID:
output = ""
for line in rawoutput.split("\n"):
# Preserve empty lines
if line == "":
output += "\n"
continue
# Suppress debug messages we know can turn up
wordlist = line.split()
if wordlist[0]=="dlopen":
if wordlist[-1]=="/system/lib/libc.so":
continue
if wordlist[-1].startswith("libpython") and \
wordlist[-1].endswith(".so"):
# We expect "libpython" + version number + ".so".
# The version number should be a string convertible to float.
# If it's not, raise an exception.
try:
versionstring = (wordlist[-1].replace("libpython",
"")).replace(".so", "")
junk = float(versionstring)
except TypeError, ValueError:
raise Exception("Unexpected debug output '" + line +
"' while evaluating code safety!")
else:
output += line + "\n"
# Strip off the last newline character we added
output = output[0:-1]
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR,
"safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen([sys.executable, path_to_safe_check],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while nonportable.getruntime() - starttime < EVALUTATION_TIMEOUT:
# Did the process finish running?
if (IS_ANDROID and os.waitpid(procpid, os.WNOHANG) != (0, 0)) or \
(not IS_ANDROID and proc.poll() != None):
break
time.sleep(0.02)
else:
# Kill the timed-out process
try:
harshexit.portablekill(procpid)
except:
pass
def safe_check_subprocess(code):
"""
<Purpose>
Runs safe_check() in a subprocess. This is done because the AST safe_check()
creates uses a large amount of RAM. By running safe_check() in a subprocess
we can guarantee that the memory will be reclaimed when the process ends.
<Arguments>
code: See safe_check.
<Exceptions>
As with safe_check.
<Return>
See safe_check.
"""
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR,
"safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen([sys.executable, path_to_safe_check],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while nonportable.getruntime() - starttime < EVALUTATION_TIMEOUT:
# Did the process finish running?
if proc.poll() != None:
break
time.sleep(0.02)
else:
# Kill the timed-out process
try:
harshexit.portablekill(proc.pid)
except:
pass
raise Exception, "Evaluation of code safety exceeded timeout threshold \
(" + str(nonportable.getruntime() -
starttime) + " seconds)"
# Read the output and close the pipe
output = proc.stdout.read()
proc.stdout.close()
# Check the output, None is success, else it is a failure
if output == "None":
return True
# If there is no output, this is a fatal error condition
elif output == "":
raise Exception, "Fatal error while evaluating code safety!"
else:
# Raise the error from the output
raise exception_hierarchy.SafeException, output
start_advert_thread(vesseldict, myname, configuration['publickey'])
# Start status thread...
start_status_thread(vesseldict,configuration['pollfrequency'])
# we should be all set up now.
servicelogger.log("[INFO]:Started")
# I will count my iterations through the loop so that I can log a message
# periodically. This makes it clear I am alive.
times_through_the_loop = 0
# Setup the initial time for checking Affix status.
last_check_affix_time = nonportable.getruntime()
# BUG: Need to exit all when we're being upgraded
while True:
# E.K Previous there was a check to ensure that the accepter
# thread was started. There is no way to actually check this
# and this code was never executed, so i removed it completely
myname = node_reset_config['name']
if not is_worker_thread_started():
servicelogger.log("[WARN]:At " + str(time.time()) + " restarting worker...")
start_worker_thread(configuration['pollfrequency'])
def startvessel_ex(vesselname, prog_platform, argstring):
# Convert the programming platform to lowercase to make
# it case insensitive.
prog_platform = prog_platform.lower()
if vesselname not in vesseldict:
raise BadRequest, "No such vessel"
if vesseldict[vesselname]['status'] == 'Started':
raise BadRequest("Vessel has already been started")
if prog_platform not in prog_platform_dir.keys():
raise BadRequest("Programming language platform is not supported.")
# remove any prior stop file so that we can start
if os.path.exists(vesseldict[vesselname]['stopfilename']):
os.remove(vesseldict[vesselname]['stopfilename'])
for char in argstring:
if char not in allowedchars:
raise BadRequest("Character '" + char +
"' not allowed in arguments")
# I'm going to capture the status and timestamp and then check the see if
# the timestamp is updated...
oldstatus, oldtimestamp = statusstorage.read_status(
vesseldict[vesselname]['statusfilename'])
# Armon: this is required to fetch the networkrestrictions information from the configuration
configuration = persist.restore_object("nodeman.cfg")
# Armon: Generate the IP/Iface preferences if they exist
ip_iface_preference_flags = []
ip_iface_preference_str = "" # Needed for Win Mobile
# Only add the flags if everything necessary exists
if 'networkrestrictions' in configuration and 'repy_restricted' in configuration['networkrestrictions'] \
and configuration['networkrestrictions']['repy_restricted'] and 'repy_user_preference' in configuration['networkrestrictions']:
# Generate the preference list
for (is_ip, value
) in configuration['networkrestrictions']['repy_user_preference']:
# Append the correct flag
if is_ip:
ip_iface_preference_flags.append("--ip")
ip_iface_preference_str += "--ip "
else:
ip_iface_preference_flags.append("--iface")
ip_iface_preference_str += "--iface "
# Append the value
ip_iface_preference_flags.append(value)
ip_iface_preference_str += "'" + value + "' "
# Check for the --nootherips flag
if 'repy_nootherips' in configuration[
'networkrestrictions'] and configuration[
'networkrestrictions']['repy_nootherips']:
# Append the flag
ip_iface_preference_flags.append("--nootherips")
ip_iface_preference_str += "--nootherips "
# Find the location where the sandbox files is located. Location of repyV1, repyV2 etc.
prog_platform_location = os.path.join(prog_platform_dir[prog_platform],
"repy.py")
# I use absolute paths so that repy can still find the files after it
# changes directories...
# Conrad: switched this to sequence-style Popen invocation so that spaces
# in files work. Switched it back to absolute paths.
command = [
sys.executable, prog_platform_location
] + ip_iface_preference_flags + [
"--logfile",
os.path.abspath(vesseldict[vesselname]['logfilename']), "--stop",
os.path.abspath(vesseldict[vesselname]['stopfilename']), "--status",
os.path.abspath(vesseldict[vesselname]['statusfilename']), "--cwd",
os.path.abspath(vesselname), "--servicelog", "--execinfo",
os.path.abspath(vesseldict[vesselname]['resourcefilename'])
] + argstring.split()
portable_popen.Popen(command)
starttime = nonportable.getruntime()
# wait for 10 seconds for it to start (else return an error)
while nonportable.getruntime() - starttime < 10:
newstatus, newtimestamp = statusstorage.read_status(
vesseldict[vesselname]['statusfilename'])
# Great! The timestamp was updated... The new status is the result of
# our work. Let's tell the user what happened...
if newtimestamp != oldtimestamp and newstatus != None:
break
# sleep while busy waiting...
time.sleep(.5)
else:
return "Did not start in a timely manner\nWarning"
# We need to update the status in the table because the status thread might
# not notice this before our next request... (else occasional failures on XP)
nmstatusmonitor.update_status(vesseldict, vesselname, newstatus,
newtimestamp)
return newstatus + "\nSuccess"
def started_waitable_thread(threadid): thread_starttime[threadid] = nonportable.getruntime() thread_waittime[threadid] = min(maxwaittime, thread_waittime[threadid] ** wait_exponent)
def safe_check(code):
"""Check the code to be safe."""
# NOTE: This code will not work in Windows Mobile due to the reliance on subprocess
# Get the path to safe_check.py by using the original start directory of python
path_to_safe_check = os.path.join(repy_constants.REPY_START_DIR, "safe_check.py")
# Start a safety check process, reading from the user code and outputing to a pipe we can read
proc = subprocess.Popen([sys.executable, path_to_safe_check],stdin=subprocess.PIPE, stdout=subprocess.PIPE)
# Write out the user code, close so the other end gets an EOF
proc.stdin.write(code)
proc.stdin.close()
# Wait for the process to terminate
starttime = nonportable.getruntime()
status = None
# Only wait up to EVALUTATION_TIMEOUT seconds before terminating
while status == None and (nonportable.getruntime() - starttime < EVALUTATION_TIMEOUT):
status = proc.poll()
time.sleep(0.02)
else:
# Check if the process is still running
if status == None:
# Try to terminate the external process
try:
harshexit.portablekill(proc.pid)
except:
pass
# Raise an exception
raise Exception, "Evaluation of code safety exceeded timeout threshold ("+str(nonportable.getruntime() - starttime)+" seconds)"
# Read the output and close the pipe
rawoutput = proc.stdout.read()
proc.stdout.close()
# Interim fix for #1080: Get rid of stray debugging output on Android
# of the form "dlopen libpython2.6.so" and "dlopen /system/lib/libc.so",
# yet preserve all of the other output (including empty lines).
output = ""
for line in rawoutput.split("\n"):
# Preserve empty lines
if line == "":
output += "\n"
continue
# Suppress debug messages we know can turn up
wordlist = line.split()
if wordlist[0]=="dlopen":
if wordlist[-1]=="/system/lib/libc.so":
continue
if wordlist[-1].startswith("libpython") and \
wordlist[-1].endswith(".so"):
# We expect "libpython" + version number + ".so".
# The version number should be a string convertible to float.
# If it's not, raise an exception.
try:
versionstring = (wordlist[-1].replace("libpython",
"")).replace(".so", "")
junk = float(versionstring)
except TypeError, ValueError:
raise Exception("Unexpected debug output '" + line +
"' while evaluating code safety!")
else:
output += line + "\n"