def _setup_vessel(vesselname, examplevessel, resourcedict, call_list):
if vesselname in vesseldict:
raise Exception, "Internal Error, setting up vessel '" + vesselname + "' already in vesseldict"
# write the new resource file
resourcemanipulation.write_resourcedict_to_file(resourcedict,
'resource.' + vesselname,
call_list)
# Set the invariants up...
item = {}
item['stopfilename'] = vesselname + ".stop"
item['logfilename'] = vesselname + ".log"
item['resourcefilename'] = "resource." + vesselname
item['status'] = 'Fresh'
item['statusfilename'] = vesselname + ".status"
# first the easy stuff... Set up the vesseldict dictionary
if examplevessel == None:
item['userkeys'] = []
item['ownerkey'] = {}
item['oldmetadata'] = None
item['advertise'] = True
item['ownerinformation'] = ''
else:
if examplevessel not in vesseldict:
raise Exception("Internal Error, examplevessel '" + examplevessel +
"' not in vesseldict")
item['userkeys'] = vesseldict[examplevessel]['userkeys']
item['ownerkey'] = vesseldict[examplevessel]['ownerkey']
item['oldmetadata'] = vesseldict[examplevessel]['oldmetadata']
item['advertise'] = vesseldict[examplevessel]['advertise']
item['ownerinformation'] = vesseldict[examplevessel][
'ownerinformation']
# create the directory on the file system
os.mkdir(vesselname)
# now we're ready to add the entry to the table (so other threads can use it)
vesseldict[vesselname] = item
def _setup_vessel(vesselname, examplevessel, resourcedict, call_list):
if vesselname in vesseldict:
raise Exception, "Internal Error, setting up vessel '"+vesselname+"' already in vesseldict"
# write the new resource file
resourcemanipulation.write_resourcedict_to_file(resourcedict, 'resource.'+vesselname, call_list)
# Set the invariants up...
item = {}
item['stopfilename'] = vesselname+".stop"
item['logfilename'] = vesselname+".log"
item['resourcefilename'] = "resource."+vesselname
item['status'] = 'Fresh'
item['statusfilename'] = vesselname+".status"
# first the easy stuff... Set up the vesseldict dictionary
if examplevessel == None:
item['userkeys'] = []
item['ownerkey'] = {}
item['oldmetadata'] = None
item['advertise'] = True
item['ownerinformation'] = ''
else:
if examplevessel not in vesseldict:
raise Exception("Internal Error, examplevessel '"+examplevessel+"' not in vesseldict")
item['userkeys'] = vesseldict[examplevessel]['userkeys']
item['ownerkey'] = vesseldict[examplevessel]['ownerkey']
item['oldmetadata'] = vesseldict[examplevessel]['oldmetadata']
item['advertise'] = vesseldict[examplevessel]['advertise']
item['ownerinformation'] = vesseldict[examplevessel]['ownerinformation']
# create the directory on the file system
os.mkdir(vesselname)
# now we're ready to add the entry to the table (so other threads can use it)
vesseldict[vesselname] = item
def handle_threading_error():
"""
<Purpose>
Handles a repy node failing with ThreadErr. If repy is allowed to use
more than 10% of the current threads, reduce the global thread count by 50%
and stop all existing vessels
<Arguments>
None
<Exceptions>
None
<Side Effects>
May re-write all resource files and stop all vessels
<Returns>
None
"""
# Make a log of this
servicelogger.log("[ERROR]:A Repy vessel has exited with ThreadErr status. Checking to determine next step")
# Get all the names of the vessels
vesselnamelist = nmAPI.vesseldict.keys()
# read in all of the resource files so that we can look at and possibly
# manipulate them.
resourcedicts = {}
for vesselname in vesselnamelist:
resourcedicts[vesselname] = resourcemanipulation.read_resourcedict_from_file('resource.'+vesselname)
# Get the number of threads Repy has allocated
allowedthreadcount = 0
for vesselname in vesselnamelist:
allowedthreadcount = allowedthreadcount + resourcedicts[vesselname]['events']
# Get the total number os system threads currently used
totalusedthreads = nonportable.os_api.get_system_thread_count()
# Log this information
servicelogger.log("[WARNING]:System Threads: "+str(totalusedthreads)+" Repy Allocated Threads: "+str(allowedthreadcount))
# Get the NM configuration
configuration = persist.restore_object("nodeman.cfg")
# Check if there is a threshold configuration,
# otherwise add the default configuration
if NOOP_CONFIG_KEY in configuration:
threshold = configuration[NOOP_CONFIG_KEY]
else:
threshold = DEFAULT_NOOP_THRESHOLD
configuration[NOOP_CONFIG_KEY] = threshold
persist.commit_object(configuration, "nodeman.cfg")
# Check if we are below the threshold, if so
# then just return, this is a noop
if allowedthreadcount < totalusedthreads * threshold:
return
servicelogger.log("[ERROR]:Reducing number of system threads!")
#### We are above the threshold! Let's cut everything by 1/2
# First, update the resource files
for vesselname in vesselnamelist:
# cut the events by 1/2
resourcedicts[vesselname]['events'] = resourcedicts[vesselname]['events'] / 2
# write out the new resource files...
resourcemanipulation.write_resourcedict_to_file(resourcedicts[vesselname], 'resource.'+vesselname)
# Create the stop tuple, exit code 57 with an error message
stoptuple = (57, "Fatal system-wide threading error! Stopping all vessels.")
# Stop each vessel
for vesselname in vesselnamelist:
try:
# Stop each vessel, using our stoptuple
nmAPI.stopvessel(vesselname,stoptuple)
except Exception, exp:
# Forge on, regardless of errors
servicelogger.log("[ERROR]:Failed to reset vessel (Handling ThreadErr). Exception: "+str(exp))
servicelogger.log_last_exception()
def handle_threading_error():
"""
<Purpose>
Handles a repy node failing with ThreadErr. If repy is allowed to use
more than 10% of the current threads, reduce the global thread count by 50%
and stop all existing vessels
<Arguments>
None
<Exceptions>
None
<Side Effects>
May re-write all resource files and stop all vessels
<Returns>
None
"""
# Make a log of this
servicelogger.log(
"[ERROR]:A Repy vessel has exited with ThreadErr status. Checking to determine next step"
)
# Get all the names of the vessels
vesselnamelist = nmAPI.vesseldict.keys()
# read in all of the resource files so that we can look at and possibly
# manipulate them.
resourcedicts = {}
for vesselname in vesselnamelist:
resourcedicts[
vesselname] = resourcemanipulation.read_resourcedict_from_file(
'resource.' + vesselname)
# Get the number of threads Repy has allocated
allowedthreadcount = 0
for vesselname in vesselnamelist:
allowedthreadcount = allowedthreadcount + resourcedicts[vesselname][
'events']
# Get the total number os system threads currently used
totalusedthreads = nonportable.os_api.get_system_thread_count()
# Log this information
servicelogger.log("[WARNING]:System Threads: " + str(totalusedthreads) +
" Repy Allocated Threads: " + str(allowedthreadcount))
# Get the NM configuration
configuration = persist.restore_object("nodeman.cfg")
# Check if there is a threshold configuration,
# otherwise add the default configuration
if NOOP_CONFIG_KEY in configuration:
threshold = configuration[NOOP_CONFIG_KEY]
else:
threshold = DEFAULT_NOOP_THRESHOLD
configuration[NOOP_CONFIG_KEY] = threshold
persist.commit_object(configuration, "nodeman.cfg")
# Check if we are below the threshold, if so
# then just return, this is a noop
if allowedthreadcount < totalusedthreads * threshold:
return
servicelogger.log("[ERROR]:Reducing number of system threads!")
#### We are above the threshold! Let's cut everything by 1/2
# First, update the resource files
for vesselname in vesselnamelist:
# cut the events by 1/2
resourcedicts[vesselname][
'events'] = resourcedicts[vesselname]['events'] / 2
# write out the new resource files...
resourcemanipulation.write_resourcedict_to_file(
resourcedicts[vesselname], 'resource.' + vesselname)
# Create the stop tuple, exit code 57 with an error message
stoptuple = (57,
"Fatal system-wide threading error! Stopping all vessels.")
# Stop each vessel
for vesselname in vesselnamelist:
try:
# Stop each vessel, using our stoptuple
nmAPI.stopvessel(vesselname, stoptuple)
except Exception, exp:
# Forge on, regardless of errors
servicelogger.log(
"[ERROR]:Failed to reset vessel (Handling ThreadErr). Exception: "
+ str(exp))
servicelogger.log_last_exception()
