def spawn(sleep_time, advertize_rate, startup_dir, glideinDescript,
frontendDescript, entries, restart_attempts, restart_interval):
"""
Spawn and keep track of the entry processes. Restart them if required.
Advertise glidefactoryglobal classad every iteration
@type sleep_time: long
@param sleep_time: Delay between every iteration
@type advertize_rate: long
@param advertize_rate: Rate at which entries advertise their classads
@type startup_dir: String
@param startup_dir: Path to glideinsubmit directory
@type glideinDescript: glideFactoryConfig.GlideinDescript
@param glideinDescript: Factory config's glidein description object
@type frontendDescript: glideFactoryConfig.FrontendDescript
@param frontendDescript: Factory config's frontend description object
@type entries: list
@param entries: Sorted list of entry names
@type restart_interval: long
@param restart_interval: Allowed restart interval in second
@type restart_attempts: long
@param restart_attempts: Number of allowed restart attempts in the interval
"""
global STARTUP_DIR
childs = {}
# Number of glideFactoryEntry processes to spawn and directly relates to
# number of concurrent condor_status processess
#
# NOTE: If number of entries gets too big, we may excede the shell args
# limit. If that becomes an issue, move the logic to identify the
# entries to serve to the group itself.
#
# Each process will handle multiple entries split as follows
# - Sort the entries alphabetically. Already done
# - Divide the list into equal chunks as possible
# - Last chunk may get fewer entries
entry_process_count = 1
starttime = time.time()
oldkey_gracetime = int(glideinDescript.data['OldPubKeyGraceTime'])
oldkey_eoltime = starttime + oldkey_gracetime
childs_uptime={}
factory_downtimes = glideFactoryDowntimeLib.DowntimeFile(glideinDescript.data['DowntimesFile'])
logSupport.log.info("Available Entries: %s" % entries)
group_size = long(math.ceil(float(len(entries))/entry_process_count))
entry_groups = entry_grouper(group_size, entries)
def _set_rlimit():
resource.setrlimit(resource.RLIMIT_NOFILE, [1024, 1024])
try:
for group in range(len(entry_groups)):
entry_names = string.join(entry_groups[group], ':')
logSupport.log.info("Starting EntryGroup %s: %s" % \
(group, entry_groups[group]))
# Converted to using the subprocess module
command_list = [sys.executable,
os.path.join(STARTUP_DIR,
"glideFactoryEntryGroup.py"),
str(os.getpid()),
str(sleep_time),
str(advertize_rate),
startup_dir,
entry_names,
str(group)]
childs[group] = subprocess.Popen(command_list, shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True,
preexec_fn=_set_rlimit)
# Get the startup time. Used to check if the entry is crashing
# periodically and needs to be restarted.
childs_uptime[group] = list()
childs_uptime[group].insert(0, time.time())
logSupport.log.info("EntryGroup startup times: %s" % childs_uptime)
for group in childs:
#childs[entry_name].tochild.close()
# set it in non blocking mode
# since we will run for a long time, we do not want to block
for fd in (childs[group].stdout.fileno(),
childs[group].stderr.fileno()):
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
# If RemoveOldCredFreq < 0, do not do credential cleanup.
if int(glideinDescript.data['RemoveOldCredFreq']) > 0:
# Convert credential removal frequency from hours to seconds
remove_old_cred_freq = int(glideinDescript.data['RemoveOldCredFreq']) * 60 * 60
curr_time = time.time()
update_time = curr_time + remove_old_cred_freq
# Convert credential removal age from days to seconds
remove_old_cred_age = int(glideinDescript.data['RemoveOldCredAge']) * 60 * 60 * 24
# Create cleaners for old credential files
logSupport.log.info("Adding cleaners for old credentials")
cred_base_dir = glideinDescript.data['ClientProxiesBaseDir']
for username in frontendDescript.get_all_usernames():
cred_base_user = os.path.join(cred_base_dir, "user_%s"%username)
cred_user_instance_dirname = os.path.join(cred_base_user, "glidein_%s" % glideinDescript.data['GlideinName'])
cred_cleaner = cleanupSupport.PrivsepDirCleanupCredentials(
username, cred_user_instance_dirname,
"(credential_*)", remove_old_cred_age)
cleanupSupport.cred_cleaners.add_cleaner(cred_cleaner)
while 1:
# Record the iteration start time
iteration_stime = time.time()
# THIS IS FOR SECURITY
# Make sure you delete the old key when its grace is up.
# If a compromised key is left around and if attacker can somehow
# trigger FactoryEntry process crash, we do not want the entry
# to pick up the old key again when factory auto restarts it.
if ( (time.time() > oldkey_eoltime) and
(glideinDescript.data['OldPubKeyObj'] is not None) ):
glideinDescript.data['OldPubKeyObj'] = None
glideinDescript.data['OldPubKeyType'] = None
try:
glideinDescript.remove_old_key()
logSupport.log.info("Removed the old public key after its grace time of %s seconds" % oldkey_gracetime)
except:
# Do not crash if delete fails. Just log it.
logSupport.log.warning("Failed to remove the old public key after its grace time")
# Only removing credentials in the v3+ protocol
# Affects Corral Frontend which only supports the v3+ protocol.
# IF freq < zero, do not do cleanup.
if ( (int(glideinDescript.data['RemoveOldCredFreq']) > 0) and
(curr_time >= update_time) ):
logSupport.log.info("Checking credentials for cleanup")
# Query queue for glideins. Don't remove proxies in use.
try:
in_use_creds = glideFactoryLib.getCondorQCredentialList()
cleanupSupport.cred_cleaners.cleanup(in_use_creds)
except:
logSupport.log.exception("Unable to cleanup old credentials")
update_time = curr_time + remove_old_cred_freq
curr_time = time.time()
logSupport.log.info("Checking for credentials %s" % entries)
# Read in the frontend globals classad
# Do this first so that the credentials are immediately
# available when the Entries startup
classads = {}
try:
classads = glideFactoryCredentials.get_globals_classads()
except Exception:
logSupport.log.error("Error occurred retrieving globals classad -- is Condor running?")
for classad_key in classads:
classad = classads[classad_key]
try:
glideFactoryCredentials.process_global(classad,
glideinDescript,
frontendDescript)
except:
logSupport.log.exception("Error occurred processing the globals classads: ")
logSupport.log.info("Checking EntryGroups %s" % group)
for group in childs:
entry_names = string.join(entry_groups[group], ':')
child = childs[group]
# empty stdout and stderr
try:
tempOut = child.stdout.read()
if len(tempOut) != 0:
logSupport.log.warning("EntryGroup %s STDOUT: %s" % (group, tempOut))
except IOError:
pass # ignore
try:
tempErr = child.stderr.read()
if len(tempErr) != 0:
logSupport.log.warning("EntryGroup %s STDERR: %s" % (group, tempErr))
except IOError:
pass # ignore
# look for exited child
if child.poll():
# the child exited
logSupport.log.warning("EntryGroup %s exited. Checking if it should be restarted." % (group))
tempOut = child.stdout.readlines()
tempErr = child.stderr.readlines()
if is_crashing_often(childs_uptime[group],
restart_interval, restart_attempts):
del childs[group]
raise RuntimeError, "EntryGroup '%s' has been crashing too often, quit the whole factory:\n%s\n%s" % (group, tempOut, tempErr)
else:
# Restart the entry setting its restart time
logSupport.log.warning("Restarting EntryGroup %s." % (group))
del childs[group]
command_list = [sys.executable,
os.path.join(STARTUP_DIR,
"glideFactoryEntryGroup.py"),
str(os.getpid()),
str(sleep_time),
str(advertize_rate),
startup_dir,
entry_names,
str(group)]
childs[group] = subprocess.Popen(command_list,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if len(childs_uptime[group]) == restart_attempts:
childs_uptime[group].pop(0)
childs_uptime[group].append(time.time())
childs[group].tochild.close()
for fd in (childs[group].stdout.fileno(),
childs[group].stderr.fileno()):
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
logSupport.log.warning("EntryGroup startup/restart times: %s" % childs_uptime)
# Aggregate Monitoring data periodically
logSupport.log.info("Aggregate monitoring data")
aggregate_stats(factory_downtimes.checkDowntime())
# Advertise the global classad with the factory keys
try:
# KEL TODO need to add factory downtime?
glideFactoryInterface.advertizeGlobal(
glideinDescript.data['FactoryName'],
glideinDescript.data['GlideinName'],
glideFactoryLib.factoryConfig.supported_signtypes,
glideinDescript.data['PubKeyObj'])
except Exception, e:
logSupport.log.exception("Error advertizing global classads: ")
cleanupSupport.cleaners.cleanup()
iteration_etime = time.time()
iteration_sleep_time = sleep_time - (iteration_etime - iteration_stime)
if (iteration_sleep_time < 0):
iteration_sleep_time = 0
logSupport.log.info("Sleep %s secs" % iteration_sleep_time)
time.sleep(iteration_sleep_time)
# end while 1:
finally:
# cleanup at exit
logSupport.log.info("Received signal...exit")
try:
try:
clean_exit(childs)
except:
# if anything goes wrong, hardkill the rest
for group in childs:
logSupport.log.info("Hard killing EntryGroup %s" % group)
try:
os.kill(childs[group].pid, signal.SIGKILL)
except OSError:
pass # ignore dead clients
finally:
logSupport.log.info("Deadvertize myself")
try:
glideFactoryInterface.deadvertizeFactory(
glideinDescript.data['FactoryName'],
glideinDescript.data['GlideinName'])
except:
logSupport.log.exception("Factory deadvertize failed!")
try:
glideFactoryInterface.deadvertizeFactoryClientMonitoring(
glideinDescript.data['FactoryName'],
glideinDescript.data['GlideinName'])
except:
logSupport.log.exception("Factory Monitoring deadvertize failed!")
logSupport.log.info("All EntryGroups should be terminated")
def spawn(sleep_time, advertize_rate, startup_dir, glideinDescript,
frontendDescript, entries, restart_attempts, restart_interval):
"""
Spawn and keep track of the entry processes. Restart them if required.
Advertise glidefactoryglobal classad every iteration
@type sleep_time: long
@param sleep_time: Delay between every iteration
@type advertize_rate: long
@param advertize_rate: Rate at which entries advertise their classads
@type startup_dir: String
@param startup_dir: Path to glideinsubmit directory
@type glideinDescript: glideFactoryConfig.GlideinDescript
@param glideinDescript: Factory config's glidein description object
@type frontendDescript: glideFactoryConfig.FrontendDescript
@param frontendDescript: Factory config's frontend description object
@type entries: list
@param entries: Sorted list of entry names
@type restart_interval: long
@param restart_interval: Allowed restart interval in second
@type restart_attempts: long
@param restart_attempts: Number of allowed restart attempts in the interval
"""
global STARTUP_DIR
childs = {}
# Number of glideFactoryEntry processes to spawn and directly relates to
# number of concurrent condor_status processess
#
# NOTE: If number of entries gets too big, we may excede the shell args
# limit. If that becomes an issue, move the logic to identify the
# entries to serve to the group itself.
#
# Each process will handle multiple entries split as follows
# - Sort the entries alphabetically. Already done
# - Divide the list into equal chunks as possible
# - Last chunk may get fewer entries
entry_process_count = 1
starttime = time.time()
oldkey_gracetime = int(glideinDescript.data['OldPubKeyGraceTime'])
oldkey_eoltime = starttime + oldkey_gracetime
childs_uptime={}
factory_downtimes = glideFactoryDowntimeLib.DowntimeFile(glideinDescript.data['DowntimesFile'])
logSupport.log.info("Available Entries: %s" % entries)
group_size = long(math.ceil(float(len(entries))/entry_process_count))
entry_groups = entry_grouper(group_size, entries)
def _set_rlimit(soft_l=None, hard_l=None):
#set new hard and soft open file limits
#if setting limits fails or no input parameters use inherited limits
#from parent process
#nb 1. it is possible to raise limits
#up to [hard_l,hard_l] but once lowered they cannot be raised
#nb 2. it may be better just to omit calling this function at
#all from subprocess - in which case it inherits limits from
#parent process
lim = resource.getrlimit(resource.RLIMIT_NOFILE)
if soft_l is not None or hard_l is not None:
if not hard_l:
hard_l = soft_l
if not soft_l:
soft_l=hard_l
try:
new_lim = [soft_l, hard_l]
resource.setrlimit(resource.RLIMIT_NOFILE, new_lim)
except:
resource.setrlimit(resource.RLIMIT_NOFILE, lim)
try:
for group in range(len(entry_groups)):
entry_names = string.join(entry_groups[group], ':')
logSupport.log.info("Starting EntryGroup %s: %s" % \
(group, entry_groups[group]))
# Converted to using the subprocess module
command_list = [sys.executable,
os.path.join(STARTUP_DIR,
"glideFactoryEntryGroup.py"),
str(os.getpid()),
str(sleep_time),
str(advertize_rate),
startup_dir,
entry_names,
str(group)]
childs[group] = subprocess.Popen(command_list, shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True,
preexec_fn=_set_rlimit)
# Get the startup time. Used to check if the entry is crashing
# periodically and needs to be restarted.
childs_uptime[group] = list()
childs_uptime[group].insert(0, time.time())
logSupport.log.info("EntryGroup startup times: %s" % childs_uptime)
for group in childs:
# set it in non blocking mode
# since we will run for a long time, we do not want to block
for fd in (childs[group].stdout.fileno(),
childs[group].stderr.fileno()):
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
# If RemoveOldCredFreq < 0, do not do credential cleanup.
curr_time = 0 # To ensure curr_time is always initialized
if int(glideinDescript.data['RemoveOldCredFreq']) > 0:
# Convert credential removal frequency from hours to seconds
remove_old_cred_freq = int(glideinDescript.data['RemoveOldCredFreq']) * 60 * 60
curr_time = time.time()
update_time = curr_time + remove_old_cred_freq
# Convert credential removal age from days to seconds
remove_old_cred_age = int(glideinDescript.data['RemoveOldCredAge']) * 60 * 60 * 24
# Create cleaners for old credential files
logSupport.log.info("Adding cleaners for old credentials")
cred_base_dir = glideinDescript.data['ClientProxiesBaseDir']
for username in frontendDescript.get_all_usernames():
cred_base_user = os.path.join(cred_base_dir, "user_%s" % username)
cred_user_instance_dirname = os.path.join(cred_base_user, "glidein_%s" % glideinDescript.data['GlideinName'])
cred_cleaner = cleanupSupport.DirCleanupCredentials(
cred_user_instance_dirname,
"(credential_*)", remove_old_cred_age)
cleanupSupport.cred_cleaners.add_cleaner(cred_cleaner)
iteration_basetime = time.time()
while True:
# retrieves WebMonitoringURL from glideclient classAd
iteration_timecheck = time.time()
iteration_timediff = iteration_timecheck - iteration_basetime
if iteration_timediff >= 3600: # every hour
iteration_basetime = time.time() # reset the start time
fronmonpath = os.path.join(startup_dir, "monitor", "frontendmonitorlink.txt")
fronmonconstraint = '(MyType=="glideclient")'
fronmonformat_list = [('WebMonitoringURL', 's'), ('FrontendName', 's')]
fronmonstatus = condorMonitor.CondorStatus(subsystem_name="any")
fronmondata = fronmonstatus.fetch(constraint=fronmonconstraint, format_list=fronmonformat_list)
fronmon_list_names = fronmondata.keys()
if fronmon_list_names is not None:
urlset = set()
if os.path.exists(fronmonpath):
os.remove(fronmonpath)
for frontend_entry in fronmon_list_names:
fronmonelement = fronmondata[frontend_entry]
fronmonurl = fronmonelement['WebMonitoringURL'].encode('utf-8')
fronmonfrt = fronmonelement['FrontendName'].encode('utf-8')
if (fronmonfrt, fronmonurl) not in urlset:
urlset.add((fronmonfrt, fronmonurl))
with open(fronmonpath, 'w') as fronmonf:
fronmonf.write("%s, %s" % (fronmonfrt, fronmonurl))
# Record the iteration start time
iteration_stime = time.time()
# THIS IS FOR SECURITY
# Make sure you delete the old key when its grace is up.
# If a compromised key is left around and if attacker can somehow
# trigger FactoryEntry process crash, we do not want the entry
# to pick up the old key again when factory auto restarts it.
if time.time() > oldkey_eoltime and glideinDescript.data['OldPubKeyObj'] is not None:
glideinDescript.data['OldPubKeyObj'] = None
glideinDescript.data['OldPubKeyType'] = None
try:
glideinDescript.remove_old_key()
logSupport.log.info("Removed the old public key after its grace time of %s seconds" % oldkey_gracetime)
except:
# Do not crash if delete fails. Just log it.
logSupport.log.warning("Failed to remove the old public key after its grace time")
# Only removing credentials in the v3+ protocol
# Affects Corral Frontend which only supports the v3+ protocol.
# IF freq < zero, do not do cleanup.
if int(glideinDescript.data['RemoveOldCredFreq']) > 0 and curr_time >= update_time:
logSupport.log.info("Checking credentials for cleanup")
# Query queue for glideins. Don't remove proxies in use.
try:
in_use_creds = glideFactoryLib.getCondorQCredentialList()
cleanupSupport.cred_cleaners.cleanup(in_use_creds)
except:
logSupport.log.exception("Unable to cleanup old credentials")
update_time = curr_time + remove_old_cred_freq
curr_time = time.time()
logSupport.log.info("Checking for credentials %s" % entries)
# Read in the frontend globals classad
# Do this first so that the credentials are immediately
# available when the Entries startup
classads = {}
try:
classads = glideFactoryCredentials.get_globals_classads()
except Exception:
logSupport.log.error("Error occurred retrieving globals classad -- is Condor running?")
for classad_key in classads:
classad = classads[classad_key]
try:
glideFactoryCredentials.process_global(classad,
glideinDescript,
frontendDescript)
except:
logSupport.log.exception("Error occurred processing the globals classads: ")
logSupport.log.info("Checking EntryGroups %s" % childs.keys())
for group in childs:
entry_names = string.join(entry_groups[group], ':')
child = childs[group]
# empty stdout and stderr
try:
tempOut = child.stdout.read()
if len(tempOut) != 0:
logSupport.log.warning("EntryGroup %s STDOUT: %s" % (group, tempOut))
except IOError:
pass # ignore
try:
tempErr = child.stderr.read()
if len(tempErr) != 0:
logSupport.log.warning("EntryGroup %s STDERR: %s" % (group, tempErr))
except IOError:
pass # ignore
# look for exited child
if child.poll():
# the child exited
logSupport.log.warning("EntryGroup %s exited. Checking if it should be restarted." % (group))
tempOut = child.stdout.readlines()
tempErr = child.stderr.readlines()
if is_crashing_often(childs_uptime[group],
restart_interval, restart_attempts):
del childs[group]
raise RuntimeError("EntryGroup '%s' has been crashing too often, quit the whole factory:\n%s\n%s" % (group, tempOut, tempErr))
else:
# Restart the entry setting its restart time
logSupport.log.warning("Restarting EntryGroup %s." % (group))
del childs[group]
command_list = [sys.executable,
os.path.join(STARTUP_DIR,
"glideFactoryEntryGroup.py"),
str(os.getpid()),
str(sleep_time),
str(advertize_rate),
startup_dir,
entry_names,
str(group)]
childs[group] = subprocess.Popen(command_list,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True,
preexec_fn=_set_rlimit)
if len(childs_uptime[group]) == restart_attempts:
childs_uptime[group].pop(0)
childs_uptime[group].append(time.time())
for fd in (childs[group].stdout.fileno(),
childs[group].stderr.fileno()):
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
logSupport.log.warning("EntryGroup startup/restart times: %s" % (childs_uptime,))
# Aggregate Monitoring data periodically
logSupport.log.info("Aggregate monitoring data")
stats = aggregate_stats(factory_downtimes.checkDowntime())
save_stats(stats, os.path.join(startup_dir, glideFactoryConfig.factoryConfig.aggregated_stats_file))
# Aggregate job data periodically
if glideinDescript.data.get('AdvertisePilotAccounting', False) in ['True', '1']: # data attributes are strings
logSupport.log.info("Starting updating job classads")
update_classads()
logSupport.log.info("Finishing updating job classads")
# Advertise the global classad with the factory keys and Factory statistics
try:
# KEL TODO need to add factory downtime?
glideFactoryInterface.advertizeGlobal(
glideinDescript.data['FactoryName'],
glideinDescript.data['GlideinName'],
glideFactoryLib.factoryConfig.supported_signtypes,
glideinDescript.data['PubKeyObj']
)
except Exception as e:
logSupport.log.exception("Error advertising global classads: %s" % e)
cleanupSupport.cleaners.cleanup()
iteration_etime = time.time()
iteration_sleep_time = sleep_time - (iteration_etime - iteration_stime)
if iteration_sleep_time < 0:
iteration_sleep_time = 0
logSupport.log.info("Sleep %s secs" % iteration_sleep_time)
time.sleep(iteration_sleep_time)
# end while 1:
finally:
# cleanup at exit
logSupport.log.info("Received signal...exit")
try:
try:
clean_exit(childs)
except:
# if anything goes wrong, hardkill the rest
for group in childs:
logSupport.log.info("Hard killing EntryGroup %s" % group)
try:
os.kill(childs[group].pid, signal.SIGKILL)
except OSError:
pass # ignore dead clients
finally:
logSupport.log.info("Deadvertize myself")
try:
glideFactoryInterface.deadvertizeFactory(
glideinDescript.data['FactoryName'],
glideinDescript.data['GlideinName'])
except:
logSupport.log.exception("Factory deadvertize failed!")
try:
glideFactoryInterface.deadvertizeFactoryClientMonitoring(
glideinDescript.data['FactoryName'],
glideinDescript.data['GlideinName'])
except:
logSupport.log.exception("Factory Monitoring deadvertize failed!")
logSupport.log.info("All EntryGroups should be terminated")
def test_get_condor_q_credential_list(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
crdl = getCondorQCredentialList(self.cnf)
self.assertEqual([], crdl)
def test_get_condor_q_credential_list(self):
glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger()
glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock()
crdl = getCondorQCredentialList(self.cnf)
self.assertEqual([], crdl)