def spawn(sleep_time, advertize_rate, startup_dir, glideinDescript, frontendDescript, entries, restart_attempts, restart_interval): """ Spawn and keep track of the entry processes. Restart them if required. Advertise glidefactoryglobal classad every iteration @type sleep_time: long @param sleep_time: Delay between every iteration @type advertize_rate: long @param advertize_rate: Rate at which entries advertise their classads @type startup_dir: String @param startup_dir: Path to glideinsubmit directory @type glideinDescript: glideFactoryConfig.GlideinDescript @param glideinDescript: Factory config's glidein description object @type frontendDescript: glideFactoryConfig.FrontendDescript @param frontendDescript: Factory config's frontend description object @type entries: list @param entries: Sorted list of entry names @type restart_interval: long @param restart_interval: Allowed restart interval in second @type restart_attempts: long @param restart_attempts: Number of allowed restart attempts in the interval """ global STARTUP_DIR childs = {} # Number of glideFactoryEntry processes to spawn and directly relates to # number of concurrent condor_status processess # # NOTE: If number of entries gets too big, we may excede the shell args # limit. If that becomes an issue, move the logic to identify the # entries to serve to the group itself. # # Each process will handle multiple entries split as follows # - Sort the entries alphabetically. Already done # - Divide the list into equal chunks as possible # - Last chunk may get fewer entries entry_process_count = 1 starttime = time.time() oldkey_gracetime = int(glideinDescript.data['OldPubKeyGraceTime']) oldkey_eoltime = starttime + oldkey_gracetime childs_uptime={} factory_downtimes = glideFactoryDowntimeLib.DowntimeFile(glideinDescript.data['DowntimesFile']) logSupport.log.info("Available Entries: %s" % entries) group_size = long(math.ceil(float(len(entries))/entry_process_count)) entry_groups = entry_grouper(group_size, entries) def _set_rlimit(): resource.setrlimit(resource.RLIMIT_NOFILE, [1024, 1024]) try: for group in range(len(entry_groups)): entry_names = string.join(entry_groups[group], ':') logSupport.log.info("Starting EntryGroup %s: %s" % \ (group, entry_groups[group])) # Converted to using the subprocess module command_list = [sys.executable, os.path.join(STARTUP_DIR, "glideFactoryEntryGroup.py"), str(os.getpid()), str(sleep_time), str(advertize_rate), startup_dir, entry_names, str(group)] childs[group] = subprocess.Popen(command_list, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, preexec_fn=_set_rlimit) # Get the startup time. Used to check if the entry is crashing # periodically and needs to be restarted. childs_uptime[group] = list() childs_uptime[group].insert(0, time.time()) logSupport.log.info("EntryGroup startup times: %s" % childs_uptime) for group in childs: #childs[entry_name].tochild.close() # set it in non blocking mode # since we will run for a long time, we do not want to block for fd in (childs[group].stdout.fileno(), childs[group].stderr.fileno()): fl = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) # If RemoveOldCredFreq < 0, do not do credential cleanup. if int(glideinDescript.data['RemoveOldCredFreq']) > 0: # Convert credential removal frequency from hours to seconds remove_old_cred_freq = int(glideinDescript.data['RemoveOldCredFreq']) * 60 * 60 curr_time = time.time() update_time = curr_time + remove_old_cred_freq # Convert credential removal age from days to seconds remove_old_cred_age = int(glideinDescript.data['RemoveOldCredAge']) * 60 * 60 * 24 # Create cleaners for old credential files logSupport.log.info("Adding cleaners for old credentials") cred_base_dir = glideinDescript.data['ClientProxiesBaseDir'] for username in frontendDescript.get_all_usernames(): cred_base_user = os.path.join(cred_base_dir, "user_%s"%username) cred_user_instance_dirname = os.path.join(cred_base_user, "glidein_%s" % glideinDescript.data['GlideinName']) cred_cleaner = cleanupSupport.PrivsepDirCleanupCredentials( username, cred_user_instance_dirname, "(credential_*)", remove_old_cred_age) cleanupSupport.cred_cleaners.add_cleaner(cred_cleaner) while 1: # Record the iteration start time iteration_stime = time.time() # THIS IS FOR SECURITY # Make sure you delete the old key when its grace is up. # If a compromised key is left around and if attacker can somehow # trigger FactoryEntry process crash, we do not want the entry # to pick up the old key again when factory auto restarts it. if ( (time.time() > oldkey_eoltime) and (glideinDescript.data['OldPubKeyObj'] is not None) ): glideinDescript.data['OldPubKeyObj'] = None glideinDescript.data['OldPubKeyType'] = None try: glideinDescript.remove_old_key() logSupport.log.info("Removed the old public key after its grace time of %s seconds" % oldkey_gracetime) except: # Do not crash if delete fails. Just log it. logSupport.log.warning("Failed to remove the old public key after its grace time") # Only removing credentials in the v3+ protocol # Affects Corral Frontend which only supports the v3+ protocol. # IF freq < zero, do not do cleanup. if ( (int(glideinDescript.data['RemoveOldCredFreq']) > 0) and (curr_time >= update_time) ): logSupport.log.info("Checking credentials for cleanup") # Query queue for glideins. Don't remove proxies in use. try: in_use_creds = glideFactoryLib.getCondorQCredentialList() cleanupSupport.cred_cleaners.cleanup(in_use_creds) except: logSupport.log.exception("Unable to cleanup old credentials") update_time = curr_time + remove_old_cred_freq curr_time = time.time() logSupport.log.info("Checking for credentials %s" % entries) # Read in the frontend globals classad # Do this first so that the credentials are immediately # available when the Entries startup classads = {} try: classads = glideFactoryCredentials.get_globals_classads() except Exception: logSupport.log.error("Error occurred retrieving globals classad -- is Condor running?") for classad_key in classads: classad = classads[classad_key] try: glideFactoryCredentials.process_global(classad, glideinDescript, frontendDescript) except: logSupport.log.exception("Error occurred processing the globals classads: ") logSupport.log.info("Checking EntryGroups %s" % group) for group in childs: entry_names = string.join(entry_groups[group], ':') child = childs[group] # empty stdout and stderr try: tempOut = child.stdout.read() if len(tempOut) != 0: logSupport.log.warning("EntryGroup %s STDOUT: %s" % (group, tempOut)) except IOError: pass # ignore try: tempErr = child.stderr.read() if len(tempErr) != 0: logSupport.log.warning("EntryGroup %s STDERR: %s" % (group, tempErr)) except IOError: pass # ignore # look for exited child if child.poll(): # the child exited logSupport.log.warning("EntryGroup %s exited. Checking if it should be restarted." % (group)) tempOut = child.stdout.readlines() tempErr = child.stderr.readlines() if is_crashing_often(childs_uptime[group], restart_interval, restart_attempts): del childs[group] raise RuntimeError, "EntryGroup '%s' has been crashing too often, quit the whole factory:\n%s\n%s" % (group, tempOut, tempErr) else: # Restart the entry setting its restart time logSupport.log.warning("Restarting EntryGroup %s." % (group)) del childs[group] command_list = [sys.executable, os.path.join(STARTUP_DIR, "glideFactoryEntryGroup.py"), str(os.getpid()), str(sleep_time), str(advertize_rate), startup_dir, entry_names, str(group)] childs[group] = subprocess.Popen(command_list, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if len(childs_uptime[group]) == restart_attempts: childs_uptime[group].pop(0) childs_uptime[group].append(time.time()) childs[group].tochild.close() for fd in (childs[group].stdout.fileno(), childs[group].stderr.fileno()): fl = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) logSupport.log.warning("EntryGroup startup/restart times: %s" % childs_uptime) # Aggregate Monitoring data periodically logSupport.log.info("Aggregate monitoring data") aggregate_stats(factory_downtimes.checkDowntime()) # Advertise the global classad with the factory keys try: # KEL TODO need to add factory downtime? glideFactoryInterface.advertizeGlobal( glideinDescript.data['FactoryName'], glideinDescript.data['GlideinName'], glideFactoryLib.factoryConfig.supported_signtypes, glideinDescript.data['PubKeyObj']) except Exception, e: logSupport.log.exception("Error advertizing global classads: ") cleanupSupport.cleaners.cleanup() iteration_etime = time.time() iteration_sleep_time = sleep_time - (iteration_etime - iteration_stime) if (iteration_sleep_time < 0): iteration_sleep_time = 0 logSupport.log.info("Sleep %s secs" % iteration_sleep_time) time.sleep(iteration_sleep_time) # end while 1: finally: # cleanup at exit logSupport.log.info("Received signal...exit") try: try: clean_exit(childs) except: # if anything goes wrong, hardkill the rest for group in childs: logSupport.log.info("Hard killing EntryGroup %s" % group) try: os.kill(childs[group].pid, signal.SIGKILL) except OSError: pass # ignore dead clients finally: logSupport.log.info("Deadvertize myself") try: glideFactoryInterface.deadvertizeFactory( glideinDescript.data['FactoryName'], glideinDescript.data['GlideinName']) except: logSupport.log.exception("Factory deadvertize failed!") try: glideFactoryInterface.deadvertizeFactoryClientMonitoring( glideinDescript.data['FactoryName'], glideinDescript.data['GlideinName']) except: logSupport.log.exception("Factory Monitoring deadvertize failed!") logSupport.log.info("All EntryGroups should be terminated")
def spawn(sleep_time, advertize_rate, startup_dir, glideinDescript, frontendDescript, entries, restart_attempts, restart_interval): """ Spawn and keep track of the entry processes. Restart them if required. Advertise glidefactoryglobal classad every iteration @type sleep_time: long @param sleep_time: Delay between every iteration @type advertize_rate: long @param advertize_rate: Rate at which entries advertise their classads @type startup_dir: String @param startup_dir: Path to glideinsubmit directory @type glideinDescript: glideFactoryConfig.GlideinDescript @param glideinDescript: Factory config's glidein description object @type frontendDescript: glideFactoryConfig.FrontendDescript @param frontendDescript: Factory config's frontend description object @type entries: list @param entries: Sorted list of entry names @type restart_interval: long @param restart_interval: Allowed restart interval in second @type restart_attempts: long @param restart_attempts: Number of allowed restart attempts in the interval """ global STARTUP_DIR childs = {} # Number of glideFactoryEntry processes to spawn and directly relates to # number of concurrent condor_status processess # # NOTE: If number of entries gets too big, we may excede the shell args # limit. If that becomes an issue, move the logic to identify the # entries to serve to the group itself. # # Each process will handle multiple entries split as follows # - Sort the entries alphabetically. Already done # - Divide the list into equal chunks as possible # - Last chunk may get fewer entries entry_process_count = 1 starttime = time.time() oldkey_gracetime = int(glideinDescript.data['OldPubKeyGraceTime']) oldkey_eoltime = starttime + oldkey_gracetime childs_uptime={} factory_downtimes = glideFactoryDowntimeLib.DowntimeFile(glideinDescript.data['DowntimesFile']) logSupport.log.info("Available Entries: %s" % entries) group_size = long(math.ceil(float(len(entries))/entry_process_count)) entry_groups = entry_grouper(group_size, entries) def _set_rlimit(soft_l=None, hard_l=None): #set new hard and soft open file limits #if setting limits fails or no input parameters use inherited limits #from parent process #nb 1. it is possible to raise limits #up to [hard_l,hard_l] but once lowered they cannot be raised #nb 2. it may be better just to omit calling this function at #all from subprocess - in which case it inherits limits from #parent process lim = resource.getrlimit(resource.RLIMIT_NOFILE) if soft_l is not None or hard_l is not None: if not hard_l: hard_l = soft_l if not soft_l: soft_l=hard_l try: new_lim = [soft_l, hard_l] resource.setrlimit(resource.RLIMIT_NOFILE, new_lim) except: resource.setrlimit(resource.RLIMIT_NOFILE, lim) try: for group in range(len(entry_groups)): entry_names = string.join(entry_groups[group], ':') logSupport.log.info("Starting EntryGroup %s: %s" % \ (group, entry_groups[group])) # Converted to using the subprocess module command_list = [sys.executable, os.path.join(STARTUP_DIR, "glideFactoryEntryGroup.py"), str(os.getpid()), str(sleep_time), str(advertize_rate), startup_dir, entry_names, str(group)] childs[group] = subprocess.Popen(command_list, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, preexec_fn=_set_rlimit) # Get the startup time. Used to check if the entry is crashing # periodically and needs to be restarted. childs_uptime[group] = list() childs_uptime[group].insert(0, time.time()) logSupport.log.info("EntryGroup startup times: %s" % childs_uptime) for group in childs: # set it in non blocking mode # since we will run for a long time, we do not want to block for fd in (childs[group].stdout.fileno(), childs[group].stderr.fileno()): fl = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) # If RemoveOldCredFreq < 0, do not do credential cleanup. curr_time = 0 # To ensure curr_time is always initialized if int(glideinDescript.data['RemoveOldCredFreq']) > 0: # Convert credential removal frequency from hours to seconds remove_old_cred_freq = int(glideinDescript.data['RemoveOldCredFreq']) * 60 * 60 curr_time = time.time() update_time = curr_time + remove_old_cred_freq # Convert credential removal age from days to seconds remove_old_cred_age = int(glideinDescript.data['RemoveOldCredAge']) * 60 * 60 * 24 # Create cleaners for old credential files logSupport.log.info("Adding cleaners for old credentials") cred_base_dir = glideinDescript.data['ClientProxiesBaseDir'] for username in frontendDescript.get_all_usernames(): cred_base_user = os.path.join(cred_base_dir, "user_%s" % username) cred_user_instance_dirname = os.path.join(cred_base_user, "glidein_%s" % glideinDescript.data['GlideinName']) cred_cleaner = cleanupSupport.DirCleanupCredentials( cred_user_instance_dirname, "(credential_*)", remove_old_cred_age) cleanupSupport.cred_cleaners.add_cleaner(cred_cleaner) iteration_basetime = time.time() while True: # retrieves WebMonitoringURL from glideclient classAd iteration_timecheck = time.time() iteration_timediff = iteration_timecheck - iteration_basetime if iteration_timediff >= 3600: # every hour iteration_basetime = time.time() # reset the start time fronmonpath = os.path.join(startup_dir, "monitor", "frontendmonitorlink.txt") fronmonconstraint = '(MyType=="glideclient")' fronmonformat_list = [('WebMonitoringURL', 's'), ('FrontendName', 's')] fronmonstatus = condorMonitor.CondorStatus(subsystem_name="any") fronmondata = fronmonstatus.fetch(constraint=fronmonconstraint, format_list=fronmonformat_list) fronmon_list_names = fronmondata.keys() if fronmon_list_names is not None: urlset = set() if os.path.exists(fronmonpath): os.remove(fronmonpath) for frontend_entry in fronmon_list_names: fronmonelement = fronmondata[frontend_entry] fronmonurl = fronmonelement['WebMonitoringURL'].encode('utf-8') fronmonfrt = fronmonelement['FrontendName'].encode('utf-8') if (fronmonfrt, fronmonurl) not in urlset: urlset.add((fronmonfrt, fronmonurl)) with open(fronmonpath, 'w') as fronmonf: fronmonf.write("%s, %s" % (fronmonfrt, fronmonurl)) # Record the iteration start time iteration_stime = time.time() # THIS IS FOR SECURITY # Make sure you delete the old key when its grace is up. # If a compromised key is left around and if attacker can somehow # trigger FactoryEntry process crash, we do not want the entry # to pick up the old key again when factory auto restarts it. if time.time() > oldkey_eoltime and glideinDescript.data['OldPubKeyObj'] is not None: glideinDescript.data['OldPubKeyObj'] = None glideinDescript.data['OldPubKeyType'] = None try: glideinDescript.remove_old_key() logSupport.log.info("Removed the old public key after its grace time of %s seconds" % oldkey_gracetime) except: # Do not crash if delete fails. Just log it. logSupport.log.warning("Failed to remove the old public key after its grace time") # Only removing credentials in the v3+ protocol # Affects Corral Frontend which only supports the v3+ protocol. # IF freq < zero, do not do cleanup. if int(glideinDescript.data['RemoveOldCredFreq']) > 0 and curr_time >= update_time: logSupport.log.info("Checking credentials for cleanup") # Query queue for glideins. Don't remove proxies in use. try: in_use_creds = glideFactoryLib.getCondorQCredentialList() cleanupSupport.cred_cleaners.cleanup(in_use_creds) except: logSupport.log.exception("Unable to cleanup old credentials") update_time = curr_time + remove_old_cred_freq curr_time = time.time() logSupport.log.info("Checking for credentials %s" % entries) # Read in the frontend globals classad # Do this first so that the credentials are immediately # available when the Entries startup classads = {} try: classads = glideFactoryCredentials.get_globals_classads() except Exception: logSupport.log.error("Error occurred retrieving globals classad -- is Condor running?") for classad_key in classads: classad = classads[classad_key] try: glideFactoryCredentials.process_global(classad, glideinDescript, frontendDescript) except: logSupport.log.exception("Error occurred processing the globals classads: ") logSupport.log.info("Checking EntryGroups %s" % childs.keys()) for group in childs: entry_names = string.join(entry_groups[group], ':') child = childs[group] # empty stdout and stderr try: tempOut = child.stdout.read() if len(tempOut) != 0: logSupport.log.warning("EntryGroup %s STDOUT: %s" % (group, tempOut)) except IOError: pass # ignore try: tempErr = child.stderr.read() if len(tempErr) != 0: logSupport.log.warning("EntryGroup %s STDERR: %s" % (group, tempErr)) except IOError: pass # ignore # look for exited child if child.poll(): # the child exited logSupport.log.warning("EntryGroup %s exited. Checking if it should be restarted." % (group)) tempOut = child.stdout.readlines() tempErr = child.stderr.readlines() if is_crashing_often(childs_uptime[group], restart_interval, restart_attempts): del childs[group] raise RuntimeError("EntryGroup '%s' has been crashing too often, quit the whole factory:\n%s\n%s" % (group, tempOut, tempErr)) else: # Restart the entry setting its restart time logSupport.log.warning("Restarting EntryGroup %s." % (group)) del childs[group] command_list = [sys.executable, os.path.join(STARTUP_DIR, "glideFactoryEntryGroup.py"), str(os.getpid()), str(sleep_time), str(advertize_rate), startup_dir, entry_names, str(group)] childs[group] = subprocess.Popen(command_list, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True, preexec_fn=_set_rlimit) if len(childs_uptime[group]) == restart_attempts: childs_uptime[group].pop(0) childs_uptime[group].append(time.time()) for fd in (childs[group].stdout.fileno(), childs[group].stderr.fileno()): fl = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) logSupport.log.warning("EntryGroup startup/restart times: %s" % (childs_uptime,)) # Aggregate Monitoring data periodically logSupport.log.info("Aggregate monitoring data") stats = aggregate_stats(factory_downtimes.checkDowntime()) save_stats(stats, os.path.join(startup_dir, glideFactoryConfig.factoryConfig.aggregated_stats_file)) # Aggregate job data periodically if glideinDescript.data.get('AdvertisePilotAccounting', False) in ['True', '1']: # data attributes are strings logSupport.log.info("Starting updating job classads") update_classads() logSupport.log.info("Finishing updating job classads") # Advertise the global classad with the factory keys and Factory statistics try: # KEL TODO need to add factory downtime? glideFactoryInterface.advertizeGlobal( glideinDescript.data['FactoryName'], glideinDescript.data['GlideinName'], glideFactoryLib.factoryConfig.supported_signtypes, glideinDescript.data['PubKeyObj'] ) except Exception as e: logSupport.log.exception("Error advertising global classads: %s" % e) cleanupSupport.cleaners.cleanup() iteration_etime = time.time() iteration_sleep_time = sleep_time - (iteration_etime - iteration_stime) if iteration_sleep_time < 0: iteration_sleep_time = 0 logSupport.log.info("Sleep %s secs" % iteration_sleep_time) time.sleep(iteration_sleep_time) # end while 1: finally: # cleanup at exit logSupport.log.info("Received signal...exit") try: try: clean_exit(childs) except: # if anything goes wrong, hardkill the rest for group in childs: logSupport.log.info("Hard killing EntryGroup %s" % group) try: os.kill(childs[group].pid, signal.SIGKILL) except OSError: pass # ignore dead clients finally: logSupport.log.info("Deadvertize myself") try: glideFactoryInterface.deadvertizeFactory( glideinDescript.data['FactoryName'], glideinDescript.data['GlideinName']) except: logSupport.log.exception("Factory deadvertize failed!") try: glideFactoryInterface.deadvertizeFactoryClientMonitoring( glideinDescript.data['FactoryName'], glideinDescript.data['GlideinName']) except: logSupport.log.exception("Factory Monitoring deadvertize failed!") logSupport.log.info("All EntryGroups should be terminated")
def test_get_condor_q_credential_list(self): glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger() glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock() crdl = getCondorQCredentialList(self.cnf) self.assertEqual([], crdl)
def test_get_condor_q_credential_list(self): glideinwms.factory.glideFactoryLib.logSupport.log = FakeLogger() glideinwms.factory.glideFactoryLib.condorMonitor = mock.Mock() crdl = getCondorQCredentialList(self.cnf) self.assertEqual([], crdl)