def bootstrapSGE(cp): """ If it exists, source $SGE_ROOT/$SGE_CELL/common/settings.sh """ sge_root = cp_get(cp, "sge", "sge_root", "") if not sge_root: log.warning("Could not locate sge_root in config file! Not " \ "bootstrapping SGE environment.") return sge_cell = cp_get(cp, "sge", "sge_cell", "") if not sge_cell: log.warning("Could not locate sge_cell in config file! Not " \ "bootstrapping SGE environment.") return settings = os.path.join(sge_root, sge_cell, "common/settings.sh") if not os.path.exists(settings): log.warning("Could not find the SGE settings file; looked in %s" % \ settings) return cmd = "/bin/sh -c 'source %s; /usr/bin/env'" % settings fd = os.popen(cmd) results = fd.read() if fd.close(): log.warning("Unable to source the SGE settings file; tried %s." % \ settings) for line in results.splitlines(): line = line.strip() info = line.split('=', 2) if len(info) != 2: continue os.environ[info[0]] = info[1]
def main(): """ Main wrapper for the Condor batch system GIP information. """ try: cp = config() condor_path = cp_get(cp, "condor", "condor_path", None) condor_location = cp_get(cp, "condor", "condor_location", None) condor_config = cp_get(cp, "condor", "condor_config", None) if condor_path != None: addToPath(condor_path) if condor_location != None: addToPath('%s/bin' % condor_location) if not condor_config: condor_config = '%s/etc/condor_config' % condor_location if condor_config: os.environ['CONDOR_CONFIG'] = condor_config #vo_map = VoMapper(cp) getLrmsInfo(cp) print_CE(cp) print_VOViewLocal(cp) except Exception, e: log.exception(e) raise
def print_classic_access(cp, siteUniqueID): """ Emit the GLUE entity for a classic SE's access protocol. """ fallback_name = siteUniqueID + "_classicSE" seUniqueID = cp_get(cp, "classic_se", "unique_name", fallback_name) try: default_host = socket.gethostname() except: default_host = 'UNKNOWN.example.org' host = cp_get(cp, "classic_se", "host", default_host) port = cp_getInt(cp, "classic_se", "port", "2811") accessTemplate = getTemplate("GlueSE", "GlueSEAccessProtocolLocalID") endpoint = 'gsiftp://%s:%i' % (host, port) info = { 'accessProtocolID': 'GFTP_%s_%i' % (host, port), 'seUniqueID': seUniqueID, 'protocol': 'gsiftp', 'endpoint': endpoint, 'capability': 'file transfer', 'maxStreams': 10, 'security': 'gsiftp', 'port': port, 'version': '1.0.0', } print accessTemplate % info
def getHTCondorCEVersion(cp): """ Returns the running version of the HTCondor CE Copied from getOSGVersion() in gip_cluster.py """ log = logging.getLogger() htcondorce_ver_backup = cp_get(cp, "ce", "htcondorce_version", "1.8") htcondorce_version_script = cp_get(cp, "gip", "htcondorce_version_script", "") htcondorce_ver = '' if len(htcondorce_version_script) == 0: htcondorce_version_script = vdtDir('$VDT_LOCATION/condor_ce_config_val', '/usr/bin/condor_ce_config_val') htcondorce_version_script = os.path.expandvars(htcondorce_version_script) if not os.path.exists(htcondorce_version_script): htcondorce_version_script = os.path.expandvars("$VDT_LOCATION/osg/bin/" \ "osg-version") if os.path.exists(htcondorce_version_script): try: htcondorce_version_script += " HTCondorCEVersion" htcondorce_ver = runCommand(htcondorce_version_script).read().strip() htcondorce_ver = htcondorce_ver.replace('"','') except Exception, e: log.exception(e)
def handle_SE(cp, section): """ Run a provider for one SE. """ # if you only have a classic SE, there will still be a [se] section # with the default_se option. This will result in a GlueService # stanza being written with garbage information in it. So, check # for default_se = "UNKNOWN" or "UNAVAILABLE" and return if it does # # default_se is set in the [Storage] section in config.ini and is # required by configure-osg.py default_se = cp_get(cp, "se", "default_se", "UNKNOWN") if default_se == "UNAVAILABLE" or default_se == "UNKNOWN": return impl = cp_get(cp, section, "implementation", "UNKNOWN") provider_impl = cp_get(cp, section, "provider_implementation", "UNKNOWN") if provider_impl == "UNKNOWN" and not cp_getBoolean( cp, section, "dynamic_dcache", False): provider_impl = 'static' se_class, cp = determine_provider(provider_impl, impl, cp) se = se_class(cp, section=section) log.info("Outputting SE %s; implementation %s; provider %s" % (se.getName(), impl, provider_impl)) try: se.run() except Exception, e: log.exception(e)
def print_VOViewLocal(cp): ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE") vo_map = VoMapper(cp) queue_jobs = getJobsInfo(vo_map, cp) vo_queues = getVoQueues(cp) VOView = getTemplate("GlueCE", "GlueVOViewLocalID") for vo, queue in vo_queues: ce_unique_id = buildCEUniqueID(cp, ce_name, 'sge', queue) info = { 'ceUniqueID' : ce_unique_id, 'voLocalID' : vo, 'acbr' : 'VO:%s' % vo, 'running' : queue_jobs.get(queue, {}).get(vo, {}).\ get('running', 0), 'waiting' : queue_jobs.get(queue, {}).get(vo, {}).\ get('waiting', 0), #'free_slots' : vo.get(queue, {}).get('free_slots', 0), 'free_slots' : 0, #TODO: fix 'ert' : 3600, 'wrt' : 3600, 'default_se' : getDefaultSE(cp), 'app' : cp_get(cp, "osg_dirs", "app", "/OSG_APP_UNKNOWN"), 'data' : cp_get(cp, "osg_dirs", "data", "/OSG_DATA_UNKNOWN"), } info['total'] = info['waiting'] + info['running'] printTemplate(VOView, info)
def interpolateConfig(cp): grid = cp_get(cp, "site", "group", "") if cp_getBoolean(cp, "gip_tests", "oim_aware", False): sitelist_cmd = "wget -O - http://oim.grid.iu.edu/pub/resource/show.php?format=plain-text 2>/dev/null | grep \",%s,\" | grep \",CE\" | cut -f1 -d," % grid sitelist = runCommand(sitelist_cmd).read().split() sitelist = ",".join(sitelist) cp.set("gip_tests", "site_names", sitelist) else: if cp_get(cp, "gip_tests", "site_names", "") == "": cp.set("gip_tests", "site_names", cp_get(cp, "site", "name", "")) if cp_get(cp, "gip_tests", "site_dns", "") == "": host_parts = cp_get(cp, "ce", "name", "").split('.') site_dns = "%s.%s" % (host_parts[:-2], host_parts[:-1]) cp.set("gip_tests", "site_dns", site_dns) if cp_get(cp, "gip_tests", "required_site", "") == "": cp.set("gip_tests", "required_sites", cp_get(cp, "gip_tests", "site_names", "")) cp.set("gip_tests", "bdii_port", "2170") cp.set("gip_tests", "egee_port", "2170") cp.set( "gip_tests", "interop_url", "http://oim.grid.iu.edu/publisher/get_osg_interop_bdii_ldap_list.php?grid=%s&format=html" % grid) if strContains(grid, "ITB"): cp.set("bdii", "endpoint", "ldap://is-itb.grid.iu.edu:2170") cp.set("gip_tests", "bdii_addr", "is-itb.grid.iu.edu") cp.set("gip_tests", "egee_bdii", "pps-bdii.cern.ch") cp.set( "gip_tests", "egee_bdii_conf_url", "http://egee-pre-production-service.web.cern.ch/egee-pre-production-service/bdii/pps-all-sites.conf" ) web_server = "http://is-itb.grid.iu.edu" else: cp.set("gip_tests", "bdii_addr", "is.grid.iu.edu") cp.set("gip_tests", "egee_bdii", "lcg-bdii.cern.ch") cp.set("gip_tests", "egee_bdii_conf_url", "http://lcg-bdii-conf.cern.ch/bdii-conf/bdii.conf") web_server = "http://is.grid.iu.edu" cp.set("gip_tests", "update_url", web_server + "/cgi-bin/status.cgi") cp.set("gip_tests", "schema_check_url", web_server + "/data/cemon_processed_osg/%s.processed?which=%s") cp.set("gip_tests", "validator_url", web_server + "/data/cemon_processed_osg/%s.processed?which=%s") if cp_get(cp, "gip_tests", "compare_excludes", "") == "": compare_excludes = "GlueCEStateFreeJobSlots,GlueCEStateRunningJobs,GlueCEStateTotalJobs,GlueSiteLocation,GlueSAStateAvailableSpace,GlueSAStateUsedSpace" cp.set("gip_tests", "compare_excludes", compare_excludes) if cp_get(cp, "gip_tests", "enable_glite", "") == "": cp.set("gip_tests", "enable_glite", "False") if cp_get(cp, "gip_tests", "results_dir", "") == "": results_dir = os.path.expandvars("$GIP_LOCATION/../apache/htdocs/") cp.set("gip_tests", "results_dir", results_dir)
def getDefaultSE(cp): global _defaultSE if _defaultSE: return _defaultSE default_se = cp_get(cp, "se", "name", "UNKNOWN") # if [se] name: ??? is "UNAVAILABLE" or not set, then try to get the # default_se if default_se == "UNKNOWN" or default_se == "UNAVAILABLE": default_se = cp_get(cp, "se", "default_se", "UNAVAILABLE") # if it is still UNAVAILABLE or not set, check to see if the classic SE # is being advertised and use that if default_se == "UNAVAILABLE" and cp_getBoolean(cp, "classic_se", "advertise_se", True): fallback_name = cp_get(cp, "site", "unique_name", "UNKNOWN") + \ "_classicSE" default_se = cp_get(cp, "classic_se", "name", fallback_name) current_se = None for sect in cp.sections(): if not sect.lower().startswith('se'): continue try: current_se = cp.get(sect, 'name') except: continue if cp_getBoolean(cp, sect, "default", False): _defaultSE = current_se return current_se if default_se == 'UNKNOWN' and current_se: _defaultSE = current_se return current_se _defaultSE = default_se return default_se
def getWLCGInfo(cp): """ Generate GRID, WLCG_TIER, WLCG_PARENT, and WLCG_NAME if appropriate. """ grid = cp_get(cp, "site", "wlcg_grid", "OSG").strip() tier = cp_get(cp, "site", "wlcg_tier", "").strip() parent = cp_get(cp, "site", "wlcg_parent", "").strip() name = cp_get(cp, "site", "wlcg_name", "").strip() log.debug('WLCG attributes: wlcg_grid, wlcg_tier, wlcg_parent, wlcg_name: ("%s", "%s", "%s", "%s")' % (grid, tier, parent, name)) otherInfoTemplate = 'GlueSiteOtherInfo: %s\n' if grid and tier and parent and name: otherInfo = otherInfoTemplate % ('GRID=WLCG') otherInfo += otherInfoTemplate % ('GRID=%s' % grid) otherInfo += otherInfoTemplate % ('WLCG_TIER=%s' % tier) otherInfo += otherInfoTemplate % ('WLCG_PARENT=%s' % parent) otherInfo += otherInfoTemplate % ('WLCG_NAME=%s' % name) else: log.debug('At least one null WLCG attribute - not publishing WLCG-style GlueSiteOtherInfo') otherInfo = '' return otherInfo
def handle_SE(cp, section): """ Run a provider for one SE. """ # if you only have a classic SE, there will still be a [se] section # with the default_se option. This will result in a GlueService # stanza being written with garbage information in it. So, check # for default_se = "UNKNOWN" or "UNAVAILABLE" and return if it does # # default_se is set in the [Storage] section in config.ini and is # required by configure-osg.py default_se = cp_get(cp, "se", "default_se", "UNKNOWN") if default_se == "UNAVAILABLE" or default_se == "UNKNOWN": return impl = cp_get(cp, section, "implementation", "UNKNOWN") provider_impl = cp_get(cp, section, "provider_implementation", "UNKNOWN") if provider_impl == "UNKNOWN" and not cp_getBoolean(cp, section, "dynamic_dcache", False): provider_impl = 'static' se_class, cp = determine_provider(provider_impl, impl, cp) se = se_class(cp, section=section) log.info("Outputting SE %s; implementation %s; provider %s" % (se.getName(), impl, provider_impl)) try: se.run() except Exception, e: log.exception(e)
def getWLCGInfo(cp): """ Generate GRID, WLCG_TIER, WLCG_PARENT, and WLCG_NAME if appropriate. """ grid = cp_get(cp, "site", "wlcg_grid", "OSG").strip() tier = cp_get(cp, "site", "wlcg_tier", "").strip() parent = cp_get(cp, "site", "wlcg_parent", "").strip() name = cp_get(cp, "site", "wlcg_name", "").strip() log.debug( 'WLCG attributes: wlcg_grid, wlcg_tier, wlcg_parent, wlcg_name: ("%s", "%s", "%s", "%s")' % (grid, tier, parent, name)) otherInfoTemplate = 'GlueSiteOtherInfo: %s\n' if grid and tier and parent and name: otherInfo = otherInfoTemplate % ('GRID=WLCG') otherInfo += otherInfoTemplate % ('GRID=%s' % grid) otherInfo += otherInfoTemplate % ('WLCG_TIER=%s' % tier) otherInfo += otherInfoTemplate % ('WLCG_PARENT=%s' % parent) otherInfo += otherInfoTemplate % ('WLCG_NAME=%s' % name) else: log.debug( 'At least one null WLCG attribute - not publishing WLCG-style GlueSiteOtherInfo' ) otherInfo = '' return otherInfo
def print_classic_access(cp, siteUniqueID): """ Emit the GLUE entity for a classic SE's access protocol. """ fallback_name = siteUniqueID + "_classicSE" seUniqueID = cp_get(cp, "classic_se", "unique_name", fallback_name) try: default_host = socket.gethostname() except: default_host = 'UNKNOWN.example.org' host = cp_get(cp, "classic_se", "host", default_host) port = cp_getInt(cp, "classic_se", "port", "2811") accessTemplate = getTemplate("GlueSE", "GlueSEAccessProtocolLocalID") endpoint = 'gsiftp://%s:%i' % (host, port) info = {'accessProtocolID' : 'GFTP_%s_%i' % (host, port), 'seUniqueID' : seUniqueID, 'protocol' : 'gsiftp', 'endpoint' : endpoint, 'capability' : 'file transfer', 'maxStreams' : 10, 'security' : 'gsiftp', 'port' : port, 'version' : '1.0.0', } print accessTemplate % info
def __init__(self, args): self.cp = getTestConfig(args) self.args = " ".join(args) # these are the test lists by category self.reports = [] self.critical_tests = [] self.glite_tests = [] # output files self.output_files = [] # command line values self.output_dir = cp_get( self.cp, "gip_tests", "results_dir", os.path.expandvars("$GIP_LOCATION/reporting/results")) self.source_cmd = "source %s/setup.sh" % os.path.expandvars( "$VDT_LOCATION") self.extList = [".xsl"] # set up the critical test list default_critical_tests = "Interop_Reporting_Check,Missing_Sites,Validate_GIP_BDII,Validate_GIP_URL" crit_conf = cp_get(self.cp, "gip_tests", "critical_tests", default_critical_tests) self.crit = [i.strip() for i in crit_conf.split(',')]
def setConfigValues(self): self.results_dir = cp_get(self.cp, "gip_tests", "results_dir", "UNKNOWN") if self.results_dir == "UNKNOWN": raise ConfigurationError("Results directory is not configured") self.oim_xml_dir = cp_get(self.cp, "gip_tests", "myosg_xml_dir", "UNKNOWN") if self.oim_xml_dir == "UNKNOWN": raise ConfigurationError("OIM XML directory is not configured") self.oim_summary_xml_file = "%s/%s" % (self.oim_xml_dir, cp_get(self.cp, "gip_tests", "myosg_summary_file", "myosg.xml")) self.oim_detail_file_template = self.oim_xml_dir + "/" + cp_get(self.cp, "gip_tests", "myosg_detail_file_template", "myosg_%s_detail.xml") self.oim_tests = cp_get(self.cp, "gip_tests", "myosg_tests", self.oim_tests)
def publish_gums(cp, template): hostname = cp_get(cp, "ce", 'name', gethostname()) siteID = cp_get(cp, "site", "unique_name", gethostname()) gumsConfig = vdtDir(os.path.expandvars('$VDT_LOCATION/gums/config/gums-client.properties'), '/etc/gums/gums-client.properties') gumsConfigFile = open(gumsConfig, 'r') gums_re = re.compile('gums.authz\s*=\s*(https://(.*):.*?/(.*))') lines = gumsConfigFile.readlines() for line in lines: m = gums_re.match(line) if m: (gums_uri, gums_host) = m.groups()[0:2] os.putenv('X509_USER_CERT', '/etc/grid-security/http/httpcert.pem') os.putenv('X509_USER_KEY' , '/etc/grid-security/http/httpkey.pem') mapping_subject_dn = '/GIP-GUMS-Probe-Identity' mapping_subject_name = '`grid-cert-info -subject` ' gums_command = vdtDir(os.path.expandvars('$VDT_LOCATION/gums/scripts/gums-service'), '/usr/bin/gums-service') gums_command += ' mapUser -s ' + mapping_subject_name + mapping_subject_dn (gums_output, pin) = popen2.popen4(gums_command) gums_id_re = re.compile('.*\[userName: (.*)\].*') status = "Warning" statusInfo = "Test mapping failed: if GUMS was not down, check logs" +\ " at " + gums_host + ':' + '$VDT_LOCATION/tomcat/v55/logs' lines = gums_output.readlines() for line in lines: m = gums_id_re.match(line) if m: uidMapping = m.groups([0]) status = "OK" statusInfo = "Test mapping successful: user id = %s" % uidMapping break info = {'serviceID': gums_uri, 'serviceType': 'GUMS', 'serviceName': 'Authorization', 'version': 'UNDEFINED', 'endpoint': gums_uri, 'semantics': 'UNDEFINED', 'owner': '', 'url': gums_uri, 'uri': gums_uri, 'status': status, 'statusInfo': statusInfo, 'wsdl': 'Not Applicable', 'startTime': 'Not Applicable', 'siteID': siteID, 'acbr': '__GIP_DELETEME' } printTemplate(template, info)
def main(): log.info('Starting CREAM service provider') try: cp = config() serviceID = buildServiceID(cp) siteID = cp_get(cp, "site", "unique_name", 'UNKNOWN_SITE') serviceName = '%s-CREAM' % siteID creamVersion = getCreamVersion(cp) endpoint = 'https://%s:8443/ce-cream/services' % cp_get(cp, "ce", "name", 'UNKNOWN_CE') allVOs = voList(cp) acbr = '' owner = '' log.debug('CREAM VOs are %s' % allVOs) if not allVOs: log.error("No VOs supported!") acbr = '__GIP_DELETEME' else: acbr = '\n'.join(['GlueServiceAccessControlBaseRule: %s\n' \ 'GlueServiceAccessControlBaseRule: VO:%s' % (vo, vo) for vo in allVOs]) owner = '\n' + '\n'.join(['GlueServiceOwner: %s' % vo for vo in allVOs]) # owner needs an extra prepended newline pid = -1 startTime = 'Not Applicable' serviceStatus = 'Not OK' serviceStatusInfo = 'Could not find tomcat process' try: (startTime, pid) = getStartTimeAndPid(cp) serviceStatus = 'OK' serviceStatusInfo = 'Tomcat (%d) is running' % pid except: log.error('Could not locate tomcat process (pgrep -f "org.apache.catalina.startup.Bootstrap start"' ' probably failed to return any output!)') info = {'serviceID': serviceID, 'serviceType': 'org.glite.ce.CREAM', 'serviceName': serviceName, 'version': creamVersion, 'endpoint': endpoint, 'semantics': 'https://edms.cern.ch/document/595770', 'owner': owner, 'url': '__GIP_DELETEME', # deprecated 'uri': '__GIP_DELETEME', # deprecated 'status': serviceStatus, 'statusInfo': serviceStatusInfo, 'wsdl': 'http://grid.pd.infn.it/cream/wsdl/org.glite.ce-cream_service.wsdl', 'startTime': startTime, 'siteID': siteID, 'acbr': acbr } template = getTemplate("GlueService", "GlueServiceUniqueID") printTemplate(template, info) except Exception, e: sys.stdout = sys.stderr log.error(e) raise
def getApplications(cp): """ Return a list of dictionaries containing the application info for each application installed at this site. Each returned dictionary should have the following keys: - locationId - locationName - version - path @param cp: Site configuration @return: List of dictionaries; each dictionary contains the information about a specific installed application. """ app_dir = cp_get(cp, "osg_dirs", "app", "/UNKNOWN") path1 = '%s/etc/grid3-locations.txt' % app_dir path2 = '%s/etc/osg-locations.txt' % app_dir paths = [path1, path2] path3 = cp_get(cp, "ce", "app_list", '') if path3: paths.append(path3) locations = {} for path in paths: try: fp = open(path, 'r') except: continue for line in fp: line = line.strip() info = line.split() # Skip blank lines and comments if len(line) == 0 or info[0].startswith('#'): continue if len(info) != 3: log.warning("Invalid line: %s" % line) continue if info[1].startswith('#') or info[1].startswith('$'): info[1] = 'UNDEFINED' info = {'locationName': info[0], 'version': info[1], 'path':info[2]} info['locationId'] = info['locationName'] locations = _addLocation(locations, info) osg_ver = getOSGVersion(cp) if osg_ver: # old method: will deprecate in a future version, but leave # for backwards compatibility info = {'locationId': osg_ver, 'locationName': osg_ver, 'version': \ osg_ver, 'path': os.environ.get('VDT_LOCATION', '/UNKNOWN')} locations = _addLocation(locations, info) # new method info = {'locationId': 'OSG_VERSION', 'locationName': 'OSG_VERSION', 'version': osg_ver, 'path': os.environ.get('VDT_LOCATION', '/UNKNOWN')} locations = _addLocation(locations, info) try: locations.update(getApplicationsV1(cp)) except Exception, e: log.exception(e)
def _generateSubClusterHelper(cp, section): """ Private helper function for generateSubClusters; do not use. Implementation note: We create a bunch of variables but never appear to use them - we actually do return the local dictionary at the end of the function. Just being syntactically lazy. """ #pylint: disable-msg=W0612 # Names subCluster = cp_get(cp, section, "name", cluster) subClusterUniqueID = cp_get(cp, section, "unique_name", subCluster) clusterUniqueID = getClusterID(cp) # Host statistics clockSpeed = cp_getInt(cp, section, "cpu_speed_mhz", 999999999) cpuCount = cp_getInt(cp, section, "cpus_per_node", 2) model = cp_get(cp, section, "cpu_model", 'UNDEFINEDVALUE') platform = cp_get(cp, section, "platform", "UNKNOWN") vendor = cp_get(cp, section, "cpu_vendor", 'UNDEFINEDVALUE') cores_per_cpu = cp_getInt(cp, section, "cores_per_cpu", 2) si2k = cp_getInt(cp, section, "SI00", 2000) sf2k = cp_getInt(cp, section, "SF00", 2000) hepspec = cp_getInt(cp, section, "HEPSPEC", 0) ram = cp_getInt(cp, section, "ram_size", 1000*cpuCount*cores_per_cpu) cores = cp_getInt(cp, section, "total_cores", 999999999) if cores_per_cpu != 0: cpus = cp_getInt(cp, section, "total_cpus", cores/cores_per_cpu) else: cpus = 0 virtualMem = ram + cp_getInt(cp, section, "swap_size", 0) inboundIP = cp_getBoolean(cp, section, "inbound_network", False) outboundIP = cp_getBoolean(cp, section, "outbound_network", True) inboundIP = ldap_boolean(inboundIP) outboundIP = ldap_boolean(outboundIP) # OS Stats osName, osRelease, osVersion = getRelease() # Temp directories default_tmp = cp_get(cp, "osg_dirs", "tmp", cp_get(cp, "osg_dirs", "data", \ "/tmp")) wn_tmp = cp_get(cp, "osg_dirs", "wn_tmp", "/tmp") tmp = cp_get(cp, section, "tmp", default_tmp) if notDefined(tmp): tmp = default_tmp app_attr = 'GlueHostApplicationSoftwareRunTimeEnvironment' apps = getApplications(cp) applications = '\n'.join(['%s: %s' % (app_attr, i['locationId']) for i in \ apps if i['locationId']]) applications += '\n' # BDII stuff bdii = cp_get(cp, "bdii", "endpoint", "ldap://is.grid.iu.edu:2170") return locals()
def getUniqueID(self): """ Return the unique ID of the SE. The base class uses the value of se.unique_name (defaults to se.name) in the configuraiton object. """ return cp_get(self._cp, self._section, 'unique_name', cp_get(self._cp, self._section, 'name', 'UNKNOWN'))
def getPath(cp, space, vo=None, return_default=True, section='se'): """ Return a path appropriate for a VO and space. Based upon the configuration info and the VO/space requested, determine a path they should use. This function tries to find option dcache.space_<space>_path; it parses this as a comma-separated list of VO:path pairs; i.e., space_CMS_path=cms:/dev/null, atlas:/pnfs/blah If that does not provide a match and return_default is true, then it will look for dcache.space_<space>_default_path and return that. If that is not there and return_default is true, it will use the standard getPath from gip_storage. If return_default is true, this is guaranteed to return a non-empty string; if return_default is false, then this might through a ValueError exception. @param cp: Site config object @param space: The name of the space to determine the path for. @param vo: The name of the VO which will be using this space; None for the default information. @kw return_default: Set to True if you want this function to return the default path if it cannot find a VO-specific one. @returns: A path string; raises a ValueError if return_default=False """ default_path = cp_get(cp, section, "space_%s_default_path" % space, None) if not default_path: if section == 'se': default_path = getStoragePath(cp, vo) else: default_path = getStoragePath(cp, vo, section=section) default_path = getStoragePath(cp, vo, section=section) log.debug("Looking up info for space %s, vo %s, section %s; default " \ "path is %s" % (space, vo, section, default_path)) if not vo: return default_path vo_specific_paths = cp_get(cp, section, "space_%s_path" % space, \ None) vo_specific_path = None if vo_specific_paths: for value in vo_specific_paths.split(','): value = value.strip() info = value.split(':') if len(info) != 2: continue fqan, path = info if matchFQAN(vo, fqan): vo_specific_path = path break if not vo_specific_path and return_default: return default_path elif not vo_specific_path: raise ValueError("Unable to determine path for %s!" % vo) return vo_specific_path
def print_clusters(cp): cluster_name = cp_get(cp, "cluster", "name", None) if not cluster_name: cluster_name = cp_get(cp, "ce", "hosting_cluster", None) if not cluster_name: cluster_name = cp_get(cp, "ce", "unique_name", None) if not cluster_name: getClusterName(cp) # raise Exception("Could not determine cluster name.") # clusterUniqueID = cp_get(cp, 'ce', 'unique_name', cluster_name) clusterUniqueID = getClusterID(cp) siteUniqueID = cp_get(cp, "site", "unique_name", "UNKNOWN_SITE") extraCEs = cp_get(cp, "cluster", "other_ces", []) if extraCEs: extraCEs = [x.strip() for x in extraCEs.split(",")] ces = getCEList(cp, extraCEs) glueClusters = "" for ce in ces: glueClusters += "GlueForeignKey: GlueCEUniqueID=%s\n" % ce bdii = cp_get(cp, "gip", "bdii", "ldap://is.grid.iu.edu:2170") info = { "cluster": cluster_name, "clusterUniqueID": clusterUniqueID, "tmp": cp_get(cp, "osg_dirs", "tmp", cp_get(cp, "osg_dirs", "data", "/tmp")), "wn_tmp": cp_get(cp, "osg_dirs", "wn_tmp", "/tmp"), "siteUniqueID": siteUniqueID, "glueClusters": glueClusters, "bdii": bdii, } template = getTemplate("GlueCluster", "GlueClusterUniqueID") printTemplate(template, info)
def print_clusters(cp): cluster_name = cp_get(cp, 'cluster', 'name', None) if not cluster_name: cluster_name = cp_get(cp, 'ce', 'hosting_cluster', None) if not cluster_name: cluster_name = cp_get(cp, 'ce', 'unique_name', None) if not cluster_name: getClusterName(cp) #raise Exception("Could not determine cluster name.") #clusterUniqueID = cp_get(cp, 'ce', 'unique_name', cluster_name) clusterUniqueID = getClusterID(cp) siteUniqueID = cp_get(cp, "site", "unique_name", 'UNKNOWN_SITE') extraCEs = cp_get(cp, 'cluster', 'other_ces', []) if extraCEs: extraCEs = [x.strip() for x in extraCEs.split(',')] ces = getCEList(cp, extraCEs) glueClusters = '' for ce in ces: glueClusters += 'GlueForeignKey: GlueCEUniqueID=%s\n' % ce bdii = cp_get(cp, 'gip', 'bdii', 'ldap://is.grid.iu.edu:2170') info = { \ 'cluster': cluster_name, 'clusterUniqueID': clusterUniqueID, 'tmp': cp_get(cp, "osg_dirs", "tmp", cp_get(cp, "osg_dirs", "data", \ "/tmp")), 'wn_tmp': cp_get(cp, "osg_dirs", "wn_tmp", "/tmp"), 'siteUniqueID': siteUniqueID, 'glueClusters': glueClusters, 'bdii': bdii, } template = getTemplate("GlueCluster", "GlueClusterUniqueID") printTemplate(template, info)
def getClusterID(cp): """ Return the unique ID of the associated cluster. """ ce_name = cp_get(cp, ce, 'unique_name', \ cp_get(cp, ce, 'name', "UNKNOWN_CE")) simple = cp.getboolean(cluster, 'simple') if simple: return ce_name else: return cp_get(cp, cluster, 'name', ce_name)
def getSubClusterIDs(cp): """ Return a list of the subcluster unique ID's for this configuration. """ subclusters = [] for section in cp.sections(): if not section.startswith(subcluster): continue subCluster = cp_get(cp, section, "name", "UNKNOWN") subclusters.append(cp_get(cp, section, "unique_name", subCluster)) return subclusters
def print_VOViewLocal(queue_info, cp): ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE") vo_map = VoMapper(cp) queue_jobs = getJobsInfo(vo_map, cp) VOView = getTemplate("GlueCE", "GlueVOViewLocalID") vo_queues = getVoQueues(cp) for vo, queue in vo_queues: vo_info = queue_jobs.get(queue, {}) info2 = vo_info.get(vo, {}) port = getPort(cp) ce_unique_id = buildCEUniqueID(cp, ce_name, 'pbs', queue) my_queue_info = queue_info.setdefault(queue, {}) max_job_time = my_queue_info.get("max_wall", 0) if cp.has_option("pbs", "max_wall"): max_job_time = cp_getInt(cp, "pbs", "max_wall", 1440) ert, wrt = responseTimes(cp, info2.get("running", 0), info2.get("wait", 0), max_job_time) free_slots = my_queue_info.get('free_slots', 0) waiting = info2.get('wait', 0) if waiting > cp_getInt(cp, 'pbs', 'idle_slack', '10'): free_slots = 0 info = { 'ceUniqueID' : ce_unique_id, 'job_slots' : my_queue_info.get('job_slots', 0), 'free_slots' : free_slots, 'ce_name' : ce_name, 'queue' : queue, 'vo' : vo, 'voLocalID' : vo, 'job_manager' : 'pbs', 'running' : info2.get('running', 0), 'max_running' : info2.get('max_running', 0), 'priority' : queue_info.get(queue, {}).get('priority', 0), 'waiting' : waiting, 'data' : cp_get(cp, "osg_dirs", "data", "UNKNOWN_DATA"), 'app' : cp_get(cp, "osg_dirs", "app", "UNKNOWN_APP"), 'default_se' : getDefaultSE(cp), 'ert' : 3600, 'wrt' : 3600, 'acbr' : 'VO:%s' % vo } info['total'] = info['waiting'] + info['running'] printTemplate(VOView, info)
def interpolateConfig(cp): grid = cp_get(cp, "site", "group", "") if cp_getBoolean(cp, "gip_tests", "oim_aware", False): sitelist_cmd = "wget -O - http://oim.grid.iu.edu/pub/resource/show.php?format=plain-text 2>/dev/null | grep \",%s,\" | grep \",CE\" | cut -f1 -d," % grid sitelist = runCommand(sitelist_cmd).read().split() sitelist = ",".join(sitelist) cp.set("gip_tests", "site_names", sitelist) else: if cp_get(cp, "gip_tests", "site_names", "") == "": cp.set("gip_tests", "site_names", cp_get(cp, "site", "name", "")) if cp_get(cp, "gip_tests", "site_dns", "") == "": host_parts = cp_get(cp, "ce", "name", "").split('.') site_dns = "%s.%s" % (host_parts[:-2], host_parts[:-1]) cp.set("gip_tests", "site_dns", site_dns) if cp_get(cp, "gip_tests", "required_site", "") == "": cp.set("gip_tests", "required_sites", cp_get(cp, "gip_tests", "site_names", "")) cp.set("gip_tests", "bdii_port", "2170") cp.set("gip_tests", "egee_port", "2170") cp.set("gip_tests", "interop_url", "http://oim.grid.iu.edu/publisher/get_osg_interop_bdii_ldap_list.php?grid=%s&format=html" % grid) if strContains(grid, "ITB"): cp.set("bdii", "endpoint", "ldap://is-itb.grid.iu.edu:2170") cp.set("gip_tests", "bdii_addr", "is-itb.grid.iu.edu") cp.set("gip_tests", "egee_bdii", "pps-bdii.cern.ch") cp.set("gip_tests", "egee_bdii_conf_url", "http://egee-pre-production-service.web.cern.ch/egee-pre-production-service/bdii/pps-all-sites.conf") web_server = "http://is-itb.grid.iu.edu" else: cp.set("gip_tests", "bdii_addr", "is.grid.iu.edu") cp.set("gip_tests", "egee_bdii", "lcg-bdii.cern.ch") cp.set("gip_tests", "egee_bdii_conf_url", "http://lcg-bdii-conf.cern.ch/bdii-conf/bdii.conf") web_server = "http://is.grid.iu.edu" cp.set("gip_tests", "update_url", web_server + "/cgi-bin/status.cgi") cp.set("gip_tests", "schema_check_url", web_server + "/data/cemon_processed_osg/%s.processed?which=%s") cp.set("gip_tests", "validator_url", web_server + "/data/cemon_processed_osg/%s.processed?which=%s") if cp_get(cp, "gip_tests", "compare_excludes", "") == "": compare_excludes="GlueCEStateFreeJobSlots,GlueCEStateRunningJobs,GlueCEStateTotalJobs,GlueSiteLocation,GlueSAStateAvailableSpace,GlueSAStateUsedSpace" cp.set("gip_tests", "compare_excludes", compare_excludes) if cp_get(cp, "gip_tests", "enable_glite", "") == "": cp.set("gip_tests", "enable_glite", "False") if cp_get(cp, "gip_tests", "results_dir", "") == "": results_dir = os.path.expandvars("$GIP_LOCATION/../apache/htdocs/") cp.set("gip_tests", "results_dir", results_dir)
def print_VOViewLocal(queue_info, cp): ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE") vo_map = VoMapper(cp) queue_jobs = getJobsInfo(vo_map, cp) VOView = getTemplate("GlueCE", "GlueVOViewLocalID") vo_queues = getVoQueues(cp) for vo, queue in vo_queues: vo_info = queue_jobs.get(queue, {}) info2 = vo_info.get(vo, {}) port = getPort(cp) ce_unique_id = buildCEUniqueID(cp, ce_name, "pbs", queue) my_queue_info = queue_info.setdefault(queue, {}) max_job_time = my_queue_info.get("max_wall", 0) if cp.has_option("pbs", "max_wall"): max_job_time = cp_getInt(cp, "pbs", "max_wall", 1440) ert, wrt = responseTimes(cp, info2.get("running", 0), info2.get("wait", 0), max_job_time) free_slots = my_queue_info.get("free_slots", 0) waiting = info2.get("wait", 0) if waiting > cp_getInt(cp, "pbs", "idle_slack", "10"): free_slots = 0 info = { "ceUniqueID": ce_unique_id, "job_slots": my_queue_info.get("job_slots", 0), "free_slots": free_slots, "ce_name": ce_name, "queue": queue, "vo": vo, "voLocalID": vo, "job_manager": "pbs", "running": info2.get("running", 0), "max_running": info2.get("max_running", 0), "priority": queue_info.get(queue, {}).get("priority", 0), "waiting": waiting, "data": cp_get(cp, "osg_dirs", "data", "UNKNOWN_DATA"), "app": cp_get(cp, "osg_dirs", "app", "UNKNOWN_APP"), "default_se": getDefaultSE(cp), "ert": 3600, "wrt": 3600, "acbr": "VO:%s" % vo, } info["total"] = info["waiting"] + info["running"] printTemplate(VOView, info)
def getCESEBindInfo(cp): """ Generates a list of information for the CESE bind groups. Each list entry is a dictionary containing the necessary information for filling out a CESE bind entry. @param cp: Site configuration @returns: List of dictionaries; each dictionary is a CESE bind entry. """ binds = [] ce_list = getCEList(cp) se_list = getSEList(cp, classicSEs=False) classicse_list = getClassicSEList(cp) se_list.extend(classicse_list) #access_point = cp_get(cp, "vo", "default", "/") section_map = getSESections(cp) access_point = getPath(cp) if not access_point: access_point = "/" classic_access_point = cp_get(cp, "osg_dirs", "data", "/") for myse in se_list: mount_point = None if myse in section_map: section = section_map[myse] mount_point = cp_get(cp, section, "mount_point", None) for myce in ce_list: if myse in classicse_list: ap = classic_access_point else: ap = access_point info = { 'ceUniqueID': myce, 'seUniqueID': myse, 'access_point': ap, } if mount_point: # Note: always start this with a newline, otherwise the # GLUE template will put this on the same line as the previous # attribute info['mount_point'] = '\nGlueCESEBindMountInfo: %s' % \ mount_point else: info['mount_point'] = '' binds.append(info) return binds
def getSRMs(self): """ Return a list of dictionaries containing information about the SRM endpoints. Each dictionary must have the following keys: - acbr - status - version - endpoint - name The base class implementation uses the following configuration entries (default value in parenthesis) - se.srm_host (default: UNKNOWN.example.com) - se.srm_version (2.2.0) - se.srm_port (8443) - se.srm_endpoint (httpg://(se.srm_host):(se.srm_port)/srm/managerv2) """ srmname = cp_get(self._cp, self._section, "srm_host", "UNKNOWN.example.com") version = cp_get(self._cp, self._section, "srm_version", "2") port = cp_getInt(self._cp, self._section, "srm_port", 8443) if version.find('2') >= 0: default_endpoint = 'httpg://%s:%i/srm/managerv2' % \ (srmname, int(port)) else: default_endpoint = 'httpg://%s:%i/srm/managerv1' % \ (srmname, int(port)) endpoint = cp_get(self._cp, self._section, "srm_endpoint", default_endpoint) acbr_tmpl = '\nGlueServiceAccessControlRule: %s\n' \ 'GlueServiceAccessControlRule: VO:%s' acbr = '' vos = voListStorage(self._cp, self._section) for vo in vos: acbr += acbr_tmpl % (vo, vo) info = {'acbr': acbr[1:], 'status': 'OK', 'version': version, 'endpoint': endpoint, 'name': srmname, } return [info]
def pbsCommand(command, cp): """ Run a command against the PBS batch system. Use this when talking to PBS; not only does it allow for integration into the GIP test framework, but it also filters and expands PBS-style line continuations. """ try: pbsHost = cp.get("pbs", "host") except: pbsHost = "" if pbsHost.lower() == "none" or pbsHost.lower() == "localhost": pbsHost = "" pbs_path = cp_get(cp, "pbs", "pbs_path", ".") addToPath(pbs_path) addToPath(pbs_path + "/bin") cmd = command % {'pbsHost': pbsHost} fp = runCommand(cmd) #pid, exitcode = os.wait() #if exitcode != 0: # raise Exception("Command failed: %s" % cmd) return pbsOutputFilter(fp)
def __init__(self, site, cp): GipUnittest.GipTestCase.__init__(self, 'testGipOutput') self.type = cp_get(self.cp, "gip_tests", "validate_type", "") self.name = 'testGipOutput_%s' % site self.site = site self.entries = None self.site_id = None
def main(): try: # Load up the site configuration cp = config() se_only = cp_getBoolean(cp, "gip", "se_only", False) if not se_only: # Load up the template for GlueLocationLocalID # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster template = getTemplate("GlueCluster", "GlueLocationLocalID") cluster_id = getClusterID(cp) osg_grid = cp_get(cp, "osg_dirs", "grid_dir", None) if not osg_grid: raise RuntimeError('grid_dir ($OSG_GRID) not defined!') for subClusterId in getSubClusterIDs(cp): # Dictionary of data to fill in for GlueLocationLocalID info = { 'locationId': 'OSG_GRID', 'subClusterId': subClusterId, 'clusterId': cluster_id, 'locationName': 'OSG_GRID', 'version': 1.0, 'path': osg_grid, } # Spit out our template, fill it with the appropriate info. printTemplate(template, info) except Exception, e: # Log error, then report it via stderr. log.error(e) sys.stdout = sys.stderr raise
def _getJobsInfoInternal(cp): """ The "alternate" way of building the jobs info; this allows for sites to filter jobs based upon an arbitrary condor_q constraint. This is not the default as large sites can have particularly bad performance for condor_q. """ global _results_cache if _results_cache: return dict(_results_cache) constraint = cp_get(cp, "condor", "jobs_constraint", "TRUE") fp = condorCommand(condor_job_status, cp, {'constraint': constraint}) handler = ClassAdParser('GlobalJobId', ['JobStatus', 'Owner', 'AccountingGroup', 'FlockFrom']); submitConstraint = _createSubmitterConstraint(cp) fp2 = condorCommand(condor_status_submitter, cp, {'constraint': submitConstraint}) handler2 = ClassAdParser('Name', ['MaxJobsRunning']) try: xmlLine = '' while xmlLine.find('<?xml') == -1: # Throw away junk lines from Condor < 7.3.2 xmlLine = fp.readline() condorXml = cStringIO.StringIO(xmlLine+fp.read()) parseCondorXml(condorXml, handler) except Exception, e: log.error("Unable to parse condor output!") log.exception(e) if 'GIP_TESTING' in os.environ: raise RuntimeError('Could not parse condor_q -xml output!') return {}
def getName(self): """ Return the name of the SE. The base class uses the value of se.name in the configuration object. """ return cp_get(self._cp, self._section, 'name', 'UNKNOWN')
def getAccessProtocols(self): """ Stub function for providing access protocol information. Return a list of dictionaries. Each dictionary will have the following keys with reference to an access endpoint: - protocol - hostname - port Optionally, the following keys may be included (default in parenthesis): - capability (file transfer) - maxStreams (1) - securityinfo (none) - version (UNKNOWN) - endpoint (<protocol>://<hostname>:<port>) For the base class, this just returns []. """ mount_point = cp_get(self._cp, self._section, 'mount_point', None) if None: return [] return [{'protocol': 'file', 'hostname': 'POSIX.example.com', 'port': '1234', 'version': '1.0.0'}]
def run(self): endpoint = cp_get(self._cp, self._section, "infoProviderEndpoint", "") try: self.handler = parse_fp(urllib2.urlopen(endpoint)) except Exception, e: log.exception(e) self.handler = None
def parseNodes(cp): """ Parse the condor nodes. @param cp: ConfigParser object for the GIP @returns: A tuple consisting of the total, claimed, and unclaimed nodes. """ global _nodes_cache #pylint: disable-msg=W0603 if _nodes_cache: return _nodes_cache subtract = cp_getBoolean(cp, "condor", "subtract_owner", True) log.debug("Parsing condor nodes.") constraint = cp_get(cp, "condor", "status_constraint", "TRUE") fp = condorCommand(condor_status, cp, {'constraint': constraint}) handler = ClassAdParser('Name', ['State']) parseCondorXml(fp, handler) total = 0 claimed = 0 unclaimed = 0 for info in handler.getClassAds().values(): total += 1 if 'State' not in info: continue if info['State'] == 'Claimed': claimed += 1 elif info['State'] == 'Unclaimed': unclaimed += 1 elif subtract and info['State'] == 'Owner': total -= 1 log.info("There are %i total; %i claimed and %i unclaimed." % \ (total, claimed, unclaimed)) _nodes_cache = total, claimed, unclaimed return total, claimed, unclaimed
def getJobsInfo(vo_map, cp): """ Retrieve information about the jobs in the Condor system. Query condor about the submitter status. The returned job information is a dictionary whose keys are the VO name of the submitting user and values the aggregate information about that VO's activities. The information is another dictionary showing the running, idle, held, and max_running jobs for that VO. @param vo_map: A vo_map object mapping users to VOs @param cp: A ConfigParser object with the GIP config information. @returns: A dictionary containing job information. """ group_jobs = {} queue_constraint = cp_get(cp, "condor", "jobs_constraint", "TRUE") if queue_constraint.upper() == 'TRUE': submitConstraint = _createSubmitterConstraint(cp) fp = condorCommand(condor_status_submitter, cp, {'constraint': submitConstraint}) handler = ClassAdParser(('Name', 'ScheddName'), ['RunningJobs', 'IdleJobs', 'HeldJobs', 'MaxJobsRunning', 'FlockedJobs']) try: parseCondorXml(fp, handler) except Exception, e: log.error("Unable to parse condor output!") log.exception(e) results = handler.getClassAds()
def buildServiceID(cp): ceName = cp_get(cp, "ce", "name", "UNKNOWN_CE") gliteSuffix = 'org.glite.ce.CREAM' uniqueHash = getUniqueHash(cp) serviceID = '%s_%s_%s' % (ceName, gliteSuffix, uniqueHash) return serviceID
def main(): try: # Load up the site configuration cp = config() se_only = cp_getBoolean(cp, "gip", "se_only", False) if not se_only: # Load up the template for GlueLocationLocalID # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster template = getTemplate("GlueCluster", "GlueLocationLocalID") cluster_id = getClusterID(cp) osg_grid = cp_get(cp, "osg_dirs", "grid_dir", None) if not osg_grid: raise RuntimeError('grid_dir ($OSG_GRID) not defined!') for subClusterId in getSubClusterIDs(cp): # Dictionary of data to fill in for GlueLocationLocalID info = {'locationId': 'OSG_GRID', 'subClusterId': subClusterId, 'clusterId': cluster_id, 'locationName': 'OSG_GRID', 'version': 1.0, 'path': osg_grid, } # Spit out our template, fill it with the appropriate info. printTemplate(template, info) except Exception, e: # Log error, then report it via stderr. log.error(e) sys.stdout = sys.stderr raise
def getCEImpl(cp): ceImpl = 'Globus' ceImplVersion = cp_get(cp, ce, 'globus_version', '4.0.6') if cp_getBoolean(cp, 'cream', 'enabled', False): ceImpl = 'CREAM' ceImplVersion = getOSGVersion(cp) return (ceImpl, ceImplVersion)
def getApplicationsV1(cp): """ Retrieves the applications in the new "v1" format; it looks in - $OSG_APP/etc/grid3-locations.txt - $OSG_APP/etc/<vo>/locations-v1.txt Here, $OSG_APP/etc may be overridden by the config variable gip.software_dir. One directory is created per VO, owned by root, world-writable, and set to sticky. The VO names will be determined by the current methods (using voList). GIP will then scan the directory $OSG_APP/etc/<vo> for all the VO names known to GIP, and read only a file named locations-v1.txt. It is assumed that locations-v1.txt is the same format as the grid3-locations.txt. """ app_dir = cp_get(cp, "osg_dirs", "app", "/UNKNOWN") base_path = os.path.join(app_dir, "etc") base_path = cp_get(cp, "gip", "software_dir", base_path) locations = {} for vo in voList(cp): vo_dir = os.path.join(base_path, vo) vo_dir = os.path.expandvars(vo_dir) vo_path = os.path.join(vo_dir, 'locations-v1.txt') if not os.path.exists(vo_path): continue try: fp = open(vo_path, 'r') except: log.warning("Unable to read VO application file: %s" % vo_path) continue for line in fp: line = line.strip() info = line.split() # Skip blank lines and comments if len(line) == 0 or info[0].startswith('#'): continue if len(info) != 3: log.warning("Invalid line: %s" % line) continue if info[1].startswith('#') or info[1].startswith('$'): info[1] = 'UNDEFINED' info = {'locationName': info[0], 'version': info[1], 'path':info[2]} info['locationId'] = info['locationName'] locations = _addLocation(locations, info) return locations
def print_SE(se, cp): """ Emit the GLUE entities for the SE, based upon the StorageElement class. """ # if the unique ID is UNKNOWN, a real SE does not exist, the classic SE # will probably be invoked seUniqueID = se.getUniqueID() if seUniqueID == "UNKNOWN" or seUniqueID == "UNAVAILABLE": return status = se.getStatus() version = se.getVersion() # Determine space information try: used, available, total = se.getSESpace(total=True, gb=True) except: used, available, total = 0, 0, 0 # Tape information, if we have it... nu, _, nt = se.getSETape() siteUniqueID = cp.get("site", "unique_name") implementation = se.getImplementation() # Try to guess the appropriate architecture arch = se.getSEArch() # port number was hard coded to 8443, get from cp now # NOTE: this field is deprecated by the schema so it should not be used port = se.getPort() # Fill in the information for the template info = { 'seName': se.getName(), 'seUniqueID': se.getUniqueID(), 'implementation': implementation, "version": version, "status": status, "port": port, "onlineTotal": total, "nearlineTotal": nt, "onlineUsed": used, "nearlineUsed": nu, "architecture": arch, "free": available, "total": total, "bdii": cp_get(cp, "bdii", "endpoint", "Unknown"), "siteUniqueID": siteUniqueID, "arch": arch, } seTemplate = getTemplate("GlueSE", "GlueSEUniqueID") log.info(str(info)) printTemplate(seTemplate, info) try: print_SA(se, cp, se.getSection()) except Exception, e: log.exception(e)
def setConfigValues(self): self.results_dir = cp_get(self.cp, "gip_tests", "results_dir", "UNKNOWN") if self.results_dir == "UNKNOWN": raise ConfigurationError("Results directory is not configured") self.oim_xml_dir = cp_get(self.cp, "gip_tests", "myosg_xml_dir", "UNKNOWN") if self.oim_xml_dir == "UNKNOWN": raise ConfigurationError("OIM XML directory is not configured") self.oim_summary_xml_file = "%s/%s" % ( self.oim_xml_dir, cp_get(self.cp, "gip_tests", "myosg_summary_file", "myosg.xml")) self.oim_detail_file_template = self.oim_xml_dir + "/" + cp_get( self.cp, "gip_tests", "myosg_detail_file_template", "myosg_%s_detail.xml") self.oim_tests = cp_get(self.cp, "gip_tests", "myosg_tests", self.oim_tests)
def getClassicSEList(cp): """ Return a list of all the ClassicSE's at this site @param cp: Site configuration @returns: List of all the ClassicSE's unique_ids """ classic_list = [] if cp_getBoolean(cp, "classic_se", "advertise_se", True): classicSE = cp_get(cp, "classic_se", "unique_name", None) if classicSE: classic_list = [classicSE] else: siteUniqueID = cp_get(cp, "site", "unique_name", "UNKNOWN") classicSE = siteUniqueID + "_classicSE" classic_list = [classicSE] return classic_list
def print_VOInfo(se, cp): """ Print out the VOInfo GLUE information for all the VOInfo objects in the SE. This will optionally alter the VOInfo object to limit the total available space for a given VO. """ vo_limit_str = cp_get(cp, "se", "vo_limits", "") vo_limit = {} cumulative_total = {} for vo_str in vo_limit_str.split(','): vo_str = vo_str.strip() info = vo_str.split(":") if len(info) != 2: continue vo = info[0].strip() try: limit = float(info[1].strip()) except: continue vo_limit[vo] = limit cumulative_total.setdefault(vo, 0) for voinfo in se.getVOInfos(): do_continue = False reduce_by_amount = 0 try: totalOnline = float(voinfo.get("totalOnline", 0)) except: continue for vo in get_vos_from_acbr(voinfo.get("acbr", "")): if vo in vo_limit: try: cumulative_total[vo] += totalOnline except: pass reduce_by_amount = max( reduce_by_amount, cumulative_total.get(vo, 0) - vo_limit[vo]) if reduce_by_amount > totalOnline: continue elif reduce_by_amount > 0: voinfo['totalOnline'] = str(totalOnline - reduce_by_amount) try: voinfo['availableSpace'] = str( max( 0, float(voinfo['availableSpace']) - reduce_by_amount * 1024**2)) voinfo['freeOnline'] = str(max(0, float(voinfo['freeOnline']) \ - reduce_by_amount)) except: pass try: print_single_VOInfo(voinfo, se, cp) except Exception, e: log.exception(e)
def doPath(cp): # add condor binaries to system path condor_path = cp_get(cp, "condor", "condor_path", None) condor_location = cp_get(cp, "condor", "condor_location", None) condor_config = cp_get(cp, "condor", "condor_config", None) if isDefined(condor_path): addToPath(condor_path) if isDefined(condor_location): log.info("Adding %s/bin to path" % condor_location) addToPath('%s/bin' % condor_location) if not isDefined(condor_config): condor_config = '%s/etc/condor_config' % condor_location if isDefined(condor_config): os.environ['CONDOR_CONFIG'] = condor_config
def getClusterName(cp): """ Return the name of the associated cluster. """ ce_name = cp_get(cp, ce, 'name', "UNKNOWN_CE") simple = cp.getboolean(cluster, 'simple') if simple: return ce_name else: return cp.get(cluster, 'name')