def bootstrapSGE(cp):
"""
If it exists, source
$SGE_ROOT/$SGE_CELL/common/settings.sh
"""
sge_root = cp_get(cp, "sge", "sge_root", "")
if not sge_root:
log.warning("Could not locate sge_root in config file! Not " \
"bootstrapping SGE environment.")
return
sge_cell = cp_get(cp, "sge", "sge_cell", "")
if not sge_cell:
log.warning("Could not locate sge_cell in config file! Not " \
"bootstrapping SGE environment.")
return
settings = os.path.join(sge_root, sge_cell, "common/settings.sh")
if not os.path.exists(settings):
log.warning("Could not find the SGE settings file; looked in %s" % \
settings)
return
cmd = "/bin/sh -c 'source %s; /usr/bin/env'" % settings
fd = os.popen(cmd)
results = fd.read()
if fd.close():
log.warning("Unable to source the SGE settings file; tried %s." % \
settings)
for line in results.splitlines():
line = line.strip()
info = line.split('=', 2)
if len(info) != 2:
continue
os.environ[info[0]] = info[1]
def main():
"""
Main wrapper for the Condor batch system GIP information.
"""
try:
cp = config()
condor_path = cp_get(cp, "condor", "condor_path", None)
condor_location = cp_get(cp, "condor", "condor_location", None)
condor_config = cp_get(cp, "condor", "condor_config", None)
if condor_path != None:
addToPath(condor_path)
if condor_location != None:
addToPath('%s/bin' % condor_location)
if not condor_config:
condor_config = '%s/etc/condor_config' % condor_location
if condor_config:
os.environ['CONDOR_CONFIG'] = condor_config
#vo_map = VoMapper(cp)
getLrmsInfo(cp)
print_CE(cp)
print_VOViewLocal(cp)
except Exception, e:
log.exception(e)
raise
def print_classic_access(cp, siteUniqueID):
"""
Emit the GLUE entity for a classic SE's access protocol.
"""
fallback_name = siteUniqueID + "_classicSE"
seUniqueID = cp_get(cp, "classic_se", "unique_name", fallback_name)
try:
default_host = socket.gethostname()
except:
default_host = 'UNKNOWN.example.org'
host = cp_get(cp, "classic_se", "host", default_host)
port = cp_getInt(cp, "classic_se", "port", "2811")
accessTemplate = getTemplate("GlueSE", "GlueSEAccessProtocolLocalID")
endpoint = 'gsiftp://%s:%i' % (host, port)
info = {
'accessProtocolID': 'GFTP_%s_%i' % (host, port),
'seUniqueID': seUniqueID,
'protocol': 'gsiftp',
'endpoint': endpoint,
'capability': 'file transfer',
'maxStreams': 10,
'security': 'gsiftp',
'port': port,
'version': '1.0.0',
}
print accessTemplate % info
def getHTCondorCEVersion(cp):
"""
Returns the running version of the HTCondor CE
Copied from getOSGVersion() in gip_cluster.py
"""
log = logging.getLogger()
htcondorce_ver_backup = cp_get(cp, "ce", "htcondorce_version", "1.8")
htcondorce_version_script = cp_get(cp, "gip", "htcondorce_version_script",
"")
htcondorce_ver = ''
if len(htcondorce_version_script) == 0:
htcondorce_version_script = vdtDir('$VDT_LOCATION/condor_ce_config_val',
'/usr/bin/condor_ce_config_val')
htcondorce_version_script = os.path.expandvars(htcondorce_version_script)
if not os.path.exists(htcondorce_version_script):
htcondorce_version_script = os.path.expandvars("$VDT_LOCATION/osg/bin/" \
"osg-version")
if os.path.exists(htcondorce_version_script):
try:
htcondorce_version_script += " HTCondorCEVersion"
htcondorce_ver = runCommand(htcondorce_version_script).read().strip()
htcondorce_ver = htcondorce_ver.replace('"','')
except Exception, e:
log.exception(e)
def handle_SE(cp, section):
"""
Run a provider for one SE.
"""
# if you only have a classic SE, there will still be a [se] section
# with the default_se option. This will result in a GlueService
# stanza being written with garbage information in it. So, check
# for default_se = "UNKNOWN" or "UNAVAILABLE" and return if it does
#
# default_se is set in the [Storage] section in config.ini and is
# required by configure-osg.py
default_se = cp_get(cp, "se", "default_se", "UNKNOWN")
if default_se == "UNAVAILABLE" or default_se == "UNKNOWN": return
impl = cp_get(cp, section, "implementation", "UNKNOWN")
provider_impl = cp_get(cp, section, "provider_implementation", "UNKNOWN")
if provider_impl == "UNKNOWN" and not cp_getBoolean(
cp, section, "dynamic_dcache", False):
provider_impl = 'static'
se_class, cp = determine_provider(provider_impl, impl, cp)
se = se_class(cp, section=section)
log.info("Outputting SE %s; implementation %s; provider %s" %
(se.getName(), impl, provider_impl))
try:
se.run()
except Exception, e:
log.exception(e)
def print_VOViewLocal(cp):
ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
vo_map = VoMapper(cp)
queue_jobs = getJobsInfo(vo_map, cp)
vo_queues = getVoQueues(cp)
VOView = getTemplate("GlueCE", "GlueVOViewLocalID")
for vo, queue in vo_queues:
ce_unique_id = buildCEUniqueID(cp, ce_name, 'sge', queue)
info = {
'ceUniqueID' : ce_unique_id,
'voLocalID' : vo,
'acbr' : 'VO:%s' % vo,
'running' : queue_jobs.get(queue, {}).get(vo, {}).\
get('running', 0),
'waiting' : queue_jobs.get(queue, {}).get(vo, {}).\
get('waiting', 0),
#'free_slots' : vo.get(queue, {}).get('free_slots', 0),
'free_slots' : 0, #TODO: fix
'ert' : 3600,
'wrt' : 3600,
'default_se' : getDefaultSE(cp),
'app' : cp_get(cp, "osg_dirs", "app", "/OSG_APP_UNKNOWN"),
'data' : cp_get(cp, "osg_dirs", "data", "/OSG_DATA_UNKNOWN"),
}
info['total'] = info['waiting'] + info['running']
printTemplate(VOView, info)
def interpolateConfig(cp):
grid = cp_get(cp, "site", "group", "")
if cp_getBoolean(cp, "gip_tests", "oim_aware", False):
sitelist_cmd = "wget -O - http://oim.grid.iu.edu/pub/resource/show.php?format=plain-text 2>/dev/null | grep \",%s,\" | grep \",CE\" | cut -f1 -d," % grid
sitelist = runCommand(sitelist_cmd).read().split()
sitelist = ",".join(sitelist)
cp.set("gip_tests", "site_names", sitelist)
else:
if cp_get(cp, "gip_tests", "site_names", "") == "":
cp.set("gip_tests", "site_names", cp_get(cp, "site", "name", ""))
if cp_get(cp, "gip_tests", "site_dns", "") == "":
host_parts = cp_get(cp, "ce", "name", "").split('.')
site_dns = "%s.%s" % (host_parts[:-2], host_parts[:-1])
cp.set("gip_tests", "site_dns", site_dns)
if cp_get(cp, "gip_tests", "required_site", "") == "":
cp.set("gip_tests", "required_sites",
cp_get(cp, "gip_tests", "site_names", ""))
cp.set("gip_tests", "bdii_port", "2170")
cp.set("gip_tests", "egee_port", "2170")
cp.set(
"gip_tests", "interop_url",
"http://oim.grid.iu.edu/publisher/get_osg_interop_bdii_ldap_list.php?grid=%s&format=html"
% grid)
if strContains(grid, "ITB"):
cp.set("bdii", "endpoint", "ldap://is-itb.grid.iu.edu:2170")
cp.set("gip_tests", "bdii_addr", "is-itb.grid.iu.edu")
cp.set("gip_tests", "egee_bdii", "pps-bdii.cern.ch")
cp.set(
"gip_tests", "egee_bdii_conf_url",
"http://egee-pre-production-service.web.cern.ch/egee-pre-production-service/bdii/pps-all-sites.conf"
)
web_server = "http://is-itb.grid.iu.edu"
else:
cp.set("gip_tests", "bdii_addr", "is.grid.iu.edu")
cp.set("gip_tests", "egee_bdii", "lcg-bdii.cern.ch")
cp.set("gip_tests", "egee_bdii_conf_url",
"http://lcg-bdii-conf.cern.ch/bdii-conf/bdii.conf")
web_server = "http://is.grid.iu.edu"
cp.set("gip_tests", "update_url", web_server + "/cgi-bin/status.cgi")
cp.set("gip_tests", "schema_check_url",
web_server + "/data/cemon_processed_osg/%s.processed?which=%s")
cp.set("gip_tests", "validator_url",
web_server + "/data/cemon_processed_osg/%s.processed?which=%s")
if cp_get(cp, "gip_tests", "compare_excludes", "") == "":
compare_excludes = "GlueCEStateFreeJobSlots,GlueCEStateRunningJobs,GlueCEStateTotalJobs,GlueSiteLocation,GlueSAStateAvailableSpace,GlueSAStateUsedSpace"
cp.set("gip_tests", "compare_excludes", compare_excludes)
if cp_get(cp, "gip_tests", "enable_glite", "") == "":
cp.set("gip_tests", "enable_glite", "False")
if cp_get(cp, "gip_tests", "results_dir", "") == "":
results_dir = os.path.expandvars("$GIP_LOCATION/../apache/htdocs/")
cp.set("gip_tests", "results_dir", results_dir)
def getDefaultSE(cp):
global _defaultSE
if _defaultSE:
return _defaultSE
default_se = cp_get(cp, "se", "name", "UNKNOWN")
# if [se] name: ??? is "UNAVAILABLE" or not set, then try to get the
# default_se
if default_se == "UNKNOWN" or default_se == "UNAVAILABLE":
default_se = cp_get(cp, "se", "default_se", "UNAVAILABLE")
# if it is still UNAVAILABLE or not set, check to see if the classic SE
# is being advertised and use that
if default_se == "UNAVAILABLE" and cp_getBoolean(cp, "classic_se",
"advertise_se", True):
fallback_name = cp_get(cp, "site", "unique_name", "UNKNOWN") + \
"_classicSE"
default_se = cp_get(cp, "classic_se", "name", fallback_name)
current_se = None
for sect in cp.sections():
if not sect.lower().startswith('se'):
continue
try:
current_se = cp.get(sect, 'name')
except:
continue
if cp_getBoolean(cp, sect, "default", False):
_defaultSE = current_se
return current_se
if default_se == 'UNKNOWN' and current_se:
_defaultSE = current_se
return current_se
_defaultSE = default_se
return default_se
def getWLCGInfo(cp):
"""
Generate GRID, WLCG_TIER, WLCG_PARENT, and WLCG_NAME if appropriate.
"""
grid = cp_get(cp, "site", "wlcg_grid", "OSG").strip()
tier = cp_get(cp, "site", "wlcg_tier", "").strip()
parent = cp_get(cp, "site", "wlcg_parent", "").strip()
name = cp_get(cp, "site", "wlcg_name", "").strip()
log.debug('WLCG attributes: wlcg_grid, wlcg_tier, wlcg_parent, wlcg_name: ("%s", "%s", "%s", "%s")' %
(grid, tier, parent, name))
otherInfoTemplate = 'GlueSiteOtherInfo: %s\n'
if grid and tier and parent and name:
otherInfo = otherInfoTemplate % ('GRID=WLCG')
otherInfo += otherInfoTemplate % ('GRID=%s' % grid)
otherInfo += otherInfoTemplate % ('WLCG_TIER=%s' % tier)
otherInfo += otherInfoTemplate % ('WLCG_PARENT=%s' % parent)
otherInfo += otherInfoTemplate % ('WLCG_NAME=%s' % name)
else:
log.debug('At least one null WLCG attribute - not publishing WLCG-style GlueSiteOtherInfo')
otherInfo = ''
return otherInfo
def handle_SE(cp, section):
"""
Run a provider for one SE.
"""
# if you only have a classic SE, there will still be a [se] section
# with the default_se option. This will result in a GlueService
# stanza being written with garbage information in it. So, check
# for default_se = "UNKNOWN" or "UNAVAILABLE" and return if it does
#
# default_se is set in the [Storage] section in config.ini and is
# required by configure-osg.py
default_se = cp_get(cp, "se", "default_se", "UNKNOWN")
if default_se == "UNAVAILABLE" or default_se == "UNKNOWN": return
impl = cp_get(cp, section, "implementation", "UNKNOWN")
provider_impl = cp_get(cp, section, "provider_implementation", "UNKNOWN")
if provider_impl == "UNKNOWN" and not cp_getBoolean(cp, section,
"dynamic_dcache", False):
provider_impl = 'static'
se_class, cp = determine_provider(provider_impl, impl, cp)
se = se_class(cp, section=section)
log.info("Outputting SE %s; implementation %s; provider %s" % (se.getName(),
impl, provider_impl))
try:
se.run()
except Exception, e:
log.exception(e)
def getWLCGInfo(cp):
"""
Generate GRID, WLCG_TIER, WLCG_PARENT, and WLCG_NAME if appropriate.
"""
grid = cp_get(cp, "site", "wlcg_grid", "OSG").strip()
tier = cp_get(cp, "site", "wlcg_tier", "").strip()
parent = cp_get(cp, "site", "wlcg_parent", "").strip()
name = cp_get(cp, "site", "wlcg_name", "").strip()
log.debug(
'WLCG attributes: wlcg_grid, wlcg_tier, wlcg_parent, wlcg_name: ("%s", "%s", "%s", "%s")'
% (grid, tier, parent, name))
otherInfoTemplate = 'GlueSiteOtherInfo: %s\n'
if grid and tier and parent and name:
otherInfo = otherInfoTemplate % ('GRID=WLCG')
otherInfo += otherInfoTemplate % ('GRID=%s' % grid)
otherInfo += otherInfoTemplate % ('WLCG_TIER=%s' % tier)
otherInfo += otherInfoTemplate % ('WLCG_PARENT=%s' % parent)
otherInfo += otherInfoTemplate % ('WLCG_NAME=%s' % name)
else:
log.debug(
'At least one null WLCG attribute - not publishing WLCG-style GlueSiteOtherInfo'
)
otherInfo = ''
return otherInfo
def print_classic_access(cp, siteUniqueID):
"""
Emit the GLUE entity for a classic SE's access protocol.
"""
fallback_name = siteUniqueID + "_classicSE"
seUniqueID = cp_get(cp, "classic_se", "unique_name", fallback_name)
try:
default_host = socket.gethostname()
except:
default_host = 'UNKNOWN.example.org'
host = cp_get(cp, "classic_se", "host", default_host)
port = cp_getInt(cp, "classic_se", "port", "2811")
accessTemplate = getTemplate("GlueSE", "GlueSEAccessProtocolLocalID")
endpoint = 'gsiftp://%s:%i' % (host, port)
info = {'accessProtocolID' : 'GFTP_%s_%i' % (host, port),
'seUniqueID' : seUniqueID,
'protocol' : 'gsiftp',
'endpoint' : endpoint,
'capability' : 'file transfer',
'maxStreams' : 10,
'security' : 'gsiftp',
'port' : port,
'version' : '1.0.0',
}
print accessTemplate % info
def __init__(self, args):
self.cp = getTestConfig(args)
self.args = " ".join(args)
# these are the test lists by category
self.reports = []
self.critical_tests = []
self.glite_tests = []
# output files
self.output_files = []
# command line values
self.output_dir = cp_get(
self.cp, "gip_tests", "results_dir",
os.path.expandvars("$GIP_LOCATION/reporting/results"))
self.source_cmd = "source %s/setup.sh" % os.path.expandvars(
"$VDT_LOCATION")
self.extList = [".xsl"]
# set up the critical test list
default_critical_tests = "Interop_Reporting_Check,Missing_Sites,Validate_GIP_BDII,Validate_GIP_URL"
crit_conf = cp_get(self.cp, "gip_tests", "critical_tests",
default_critical_tests)
self.crit = [i.strip() for i in crit_conf.split(',')]
def print_VOViewLocal(cp):
ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
vo_map = VoMapper(cp)
queue_jobs = getJobsInfo(vo_map, cp)
vo_queues = getVoQueues(cp)
VOView = getTemplate("GlueCE", "GlueVOViewLocalID")
for vo, queue in vo_queues:
ce_unique_id = buildCEUniqueID(cp, ce_name, 'sge', queue)
info = {
'ceUniqueID' : ce_unique_id,
'voLocalID' : vo,
'acbr' : 'VO:%s' % vo,
'running' : queue_jobs.get(queue, {}).get(vo, {}).\
get('running', 0),
'waiting' : queue_jobs.get(queue, {}).get(vo, {}).\
get('waiting', 0),
#'free_slots' : vo.get(queue, {}).get('free_slots', 0),
'free_slots' : 0, #TODO: fix
'ert' : 3600,
'wrt' : 3600,
'default_se' : getDefaultSE(cp),
'app' : cp_get(cp, "osg_dirs", "app", "/OSG_APP_UNKNOWN"),
'data' : cp_get(cp, "osg_dirs", "data", "/OSG_DATA_UNKNOWN"),
}
info['total'] = info['waiting'] + info['running']
printTemplate(VOView, info)
def bootstrapSGE(cp):
"""
If it exists, source
$SGE_ROOT/$SGE_CELL/common/settings.sh
"""
sge_root = cp_get(cp, "sge", "sge_root", "")
if not sge_root:
log.warning("Could not locate sge_root in config file! Not " \
"bootstrapping SGE environment.")
return
sge_cell = cp_get(cp, "sge", "sge_cell", "")
if not sge_cell:
log.warning("Could not locate sge_cell in config file! Not " \
"bootstrapping SGE environment.")
return
settings = os.path.join(sge_root, sge_cell, "common/settings.sh")
if not os.path.exists(settings):
log.warning("Could not find the SGE settings file; looked in %s" % \
settings)
return
cmd = "/bin/sh -c 'source %s; /usr/bin/env'" % settings
fd = os.popen(cmd)
results = fd.read()
if fd.close():
log.warning("Unable to source the SGE settings file; tried %s." % \
settings)
for line in results.splitlines():
line = line.strip()
info = line.split('=', 2)
if len(info) != 2:
continue
os.environ[info[0]] = info[1]
def setConfigValues(self):
self.results_dir = cp_get(self.cp, "gip_tests", "results_dir", "UNKNOWN")
if self.results_dir == "UNKNOWN": raise ConfigurationError("Results directory is not configured")
self.oim_xml_dir = cp_get(self.cp, "gip_tests", "myosg_xml_dir", "UNKNOWN")
if self.oim_xml_dir == "UNKNOWN": raise ConfigurationError("OIM XML directory is not configured")
self.oim_summary_xml_file = "%s/%s" % (self.oim_xml_dir, cp_get(self.cp, "gip_tests", "myosg_summary_file", "myosg.xml"))
self.oim_detail_file_template = self.oim_xml_dir + "/" + cp_get(self.cp, "gip_tests", "myosg_detail_file_template", "myosg_%s_detail.xml")
self.oim_tests = cp_get(self.cp, "gip_tests", "myosg_tests", self.oim_tests)
def publish_gums(cp, template):
hostname = cp_get(cp, "ce", 'name', gethostname())
siteID = cp_get(cp, "site", "unique_name", gethostname())
gumsConfig = vdtDir(os.path.expandvars('$VDT_LOCATION/gums/config/gums-client.properties'),
'/etc/gums/gums-client.properties')
gumsConfigFile = open(gumsConfig, 'r')
gums_re = re.compile('gums.authz\s*=\s*(https://(.*):.*?/(.*))')
lines = gumsConfigFile.readlines()
for line in lines:
m = gums_re.match(line)
if m: (gums_uri, gums_host) = m.groups()[0:2]
os.putenv('X509_USER_CERT', '/etc/grid-security/http/httpcert.pem')
os.putenv('X509_USER_KEY' , '/etc/grid-security/http/httpkey.pem')
mapping_subject_dn = '/GIP-GUMS-Probe-Identity'
mapping_subject_name = '`grid-cert-info -subject` '
gums_command = vdtDir(os.path.expandvars('$VDT_LOCATION/gums/scripts/gums-service'),
'/usr/bin/gums-service')
gums_command += ' mapUser -s ' + mapping_subject_name + mapping_subject_dn
(gums_output, pin) = popen2.popen4(gums_command)
gums_id_re = re.compile('.*\[userName: (.*)\].*')
status = "Warning"
statusInfo = "Test mapping failed: if GUMS was not down, check logs" +\
" at " + gums_host + ':' + '$VDT_LOCATION/tomcat/v55/logs'
lines = gums_output.readlines()
for line in lines:
m = gums_id_re.match(line)
if m:
uidMapping = m.groups([0])
status = "OK"
statusInfo = "Test mapping successful: user id = %s" % uidMapping
break
info = {'serviceID': gums_uri,
'serviceType': 'GUMS',
'serviceName': 'Authorization',
'version': 'UNDEFINED',
'endpoint': gums_uri,
'semantics': 'UNDEFINED',
'owner': '',
'url': gums_uri,
'uri': gums_uri,
'status': status,
'statusInfo': statusInfo,
'wsdl': 'Not Applicable',
'startTime': 'Not Applicable',
'siteID': siteID,
'acbr': '__GIP_DELETEME'
}
printTemplate(template, info)
def main():
log.info('Starting CREAM service provider')
try:
cp = config()
serviceID = buildServiceID(cp)
siteID = cp_get(cp, "site", "unique_name", 'UNKNOWN_SITE')
serviceName = '%s-CREAM' % siteID
creamVersion = getCreamVersion(cp)
endpoint = 'https://%s:8443/ce-cream/services' % cp_get(cp, "ce", "name", 'UNKNOWN_CE')
allVOs = voList(cp)
acbr = ''
owner = ''
log.debug('CREAM VOs are %s' % allVOs)
if not allVOs:
log.error("No VOs supported!")
acbr = '__GIP_DELETEME'
else:
acbr = '\n'.join(['GlueServiceAccessControlBaseRule: %s\n' \
'GlueServiceAccessControlBaseRule: VO:%s' % (vo, vo) for vo in allVOs])
owner = '\n' + '\n'.join(['GlueServiceOwner: %s' % vo for vo in allVOs]) # owner needs an extra prepended newline
pid = -1
startTime = 'Not Applicable'
serviceStatus = 'Not OK'
serviceStatusInfo = 'Could not find tomcat process'
try:
(startTime, pid) = getStartTimeAndPid(cp)
serviceStatus = 'OK'
serviceStatusInfo = 'Tomcat (%d) is running' % pid
except:
log.error('Could not locate tomcat process (pgrep -f "org.apache.catalina.startup.Bootstrap start"'
' probably failed to return any output!)')
info = {'serviceID': serviceID,
'serviceType': 'org.glite.ce.CREAM',
'serviceName': serviceName,
'version': creamVersion,
'endpoint': endpoint,
'semantics': 'https://edms.cern.ch/document/595770',
'owner': owner,
'url': '__GIP_DELETEME', # deprecated
'uri': '__GIP_DELETEME', # deprecated
'status': serviceStatus,
'statusInfo': serviceStatusInfo,
'wsdl': 'http://grid.pd.infn.it/cream/wsdl/org.glite.ce-cream_service.wsdl',
'startTime': startTime,
'siteID': siteID,
'acbr': acbr
}
template = getTemplate("GlueService", "GlueServiceUniqueID")
printTemplate(template, info)
except Exception, e:
sys.stdout = sys.stderr
log.error(e)
raise
def getApplications(cp):
"""
Return a list of dictionaries containing the application info for each
application installed at this site.
Each returned dictionary should have the following keys:
- locationId
- locationName
- version
- path
@param cp: Site configuration
@return: List of dictionaries; each dictionary contains the information
about a specific installed application.
"""
app_dir = cp_get(cp, "osg_dirs", "app", "/UNKNOWN")
path1 = '%s/etc/grid3-locations.txt' % app_dir
path2 = '%s/etc/osg-locations.txt' % app_dir
paths = [path1, path2]
path3 = cp_get(cp, "ce", "app_list", '')
if path3:
paths.append(path3)
locations = {}
for path in paths:
try:
fp = open(path, 'r')
except:
continue
for line in fp:
line = line.strip()
info = line.split()
# Skip blank lines and comments
if len(line) == 0 or info[0].startswith('#'):
continue
if len(info) != 3:
log.warning("Invalid line: %s" % line)
continue
if info[1].startswith('#') or info[1].startswith('$'):
info[1] = 'UNDEFINED'
info = {'locationName': info[0], 'version': info[1], 'path':info[2]}
info['locationId'] = info['locationName']
locations = _addLocation(locations, info)
osg_ver = getOSGVersion(cp)
if osg_ver:
# old method: will deprecate in a future version, but leave
# for backwards compatibility
info = {'locationId': osg_ver, 'locationName': osg_ver, 'version': \
osg_ver, 'path': os.environ.get('VDT_LOCATION', '/UNKNOWN')}
locations = _addLocation(locations, info)
# new method
info = {'locationId': 'OSG_VERSION', 'locationName': 'OSG_VERSION',
'version': osg_ver, 'path': os.environ.get('VDT_LOCATION', '/UNKNOWN')}
locations = _addLocation(locations, info)
try:
locations.update(getApplicationsV1(cp))
except Exception, e:
log.exception(e)
def _generateSubClusterHelper(cp, section):
"""
Private helper function for generateSubClusters; do not use.
Implementation note: We create a bunch of variables but never
appear to use them - we actually do return the local dictionary
at the end of the function. Just being syntactically lazy.
"""
#pylint: disable-msg=W0612
# Names
subCluster = cp_get(cp, section, "name", cluster)
subClusterUniqueID = cp_get(cp, section, "unique_name", subCluster)
clusterUniqueID = getClusterID(cp)
# Host statistics
clockSpeed = cp_getInt(cp, section, "cpu_speed_mhz", 999999999)
cpuCount = cp_getInt(cp, section, "cpus_per_node", 2)
model = cp_get(cp, section, "cpu_model", 'UNDEFINEDVALUE')
platform = cp_get(cp, section, "platform", "UNKNOWN")
vendor = cp_get(cp, section, "cpu_vendor", 'UNDEFINEDVALUE')
cores_per_cpu = cp_getInt(cp, section, "cores_per_cpu", 2)
si2k = cp_getInt(cp, section, "SI00", 2000)
sf2k = cp_getInt(cp, section, "SF00", 2000)
hepspec = cp_getInt(cp, section, "HEPSPEC", 0)
ram = cp_getInt(cp, section, "ram_size", 1000*cpuCount*cores_per_cpu)
cores = cp_getInt(cp, section, "total_cores", 999999999)
if cores_per_cpu != 0:
cpus = cp_getInt(cp, section, "total_cpus", cores/cores_per_cpu)
else:
cpus = 0
virtualMem = ram + cp_getInt(cp, section, "swap_size", 0)
inboundIP = cp_getBoolean(cp, section, "inbound_network", False)
outboundIP = cp_getBoolean(cp, section, "outbound_network", True)
inboundIP = ldap_boolean(inboundIP)
outboundIP = ldap_boolean(outboundIP)
# OS Stats
osName, osRelease, osVersion = getRelease()
# Temp directories
default_tmp = cp_get(cp, "osg_dirs", "tmp", cp_get(cp, "osg_dirs", "data", \
"/tmp"))
wn_tmp = cp_get(cp, "osg_dirs", "wn_tmp", "/tmp")
tmp = cp_get(cp, section, "tmp", default_tmp)
if notDefined(tmp):
tmp = default_tmp
app_attr = 'GlueHostApplicationSoftwareRunTimeEnvironment'
apps = getApplications(cp)
applications = '\n'.join(['%s: %s' % (app_attr, i['locationId']) for i in \
apps if i['locationId']])
applications += '\n'
# BDII stuff
bdii = cp_get(cp, "bdii", "endpoint", "ldap://is.grid.iu.edu:2170")
return locals()
def getUniqueID(self):
"""
Return the unique ID of the SE.
The base class uses the value of se.unique_name (defaults to se.name)
in the configuraiton object.
"""
return cp_get(self._cp, self._section, 'unique_name',
cp_get(self._cp, self._section, 'name', 'UNKNOWN'))
def getPath(cp, space, vo=None, return_default=True, section='se'):
"""
Return a path appropriate for a VO and space.
Based upon the configuration info and the VO/space requested, determine
a path they should use.
This function tries to find option dcache.space_<space>_path; it parses
this as a comma-separated list of VO:path pairs; i.e.,
space_CMS_path=cms:/dev/null, atlas:/pnfs/blah
If that does not provide a match and return_default is true, then it will
look for dcache.space_<space>_default_path and return that.
If that is not there and return_default is true, it will use the standard
getPath from gip_storage.
If return_default is true, this is guaranteed to return a non-empty string;
if return_default is false, then this might through a ValueError exception.
@param cp: Site config object
@param space: The name of the space to determine the path for.
@param vo: The name of the VO which will be using this space; None for
the default information.
@kw return_default: Set to True if you want this function to return the
default path if it cannot find a VO-specific one.
@returns: A path string; raises a ValueError if return_default=False
"""
default_path = cp_get(cp, section, "space_%s_default_path" % space, None)
if not default_path:
if section == 'se':
default_path = getStoragePath(cp, vo)
else:
default_path = getStoragePath(cp, vo, section=section)
default_path = getStoragePath(cp, vo, section=section)
log.debug("Looking up info for space %s, vo %s, section %s; default " \
"path is %s" % (space, vo, section, default_path))
if not vo:
return default_path
vo_specific_paths = cp_get(cp, section, "space_%s_path" % space, \
None)
vo_specific_path = None
if vo_specific_paths:
for value in vo_specific_paths.split(','):
value = value.strip()
info = value.split(':')
if len(info) != 2:
continue
fqan, path = info
if matchFQAN(vo, fqan):
vo_specific_path = path
break
if not vo_specific_path and return_default:
return default_path
elif not vo_specific_path:
raise ValueError("Unable to determine path for %s!" % vo)
return vo_specific_path
def getPath(cp, space, vo=None, return_default=True, section='se'):
"""
Return a path appropriate for a VO and space.
Based upon the configuration info and the VO/space requested, determine
a path they should use.
This function tries to find option dcache.space_<space>_path; it parses
this as a comma-separated list of VO:path pairs; i.e.,
space_CMS_path=cms:/dev/null, atlas:/pnfs/blah
If that does not provide a match and return_default is true, then it will
look for dcache.space_<space>_default_path and return that.
If that is not there and return_default is true, it will use the standard
getPath from gip_storage.
If return_default is true, this is guaranteed to return a non-empty string;
if return_default is false, then this might through a ValueError exception.
@param cp: Site config object
@param space: The name of the space to determine the path for.
@param vo: The name of the VO which will be using this space; None for
the default information.
@kw return_default: Set to True if you want this function to return the
default path if it cannot find a VO-specific one.
@returns: A path string; raises a ValueError if return_default=False
"""
default_path = cp_get(cp, section, "space_%s_default_path" % space, None)
if not default_path:
if section == 'se':
default_path = getStoragePath(cp, vo)
else:
default_path = getStoragePath(cp, vo, section=section)
default_path = getStoragePath(cp, vo, section=section)
log.debug("Looking up info for space %s, vo %s, section %s; default " \
"path is %s" % (space, vo, section, default_path))
if not vo:
return default_path
vo_specific_paths = cp_get(cp, section, "space_%s_path" % space, \
None)
vo_specific_path = None
if vo_specific_paths:
for value in vo_specific_paths.split(','):
value = value.strip()
info = value.split(':')
if len(info) != 2:
continue
fqan, path = info
if matchFQAN(vo, fqan):
vo_specific_path = path
break
if not vo_specific_path and return_default:
return default_path
elif not vo_specific_path:
raise ValueError("Unable to determine path for %s!" % vo)
return vo_specific_path
def print_clusters(cp):
cluster_name = cp_get(cp, "cluster", "name", None)
if not cluster_name:
cluster_name = cp_get(cp, "ce", "hosting_cluster", None)
if not cluster_name:
cluster_name = cp_get(cp, "ce", "unique_name", None)
if not cluster_name:
getClusterName(cp)
# raise Exception("Could not determine cluster name.")
# clusterUniqueID = cp_get(cp, 'ce', 'unique_name', cluster_name)
clusterUniqueID = getClusterID(cp)
siteUniqueID = cp_get(cp, "site", "unique_name", "UNKNOWN_SITE")
extraCEs = cp_get(cp, "cluster", "other_ces", [])
if extraCEs:
extraCEs = [x.strip() for x in extraCEs.split(",")]
ces = getCEList(cp, extraCEs)
glueClusters = ""
for ce in ces:
glueClusters += "GlueForeignKey: GlueCEUniqueID=%s\n" % ce
bdii = cp_get(cp, "gip", "bdii", "ldap://is.grid.iu.edu:2170")
info = {
"cluster": cluster_name,
"clusterUniqueID": clusterUniqueID,
"tmp": cp_get(cp, "osg_dirs", "tmp", cp_get(cp, "osg_dirs", "data", "/tmp")),
"wn_tmp": cp_get(cp, "osg_dirs", "wn_tmp", "/tmp"),
"siteUniqueID": siteUniqueID,
"glueClusters": glueClusters,
"bdii": bdii,
}
template = getTemplate("GlueCluster", "GlueClusterUniqueID")
printTemplate(template, info)
def print_clusters(cp):
cluster_name = cp_get(cp, 'cluster', 'name', None)
if not cluster_name:
cluster_name = cp_get(cp, 'ce', 'hosting_cluster', None)
if not cluster_name:
cluster_name = cp_get(cp, 'ce', 'unique_name', None)
if not cluster_name:
getClusterName(cp)
#raise Exception("Could not determine cluster name.")
#clusterUniqueID = cp_get(cp, 'ce', 'unique_name', cluster_name)
clusterUniqueID = getClusterID(cp)
siteUniqueID = cp_get(cp, "site", "unique_name", 'UNKNOWN_SITE')
extraCEs = cp_get(cp, 'cluster', 'other_ces', [])
if extraCEs:
extraCEs = [x.strip() for x in extraCEs.split(',')]
ces = getCEList(cp, extraCEs)
glueClusters = ''
for ce in ces:
glueClusters += 'GlueForeignKey: GlueCEUniqueID=%s\n' % ce
bdii = cp_get(cp, 'gip', 'bdii', 'ldap://is.grid.iu.edu:2170')
info = { \
'cluster': cluster_name,
'clusterUniqueID': clusterUniqueID,
'tmp': cp_get(cp, "osg_dirs", "tmp", cp_get(cp, "osg_dirs", "data", \
"/tmp")),
'wn_tmp': cp_get(cp, "osg_dirs", "wn_tmp", "/tmp"),
'siteUniqueID': siteUniqueID,
'glueClusters': glueClusters,
'bdii': bdii,
}
template = getTemplate("GlueCluster", "GlueClusterUniqueID")
printTemplate(template, info)
def getClusterID(cp):
"""
Return the unique ID of the associated cluster.
"""
ce_name = cp_get(cp, ce, 'unique_name', \
cp_get(cp, ce, 'name', "UNKNOWN_CE"))
simple = cp.getboolean(cluster, 'simple')
if simple:
return ce_name
else:
return cp_get(cp, cluster, 'name', ce_name)
def getSubClusterIDs(cp):
"""
Return a list of the subcluster unique ID's for this configuration.
"""
subclusters = []
for section in cp.sections():
if not section.startswith(subcluster):
continue
subCluster = cp_get(cp, section, "name", "UNKNOWN")
subclusters.append(cp_get(cp, section, "unique_name", subCluster))
return subclusters
def print_VOViewLocal(queue_info, cp):
ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
vo_map = VoMapper(cp)
queue_jobs = getJobsInfo(vo_map, cp)
VOView = getTemplate("GlueCE", "GlueVOViewLocalID")
vo_queues = getVoQueues(cp)
for vo, queue in vo_queues:
vo_info = queue_jobs.get(queue, {})
info2 = vo_info.get(vo, {})
port = getPort(cp)
ce_unique_id = buildCEUniqueID(cp, ce_name, 'pbs', queue)
my_queue_info = queue_info.setdefault(queue, {})
max_job_time = my_queue_info.get("max_wall", 0)
if cp.has_option("pbs", "max_wall"):
max_job_time = cp_getInt(cp, "pbs", "max_wall", 1440)
ert, wrt = responseTimes(cp, info2.get("running", 0),
info2.get("wait", 0),
max_job_time)
free_slots = my_queue_info.get('free_slots', 0)
waiting = info2.get('wait', 0)
if waiting > cp_getInt(cp, 'pbs', 'idle_slack', '10'):
free_slots = 0
info = {
'ceUniqueID' : ce_unique_id,
'job_slots' : my_queue_info.get('job_slots', 0),
'free_slots' : free_slots,
'ce_name' : ce_name,
'queue' : queue,
'vo' : vo,
'voLocalID' : vo,
'job_manager' : 'pbs',
'running' : info2.get('running', 0),
'max_running' : info2.get('max_running', 0),
'priority' : queue_info.get(queue, {}).get('priority', 0),
'waiting' : waiting,
'data' : cp_get(cp, "osg_dirs", "data", "UNKNOWN_DATA"),
'app' : cp_get(cp, "osg_dirs", "app", "UNKNOWN_APP"),
'default_se' : getDefaultSE(cp),
'ert' : 3600,
'wrt' : 3600,
'acbr' : 'VO:%s' % vo
}
info['total'] = info['waiting'] + info['running']
printTemplate(VOView, info)
def interpolateConfig(cp):
grid = cp_get(cp, "site", "group", "")
if cp_getBoolean(cp, "gip_tests", "oim_aware", False):
sitelist_cmd = "wget -O - http://oim.grid.iu.edu/pub/resource/show.php?format=plain-text 2>/dev/null | grep \",%s,\" | grep \",CE\" | cut -f1 -d," % grid
sitelist = runCommand(sitelist_cmd).read().split()
sitelist = ",".join(sitelist)
cp.set("gip_tests", "site_names", sitelist)
else:
if cp_get(cp, "gip_tests", "site_names", "") == "":
cp.set("gip_tests", "site_names", cp_get(cp, "site", "name", ""))
if cp_get(cp, "gip_tests", "site_dns", "") == "":
host_parts = cp_get(cp, "ce", "name", "").split('.')
site_dns = "%s.%s" % (host_parts[:-2], host_parts[:-1])
cp.set("gip_tests", "site_dns", site_dns)
if cp_get(cp, "gip_tests", "required_site", "") == "":
cp.set("gip_tests", "required_sites", cp_get(cp, "gip_tests", "site_names", ""))
cp.set("gip_tests", "bdii_port", "2170")
cp.set("gip_tests", "egee_port", "2170")
cp.set("gip_tests", "interop_url", "http://oim.grid.iu.edu/publisher/get_osg_interop_bdii_ldap_list.php?grid=%s&format=html" % grid)
if strContains(grid, "ITB"):
cp.set("bdii", "endpoint", "ldap://is-itb.grid.iu.edu:2170")
cp.set("gip_tests", "bdii_addr", "is-itb.grid.iu.edu")
cp.set("gip_tests", "egee_bdii", "pps-bdii.cern.ch")
cp.set("gip_tests", "egee_bdii_conf_url", "http://egee-pre-production-service.web.cern.ch/egee-pre-production-service/bdii/pps-all-sites.conf")
web_server = "http://is-itb.grid.iu.edu"
else:
cp.set("gip_tests", "bdii_addr", "is.grid.iu.edu")
cp.set("gip_tests", "egee_bdii", "lcg-bdii.cern.ch")
cp.set("gip_tests", "egee_bdii_conf_url", "http://lcg-bdii-conf.cern.ch/bdii-conf/bdii.conf")
web_server = "http://is.grid.iu.edu"
cp.set("gip_tests", "update_url", web_server + "/cgi-bin/status.cgi")
cp.set("gip_tests", "schema_check_url", web_server + "/data/cemon_processed_osg/%s.processed?which=%s")
cp.set("gip_tests", "validator_url", web_server + "/data/cemon_processed_osg/%s.processed?which=%s")
if cp_get(cp, "gip_tests", "compare_excludes", "") == "":
compare_excludes="GlueCEStateFreeJobSlots,GlueCEStateRunningJobs,GlueCEStateTotalJobs,GlueSiteLocation,GlueSAStateAvailableSpace,GlueSAStateUsedSpace"
cp.set("gip_tests", "compare_excludes", compare_excludes)
if cp_get(cp, "gip_tests", "enable_glite", "") == "":
cp.set("gip_tests", "enable_glite", "False")
if cp_get(cp, "gip_tests", "results_dir", "") == "":
results_dir = os.path.expandvars("$GIP_LOCATION/../apache/htdocs/")
cp.set("gip_tests", "results_dir", results_dir)
def print_VOViewLocal(queue_info, cp):
ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
vo_map = VoMapper(cp)
queue_jobs = getJobsInfo(vo_map, cp)
VOView = getTemplate("GlueCE", "GlueVOViewLocalID")
vo_queues = getVoQueues(cp)
for vo, queue in vo_queues:
vo_info = queue_jobs.get(queue, {})
info2 = vo_info.get(vo, {})
port = getPort(cp)
ce_unique_id = buildCEUniqueID(cp, ce_name, "pbs", queue)
my_queue_info = queue_info.setdefault(queue, {})
max_job_time = my_queue_info.get("max_wall", 0)
if cp.has_option("pbs", "max_wall"):
max_job_time = cp_getInt(cp, "pbs", "max_wall", 1440)
ert, wrt = responseTimes(cp, info2.get("running", 0), info2.get("wait", 0), max_job_time)
free_slots = my_queue_info.get("free_slots", 0)
waiting = info2.get("wait", 0)
if waiting > cp_getInt(cp, "pbs", "idle_slack", "10"):
free_slots = 0
info = {
"ceUniqueID": ce_unique_id,
"job_slots": my_queue_info.get("job_slots", 0),
"free_slots": free_slots,
"ce_name": ce_name,
"queue": queue,
"vo": vo,
"voLocalID": vo,
"job_manager": "pbs",
"running": info2.get("running", 0),
"max_running": info2.get("max_running", 0),
"priority": queue_info.get(queue, {}).get("priority", 0),
"waiting": waiting,
"data": cp_get(cp, "osg_dirs", "data", "UNKNOWN_DATA"),
"app": cp_get(cp, "osg_dirs", "app", "UNKNOWN_APP"),
"default_se": getDefaultSE(cp),
"ert": 3600,
"wrt": 3600,
"acbr": "VO:%s" % vo,
}
info["total"] = info["waiting"] + info["running"]
printTemplate(VOView, info)
def getCESEBindInfo(cp):
"""
Generates a list of information for the CESE bind groups.
Each list entry is a dictionary containing the necessary information for
filling out a CESE bind entry.
@param cp: Site configuration
@returns: List of dictionaries; each dictionary is a CESE bind entry.
"""
binds = []
ce_list = getCEList(cp)
se_list = getSEList(cp, classicSEs=False)
classicse_list = getClassicSEList(cp)
se_list.extend(classicse_list)
#access_point = cp_get(cp, "vo", "default", "/")
section_map = getSESections(cp)
access_point = getPath(cp)
if not access_point:
access_point = "/"
classic_access_point = cp_get(cp, "osg_dirs", "data", "/")
for myse in se_list:
mount_point = None
if myse in section_map:
section = section_map[myse]
mount_point = cp_get(cp, section, "mount_point", None)
for myce in ce_list:
if myse in classicse_list:
ap = classic_access_point
else:
ap = access_point
info = {
'ceUniqueID': myce,
'seUniqueID': myse,
'access_point': ap,
}
if mount_point:
# Note: always start this with a newline, otherwise the
# GLUE template will put this on the same line as the previous
# attribute
info['mount_point'] = '\nGlueCESEBindMountInfo: %s' % \
mount_point
else:
info['mount_point'] = ''
binds.append(info)
return binds
def getSRMs(self):
"""
Return a list of dictionaries containing information about the SRM
endpoints.
Each dictionary must have the following keys:
- acbr
- status
- version
- endpoint
- name
The base class implementation uses the following configuration entries
(default value in parenthesis)
- se.srm_host (default: UNKNOWN.example.com)
- se.srm_version (2.2.0)
- se.srm_port (8443)
- se.srm_endpoint
(httpg://(se.srm_host):(se.srm_port)/srm/managerv2)
"""
srmname = cp_get(self._cp, self._section, "srm_host",
"UNKNOWN.example.com")
version = cp_get(self._cp, self._section, "srm_version", "2")
port = cp_getInt(self._cp, self._section, "srm_port", 8443)
if version.find('2') >= 0:
default_endpoint = 'httpg://%s:%i/srm/managerv2' % \
(srmname, int(port))
else:
default_endpoint = 'httpg://%s:%i/srm/managerv1' % \
(srmname, int(port))
endpoint = cp_get(self._cp, self._section, "srm_endpoint",
default_endpoint)
acbr_tmpl = '\nGlueServiceAccessControlRule: %s\n' \
'GlueServiceAccessControlRule: VO:%s'
acbr = ''
vos = voListStorage(self._cp, self._section)
for vo in vos:
acbr += acbr_tmpl % (vo, vo)
info = {'acbr': acbr[1:],
'status': 'OK',
'version': version,
'endpoint': endpoint,
'name': srmname,
}
return [info]
def pbsCommand(command, cp):
"""
Run a command against the PBS batch system.
Use this when talking to PBS; not only does it allow for integration into
the GIP test framework, but it also filters and expands PBS-style line
continuations.
"""
try:
pbsHost = cp.get("pbs", "host")
except:
pbsHost = ""
if pbsHost.lower() == "none" or pbsHost.lower() == "localhost":
pbsHost = ""
pbs_path = cp_get(cp, "pbs", "pbs_path", ".")
addToPath(pbs_path)
addToPath(pbs_path + "/bin")
cmd = command % {'pbsHost': pbsHost}
fp = runCommand(cmd)
#pid, exitcode = os.wait()
#if exitcode != 0:
# raise Exception("Command failed: %s" % cmd)
return pbsOutputFilter(fp)
def __init__(self, site, cp):
GipUnittest.GipTestCase.__init__(self, 'testGipOutput')
self.type = cp_get(self.cp, "gip_tests", "validate_type", "")
self.name = 'testGipOutput_%s' % site
self.site = site
self.entries = None
self.site_id = None
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
osg_grid = cp_get(cp, "osg_dirs", "grid_dir", None)
if not osg_grid:
raise RuntimeError('grid_dir ($OSG_GRID) not defined!')
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {
'locationId': 'OSG_GRID',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'OSG_GRID',
'version': 1.0,
'path': osg_grid,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.error(e)
sys.stdout = sys.stderr
raise
def _getJobsInfoInternal(cp):
"""
The "alternate" way of building the jobs info; this allows for sites to
filter jobs based upon an arbitrary condor_q constraint.
This is not the default as large sites can have particularly bad performance
for condor_q.
"""
global _results_cache
if _results_cache:
return dict(_results_cache)
constraint = cp_get(cp, "condor", "jobs_constraint", "TRUE")
fp = condorCommand(condor_job_status, cp, {'constraint': constraint})
handler = ClassAdParser('GlobalJobId', ['JobStatus', 'Owner',
'AccountingGroup', 'FlockFrom']);
submitConstraint = _createSubmitterConstraint(cp)
fp2 = condorCommand(condor_status_submitter, cp, {'constraint': submitConstraint})
handler2 = ClassAdParser('Name', ['MaxJobsRunning'])
try:
xmlLine = ''
while xmlLine.find('<?xml') == -1: # Throw away junk lines from Condor < 7.3.2
xmlLine = fp.readline()
condorXml = cStringIO.StringIO(xmlLine+fp.read())
parseCondorXml(condorXml, handler)
except Exception, e:
log.error("Unable to parse condor output!")
log.exception(e)
if 'GIP_TESTING' in os.environ: raise RuntimeError('Could not parse condor_q -xml output!')
return {}
def getName(self):
"""
Return the name of the SE.
The base class uses the value of se.name in the configuration object.
"""
return cp_get(self._cp, self._section, 'name', 'UNKNOWN')
def getAccessProtocols(self):
"""
Stub function for providing access protocol information.
Return a list of dictionaries. Each dictionary will
have the following keys with reference to an access endpoint:
- protocol
- hostname
- port
Optionally, the following keys may be included (default in parenthesis):
- capability (file transfer)
- maxStreams (1)
- securityinfo (none)
- version (UNKNOWN)
- endpoint (<protocol>://<hostname>:<port>)
For the base class, this just returns [].
"""
mount_point = cp_get(self._cp, self._section, 'mount_point', None)
if None:
return []
return [{'protocol': 'file', 'hostname': 'POSIX.example.com', 'port': '1234',
'version': '1.0.0'}]
def run(self):
endpoint = cp_get(self._cp, self._section, "infoProviderEndpoint", "")
try:
self.handler = parse_fp(urllib2.urlopen(endpoint))
except Exception, e:
log.exception(e)
self.handler = None
def parseNodes(cp):
"""
Parse the condor nodes.
@param cp: ConfigParser object for the GIP
@returns: A tuple consisting of the total, claimed, and unclaimed nodes.
"""
global _nodes_cache #pylint: disable-msg=W0603
if _nodes_cache:
return _nodes_cache
subtract = cp_getBoolean(cp, "condor", "subtract_owner", True)
log.debug("Parsing condor nodes.")
constraint = cp_get(cp, "condor", "status_constraint", "TRUE")
fp = condorCommand(condor_status, cp, {'constraint': constraint})
handler = ClassAdParser('Name', ['State'])
parseCondorXml(fp, handler)
total = 0
claimed = 0
unclaimed = 0
for info in handler.getClassAds().values():
total += 1
if 'State' not in info:
continue
if info['State'] == 'Claimed':
claimed += 1
elif info['State'] == 'Unclaimed':
unclaimed += 1
elif subtract and info['State'] == 'Owner':
total -= 1
log.info("There are %i total; %i claimed and %i unclaimed." % \
(total, claimed, unclaimed))
_nodes_cache = total, claimed, unclaimed
return total, claimed, unclaimed
def getJobsInfo(vo_map, cp):
"""
Retrieve information about the jobs in the Condor system.
Query condor about the submitter status. The returned job information is
a dictionary whose keys are the VO name of the submitting user and values
the aggregate information about that VO's activities. The information is
another dictionary showing the running, idle, held, and max_running jobs
for that VO.
@param vo_map: A vo_map object mapping users to VOs
@param cp: A ConfigParser object with the GIP config information.
@returns: A dictionary containing job information.
"""
group_jobs = {}
queue_constraint = cp_get(cp, "condor", "jobs_constraint", "TRUE")
if queue_constraint.upper() == 'TRUE':
submitConstraint = _createSubmitterConstraint(cp)
fp = condorCommand(condor_status_submitter, cp, {'constraint': submitConstraint})
handler = ClassAdParser(('Name', 'ScheddName'), ['RunningJobs',
'IdleJobs', 'HeldJobs', 'MaxJobsRunning', 'FlockedJobs'])
try:
parseCondorXml(fp, handler)
except Exception, e:
log.error("Unable to parse condor output!")
log.exception(e)
results = handler.getClassAds()
def buildServiceID(cp):
ceName = cp_get(cp, "ce", "name", "UNKNOWN_CE")
gliteSuffix = 'org.glite.ce.CREAM'
uniqueHash = getUniqueHash(cp)
serviceID = '%s_%s_%s' % (ceName, gliteSuffix, uniqueHash)
return serviceID
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
osg_grid = cp_get(cp, "osg_dirs", "grid_dir", None)
if not osg_grid:
raise RuntimeError('grid_dir ($OSG_GRID) not defined!')
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {'locationId': 'OSG_GRID',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'OSG_GRID',
'version': 1.0,
'path': osg_grid,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.error(e)
sys.stdout = sys.stderr
raise
def getCEImpl(cp):
ceImpl = 'Globus'
ceImplVersion = cp_get(cp, ce, 'globus_version', '4.0.6')
if cp_getBoolean(cp, 'cream', 'enabled', False):
ceImpl = 'CREAM'
ceImplVersion = getOSGVersion(cp)
return (ceImpl, ceImplVersion)
def pbsCommand(command, cp):
"""
Run a command against the PBS batch system.
Use this when talking to PBS; not only does it allow for integration into
the GIP test framework, but it also filters and expands PBS-style line
continuations.
"""
try:
pbsHost = cp.get("pbs", "host")
except:
pbsHost = ""
if pbsHost.lower() == "none" or pbsHost.lower() == "localhost":
pbsHost = ""
pbs_path = cp_get(cp, "pbs", "pbs_path", ".")
addToPath(pbs_path)
addToPath(pbs_path + "/bin")
cmd = command % {'pbsHost': pbsHost}
fp = runCommand(cmd)
#pid, exitcode = os.wait()
#if exitcode != 0:
# raise Exception("Command failed: %s" % cmd)
return pbsOutputFilter(fp)
def parseNodes(cp):
"""
Parse the condor nodes.
@param cp: ConfigParser object for the GIP
@returns: A tuple consisting of the total, claimed, and unclaimed nodes.
"""
global _nodes_cache #pylint: disable-msg=W0603
if _nodes_cache:
return _nodes_cache
subtract = cp_getBoolean(cp, "condor", "subtract_owner", True)
log.debug("Parsing condor nodes.")
constraint = cp_get(cp, "condor", "status_constraint", "TRUE")
fp = condorCommand(condor_status, cp, {'constraint': constraint})
handler = ClassAdParser('Name', ['State'])
parseCondorXml(fp, handler)
total = 0
claimed = 0
unclaimed = 0
for info in handler.getClassAds().values():
total += 1
if 'State' not in info:
continue
if info['State'] == 'Claimed':
claimed += 1
elif info['State'] == 'Unclaimed':
unclaimed += 1
elif subtract and info['State'] == 'Owner':
total -= 1
log.info("There are %i total; %i claimed and %i unclaimed." % \
(total, claimed, unclaimed))
_nodes_cache = total, claimed, unclaimed
return total, claimed, unclaimed
def getJobsInfo(vo_map, cp):
"""
Retrieve information about the jobs in the Condor system.
Query condor about the submitter status. The returned job information is
a dictionary whose keys are the VO name of the submitting user and values
the aggregate information about that VO's activities. The information is
another dictionary showing the running, idle, held, and max_running jobs
for that VO.
@param vo_map: A vo_map object mapping users to VOs
@param cp: A ConfigParser object with the GIP config information.
@returns: A dictionary containing job information.
"""
group_jobs = {}
queue_constraint = cp_get(cp, "condor", "jobs_constraint", "TRUE")
if queue_constraint.upper() == 'TRUE':
submitConstraint = _createSubmitterConstraint(cp)
fp = condorCommand(condor_status_submitter, cp, {'constraint': submitConstraint})
handler = ClassAdParser(('Name', 'ScheddName'), ['RunningJobs',
'IdleJobs', 'HeldJobs', 'MaxJobsRunning', 'FlockedJobs'])
try:
parseCondorXml(fp, handler)
except Exception, e:
log.error("Unable to parse condor output!")
log.exception(e)
results = handler.getClassAds()
def _getJobsInfoInternal(cp):
"""
The "alternate" way of building the jobs info; this allows for sites to
filter jobs based upon an arbitrary condor_q constraint.
This is not the default as large sites can have particularly bad performance
for condor_q.
"""
global _results_cache
if _results_cache:
return dict(_results_cache)
constraint = cp_get(cp, "condor", "jobs_constraint", "TRUE")
fp = condorCommand(condor_job_status, cp, {'constraint': constraint})
handler = ClassAdParser('GlobalJobId', ['JobStatus', 'Owner',
'AccountingGroup', 'FlockFrom']);
submitConstraint = _createSubmitterConstraint(cp)
fp2 = condorCommand(condor_status_submitter, cp, {'constraint': submitConstraint})
handler2 = ClassAdParser('Name', ['MaxJobsRunning'])
try:
xmlLine = ''
while xmlLine.find('<?xml') == -1: # Throw away junk lines from Condor < 7.3.2
xmlLine = fp.readline()
condorXml = cStringIO.StringIO(xmlLine+fp.read())
parseCondorXml(condorXml, handler)
except Exception, e:
log.error("Unable to parse condor output!")
log.exception(e)
if 'GIP_TESTING' in os.environ: raise RuntimeError('Could not parse condor_q -xml output!')
return {}
def getApplicationsV1(cp):
"""
Retrieves the applications in the new "v1" format; it looks in
- $OSG_APP/etc/grid3-locations.txt
- $OSG_APP/etc/<vo>/locations-v1.txt
Here, $OSG_APP/etc may be overridden by the config variable
gip.software_dir.
One directory is created per VO, owned by root, world-writable, and set to
sticky. The VO names will be determined by the current methods (using
voList). GIP will then scan the directory $OSG_APP/etc/<vo> for all the
VO names known to GIP, and read only a file named locations-v1.txt. It is
assumed that locations-v1.txt is the same format as the grid3-locations.txt.
"""
app_dir = cp_get(cp, "osg_dirs", "app", "/UNKNOWN")
base_path = os.path.join(app_dir, "etc")
base_path = cp_get(cp, "gip", "software_dir", base_path)
locations = {}
for vo in voList(cp):
vo_dir = os.path.join(base_path, vo)
vo_dir = os.path.expandvars(vo_dir)
vo_path = os.path.join(vo_dir, 'locations-v1.txt')
if not os.path.exists(vo_path):
continue
try:
fp = open(vo_path, 'r')
except:
log.warning("Unable to read VO application file: %s" % vo_path)
continue
for line in fp:
line = line.strip()
info = line.split()
# Skip blank lines and comments
if len(line) == 0 or info[0].startswith('#'):
continue
if len(info) != 3:
log.warning("Invalid line: %s" % line)
continue
if info[1].startswith('#') or info[1].startswith('$'):
info[1] = 'UNDEFINED'
info = {'locationName': info[0], 'version': info[1], 'path':info[2]}
info['locationId'] = info['locationName']
locations = _addLocation(locations, info)
return locations
def print_SE(se, cp):
"""
Emit the GLUE entities for the SE, based upon the StorageElement class.
"""
# if the unique ID is UNKNOWN, a real SE does not exist, the classic SE
# will probably be invoked
seUniqueID = se.getUniqueID()
if seUniqueID == "UNKNOWN" or seUniqueID == "UNAVAILABLE": return
status = se.getStatus()
version = se.getVersion()
# Determine space information
try:
used, available, total = se.getSESpace(total=True, gb=True)
except:
used, available, total = 0, 0, 0
# Tape information, if we have it...
nu, _, nt = se.getSETape()
siteUniqueID = cp.get("site", "unique_name")
implementation = se.getImplementation()
# Try to guess the appropriate architecture
arch = se.getSEArch()
# port number was hard coded to 8443, get from cp now
# NOTE: this field is deprecated by the schema so it should not be used
port = se.getPort()
# Fill in the information for the template
info = {
'seName': se.getName(),
'seUniqueID': se.getUniqueID(),
'implementation': implementation,
"version": version,
"status": status,
"port": port,
"onlineTotal": total,
"nearlineTotal": nt,
"onlineUsed": used,
"nearlineUsed": nu,
"architecture": arch,
"free": available,
"total": total,
"bdii": cp_get(cp, "bdii", "endpoint", "Unknown"),
"siteUniqueID": siteUniqueID,
"arch": arch,
}
seTemplate = getTemplate("GlueSE", "GlueSEUniqueID")
log.info(str(info))
printTemplate(seTemplate, info)
try:
print_SA(se, cp, se.getSection())
except Exception, e:
log.exception(e)
def setConfigValues(self):
self.results_dir = cp_get(self.cp, "gip_tests", "results_dir",
"UNKNOWN")
if self.results_dir == "UNKNOWN":
raise ConfigurationError("Results directory is not configured")
self.oim_xml_dir = cp_get(self.cp, "gip_tests", "myosg_xml_dir",
"UNKNOWN")
if self.oim_xml_dir == "UNKNOWN":
raise ConfigurationError("OIM XML directory is not configured")
self.oim_summary_xml_file = "%s/%s" % (
self.oim_xml_dir,
cp_get(self.cp, "gip_tests", "myosg_summary_file", "myosg.xml"))
self.oim_detail_file_template = self.oim_xml_dir + "/" + cp_get(
self.cp, "gip_tests", "myosg_detail_file_template",
"myosg_%s_detail.xml")
self.oim_tests = cp_get(self.cp, "gip_tests", "myosg_tests",
self.oim_tests)
def getClassicSEList(cp):
"""
Return a list of all the ClassicSE's at this site
@param cp: Site configuration
@returns: List of all the ClassicSE's unique_ids
"""
classic_list = []
if cp_getBoolean(cp, "classic_se", "advertise_se", True):
classicSE = cp_get(cp, "classic_se", "unique_name", None)
if classicSE:
classic_list = [classicSE]
else:
siteUniqueID = cp_get(cp, "site", "unique_name", "UNKNOWN")
classicSE = siteUniqueID + "_classicSE"
classic_list = [classicSE]
return classic_list
def print_VOInfo(se, cp):
"""
Print out the VOInfo GLUE information for all the VOInfo
objects in the SE.
This will optionally alter the VOInfo object to limit the total available
space for a given VO.
"""
vo_limit_str = cp_get(cp, "se", "vo_limits", "")
vo_limit = {}
cumulative_total = {}
for vo_str in vo_limit_str.split(','):
vo_str = vo_str.strip()
info = vo_str.split(":")
if len(info) != 2:
continue
vo = info[0].strip()
try:
limit = float(info[1].strip())
except:
continue
vo_limit[vo] = limit
cumulative_total.setdefault(vo, 0)
for voinfo in se.getVOInfos():
do_continue = False
reduce_by_amount = 0
try:
totalOnline = float(voinfo.get("totalOnline", 0))
except:
continue
for vo in get_vos_from_acbr(voinfo.get("acbr", "")):
if vo in vo_limit:
try:
cumulative_total[vo] += totalOnline
except:
pass
reduce_by_amount = max(
reduce_by_amount,
cumulative_total.get(vo, 0) - vo_limit[vo])
if reduce_by_amount > totalOnline:
continue
elif reduce_by_amount > 0:
voinfo['totalOnline'] = str(totalOnline - reduce_by_amount)
try:
voinfo['availableSpace'] = str(
max(
0,
float(voinfo['availableSpace']) -
reduce_by_amount * 1024**2))
voinfo['freeOnline'] = str(max(0, float(voinfo['freeOnline']) \
- reduce_by_amount))
except:
pass
try:
print_single_VOInfo(voinfo, se, cp)
except Exception, e:
log.exception(e)
def doPath(cp):
# add condor binaries to system path
condor_path = cp_get(cp, "condor", "condor_path", None)
condor_location = cp_get(cp, "condor", "condor_location", None)
condor_config = cp_get(cp, "condor", "condor_config", None)
if isDefined(condor_path):
addToPath(condor_path)
if isDefined(condor_location):
log.info("Adding %s/bin to path" % condor_location)
addToPath('%s/bin' % condor_location)
if not isDefined(condor_config):
condor_config = '%s/etc/condor_config' % condor_location
if isDefined(condor_config):
os.environ['CONDOR_CONFIG'] = condor_config
def getClusterName(cp):
"""
Return the name of the associated cluster.
"""
ce_name = cp_get(cp, ce, 'name', "UNKNOWN_CE")
simple = cp.getboolean(cluster, 'simple')
if simple:
return ce_name
else:
return cp.get(cluster, 'name')
