def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
# Load up the template for GlueService
# To view its contents, see $VDT_LOCATION/gip/templates/GlueService
template = getTemplate("GlueService", "GlueServiceUniqueID")
if not cp_getBoolean(cp, "site", "advertise_gums", True):
log.info("Not advertising authorization service.")
return
else:
log.info("Advertising authorization service.")
try:
authfile = open('/etc/grid-security/gsi-authz.conf', 'r')
authlines = authfile.readlines()
authmod = re.compile('^(?!#)(.*)libprima_authz_module')
for line in authlines:
m = authmod.match(line)
if m:
publish_gums(cp, template)
return
except IOError:
log.info("/etc/grid-security/gsi-authz.conf not found; assuming gridmap file authorization")
publish_gridmap_file(cp, template)
except Exception, e:
# Log error, then report it via stderr.
log.exception(e)
sys.stdout = sys.stderr
raise
def main():
"""
The primary wrapper function for emitting GLUE for storage elements.
"""
cp = config()
# Handle full-fledged SEs
found_se = False
for section in cp.sections():
# need to search for sections with "_" because if you are only
# advertising a classic SE, then you do NOT want to run the
# handle_SE function or you will get duplicate and incorrect info
if section.lower().startswith("se_"):
advertise_se = cp_getBoolean(cp, section, "advertise_se", True)
if advertise_se:
handle_SE(cp, section)
found_se = True
if found_se == False and 'se' in cp.sections():
handle_SE(cp, 'se')
# Handle the "classic" SE.
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
try:
print_classicSE(cp)
except Exception, e:
log.exception(e)
def getSEList(cp, classicSEs=True):
"""
Return a list of all the SE's at this site.
@param cp: Site configuration.
@keyword classicSEs: Return list should contain classicSEs; default is True.
@returns: List of strings containing all the local SE unique_ids.
"""
simple = cp.getboolean(cesebind, 'simple')
se_list = []
if simple:
try:
if cp_getBoolean(cp, se, 'advertise_se', True):
se_list = [cp.get(se, 'unique_name')]
except:
pass
for sect in cp.sections():
if not sect.lower().startswith('se') or sect.lower() == 'se':
continue
if not cp_getBoolean(cp, sect, 'advertise_se', True):
continue
try:
se_list += [cp.get(sect, 'unique_name')]
except:
pass
else:
try:
se_list = list(split_re.split(cp.get(cesebind, 'se_list')))
except:
se_list = getSEList(cp, classicSEs=classicSEs)
if classicSEs:
se_list.extend(getClassicSEList(cp))
se_list = list(Set(se_list))
return se_list
def main():
"""
The primary wrapper function for emitting GLUE for storage elements.
"""
cp = config()
# Handle full-fledged SEs
found_se = False
for section in cp.sections():
# need to search for sections with "_" because if you are only
# advertising a classic SE, then you do NOT want to run the
# handle_SE function or you will get duplicate and incorrect info
if section.lower().startswith("se_"):
advertise_se = cp_getBoolean(cp, section, "advertise_se", True)
if advertise_se:
handle_SE(cp, section)
found_se = True
if found_se == False and 'se' in cp.sections():
handle_SE(cp, 'se')
# Handle the "classic" SE.
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
try:
print_classicSE(cp)
except Exception, e:
log.exception(e)
def getDefaultSE(cp):
global _defaultSE
if _defaultSE:
return _defaultSE
default_se = cp_get(cp, "se", "name", "UNKNOWN")
# if [se] name: ??? is "UNAVAILABLE" or not set, then try to get the
# default_se
if default_se == "UNKNOWN" or default_se == "UNAVAILABLE":
default_se = cp_get(cp, "se", "default_se", "UNAVAILABLE")
# if it is still UNAVAILABLE or not set, check to see if the classic SE
# is being advertised and use that
if default_se == "UNAVAILABLE" and cp_getBoolean(cp, "classic_se",
"advertise_se", True):
fallback_name = cp_get(cp, "site", "unique_name", "UNKNOWN") + \
"_classicSE"
default_se = cp_get(cp, "classic_se", "name", fallback_name)
current_se = None
for sect in cp.sections():
if not sect.lower().startswith('se'):
continue
try:
current_se = cp.get(sect, 'name')
except:
continue
if cp_getBoolean(cp, sect, "default", False):
_defaultSE = current_se
return current_se
if default_se == 'UNKNOWN' and current_se:
_defaultSE = current_se
return current_se
_defaultSE = default_se
return default_se
def getSEList(cp, classicSEs=True):
"""
Return a list of all the SE's at this site.
@param cp: Site configuration.
@keyword classicSEs: Return list should contain classicSEs; default is True.
@returns: List of strings containing all the local SE unique_ids.
"""
simple = cp.getboolean(cesebind, 'simple')
se_list = []
if simple:
try:
if cp_getBoolean(cp, se, 'advertise_se', True):
se_list = [cp.get(se, 'unique_name')]
except:
pass
for sect in cp.sections():
if not sect.lower().startswith('se') or sect.lower() == 'se':
continue
if not cp_getBoolean(cp, sect, 'advertise_se', True):
continue
try:
se_list += [cp.get(sect, 'unique_name')]
except:
pass
else:
try:
se_list = list(split_re.split(cp.get(cesebind, 'se_list')))
except:
se_list = getSEList(cp, classicSEs=classicSEs)
if classicSEs:
se_list.extend(getClassicSEList(cp))
se_list = list(Set(se_list))
return se_list
def getPort(cp):
port = 2119
if cp_getBoolean(cp, 'cream', 'enabled', False):
port = 8443
if cp_getBoolean(cp, 'htcondorce', 'enabled', False):
port = getHTCondorCEPort()
return port
def getGramVersion(cp):
gramVersion = '\n' + 'GlueCEInfoGRAMVersion: 5.0'
if cp_getBoolean(cp, 'cream', 'enabled', False):
gramVersion = ''
if cp_getBoolean(cp, 'htcondorce', 'enabled', False):
gramVersion = ''
return gramVersion
def _generateSubClusterHelper(cp, section):
"""
Private helper function for generateSubClusters; do not use.
Implementation note: We create a bunch of variables but never
appear to use them - we actually do return the local dictionary
at the end of the function. Just being syntactically lazy.
"""
#pylint: disable-msg=W0612
# Names
subCluster = cp_get(cp, section, "name", cluster)
subClusterUniqueID = cp_get(cp, section, "unique_name", subCluster)
clusterUniqueID = getClusterID(cp)
# Host statistics
clockSpeed = cp_getInt(cp, section, "cpu_speed_mhz", 999999999)
cpuCount = cp_getInt(cp, section, "cpus_per_node", 2)
model = cp_get(cp, section, "cpu_model", 'UNDEFINEDVALUE')
platform = cp_get(cp, section, "platform", "UNKNOWN")
vendor = cp_get(cp, section, "cpu_vendor", 'UNDEFINEDVALUE')
cores_per_cpu = cp_getInt(cp, section, "cores_per_cpu", 2)
si2k = cp_getInt(cp, section, "SI00", 2000)
sf2k = cp_getInt(cp, section, "SF00", 2000)
hepspec = cp_getInt(cp, section, "HEPSPEC", 0)
ram = cp_getInt(cp, section, "ram_size", 1000*cpuCount*cores_per_cpu)
cores = cp_getInt(cp, section, "total_cores", 999999999)
if cores_per_cpu != 0:
cpus = cp_getInt(cp, section, "total_cpus", cores/cores_per_cpu)
else:
cpus = 0
virtualMem = ram + cp_getInt(cp, section, "swap_size", 0)
inboundIP = cp_getBoolean(cp, section, "inbound_network", False)
outboundIP = cp_getBoolean(cp, section, "outbound_network", True)
inboundIP = ldap_boolean(inboundIP)
outboundIP = ldap_boolean(outboundIP)
# OS Stats
osName, osRelease, osVersion = getRelease()
# Temp directories
default_tmp = cp_get(cp, "osg_dirs", "tmp", cp_get(cp, "osg_dirs", "data", \
"/tmp"))
wn_tmp = cp_get(cp, "osg_dirs", "wn_tmp", "/tmp")
tmp = cp_get(cp, section, "tmp", default_tmp)
if notDefined(tmp):
tmp = default_tmp
app_attr = 'GlueHostApplicationSoftwareRunTimeEnvironment'
apps = getApplications(cp)
applications = '\n'.join(['%s: %s' % (app_attr, i['locationId']) for i in \
apps if i['locationId']])
applications += '\n'
# BDII stuff
bdii = cp_get(cp, "bdii", "endpoint", "ldap://is.grid.iu.edu:2170")
return locals()
def getCEImpl(cp):
ceImpl = 'Globus'
ceImplVersion = cp_get(cp, ce, 'globus_version', '4.0.6')
if cp_getBoolean(cp, 'cream', 'enabled', False):
ceImpl = 'CREAM'
ceImplVersion = getOSGVersion(cp)
if cp_getBoolean(cp, 'htcondorce', 'enabled', False):
ceImpl = 'HTCondorCE'
ceImplVersion = getHTCondorCEVersion(cp)
return (ceImpl, ceImplVersion)
def buildCEUniqueID(cp, ce_name, batch, queue):
ce_prefix = 'jobmanager'
if cp_getBoolean(cp, 'cream', 'enabled', False):
ce_prefix = 'cream'
if cp_getBoolean(cp, 'htcondorce', 'enabled', False):
ce_prefix = 'htcondorce'
port = getPort(cp)
ce_unique_id = '%s:%d/%s-%s-%s' % (ce_name, port, ce_prefix, batch, queue)
return ce_unique_id
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
osg_grid = cp_get(cp, "osg_dirs", "grid_dir", None)
if not osg_grid:
raise RuntimeError('grid_dir ($OSG_GRID) not defined!')
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {'locationId': 'OSG_GRID',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'OSG_GRID',
'version': 1.0,
'path': osg_grid,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.error(e)
sys.stdout = sys.stderr
raise
def handle_SE(cp, section):
"""
Run a provider for one SE.
"""
# if you only have a classic SE, there will still be a [se] section
# with the default_se option. This will result in a GlueService
# stanza being written with garbage information in it. So, check
# for default_se = "UNKNOWN" or "UNAVAILABLE" and return if it does
#
# default_se is set in the [Storage] section in config.ini and is
# required by configure-osg.py
default_se = cp_get(cp, "se", "default_se", "UNKNOWN")
if default_se == "UNAVAILABLE" or default_se == "UNKNOWN": return
impl = cp_get(cp, section, "implementation", "UNKNOWN")
provider_impl = cp_get(cp, section, "provider_implementation", "UNKNOWN")
if provider_impl == "UNKNOWN" and not cp_getBoolean(cp, section,
"dynamic_dcache", False):
provider_impl = 'static'
se_class, cp = determine_provider(provider_impl, impl, cp)
se = se_class(cp, section=section)
log.info("Outputting SE %s; implementation %s; provider %s" % (se.getName(),
impl, provider_impl))
try:
se.run()
except Exception, e:
log.exception(e)
def test_gip_conf(self):
"""
Make sure that the $GIP_LOCATION/etc/gip.conf file is read.
"""
old_gip_location = os.environ['GIP_LOCATION']
tmpdir = tempfile.mkdtemp()
try:
os.environ['GIP_LOCATION'] = tmpdir
etc_dir = os.path.join(tmpdir, 'etc')
try:
os.mkdir(etc_dir)
cp_orig = ConfigParser.ConfigParser()
cp_orig.add_section("gip_test")
cp_orig.set("gip_test", "gip_conf", "True")
gip_conf = os.path.join(etc_dir, 'gip.conf')
fp = open(gip_conf, 'w')
try:
cp_orig.write(fp)
fp.close()
cp = ConfigParser.ConfigParser()
cp.read([gip_conf])
result = cp_getBoolean(cp, "gip_test", "gip_conf", False)
self.failUnless(result, msg="Failed to load $GIP_LOCATION"\
"/etc/gip.conf")
finally:
os.unlink(gip_conf)
finally:
os.rmdir(etc_dir)
finally:
os.rmdir(tmpdir)
os.environ['GIP_LOCATION'] = old_gip_location
def handle_SE(cp, section):
"""
Run a provider for one SE.
"""
# if you only have a classic SE, there will still be a [se] section
# with the default_se option. This will result in a GlueService
# stanza being written with garbage information in it. So, check
# for default_se = "UNKNOWN" or "UNAVAILABLE" and return if it does
#
# default_se is set in the [Storage] section in config.ini and is
# required by configure-osg.py
default_se = cp_get(cp, "se", "default_se", "UNKNOWN")
if default_se == "UNAVAILABLE" or default_se == "UNKNOWN": return
impl = cp_get(cp, section, "implementation", "UNKNOWN")
provider_impl = cp_get(cp, section, "provider_implementation", "UNKNOWN")
if provider_impl == "UNKNOWN" and not cp_getBoolean(
cp, section, "dynamic_dcache", False):
provider_impl = 'static'
se_class, cp = determine_provider(provider_impl, impl, cp)
se = se_class(cp, section=section)
log.info("Outputting SE %s; implementation %s; provider %s" %
(se.getName(), impl, provider_impl))
try:
se.run()
except Exception, e:
log.exception(e)
def interpolateConfig(cp):
grid = cp_get(cp, "site", "group", "")
if cp_getBoolean(cp, "gip_tests", "oim_aware", False):
sitelist_cmd = "wget -O - http://oim.grid.iu.edu/pub/resource/show.php?format=plain-text 2>/dev/null | grep \",%s,\" | grep \",CE\" | cut -f1 -d," % grid
sitelist = runCommand(sitelist_cmd).read().split()
sitelist = ",".join(sitelist)
cp.set("gip_tests", "site_names", sitelist)
else:
if cp_get(cp, "gip_tests", "site_names", "") == "":
cp.set("gip_tests", "site_names", cp_get(cp, "site", "name", ""))
if cp_get(cp, "gip_tests", "site_dns", "") == "":
host_parts = cp_get(cp, "ce", "name", "").split('.')
site_dns = "%s.%s" % (host_parts[:-2], host_parts[:-1])
cp.set("gip_tests", "site_dns", site_dns)
if cp_get(cp, "gip_tests", "required_site", "") == "":
cp.set("gip_tests", "required_sites",
cp_get(cp, "gip_tests", "site_names", ""))
cp.set("gip_tests", "bdii_port", "2170")
cp.set("gip_tests", "egee_port", "2170")
cp.set(
"gip_tests", "interop_url",
"http://oim.grid.iu.edu/publisher/get_osg_interop_bdii_ldap_list.php?grid=%s&format=html"
% grid)
if strContains(grid, "ITB"):
cp.set("bdii", "endpoint", "ldap://is-itb.grid.iu.edu:2170")
cp.set("gip_tests", "bdii_addr", "is-itb.grid.iu.edu")
cp.set("gip_tests", "egee_bdii", "pps-bdii.cern.ch")
cp.set(
"gip_tests", "egee_bdii_conf_url",
"http://egee-pre-production-service.web.cern.ch/egee-pre-production-service/bdii/pps-all-sites.conf"
)
web_server = "http://is-itb.grid.iu.edu"
else:
cp.set("gip_tests", "bdii_addr", "is.grid.iu.edu")
cp.set("gip_tests", "egee_bdii", "lcg-bdii.cern.ch")
cp.set("gip_tests", "egee_bdii_conf_url",
"http://lcg-bdii-conf.cern.ch/bdii-conf/bdii.conf")
web_server = "http://is.grid.iu.edu"
cp.set("gip_tests", "update_url", web_server + "/cgi-bin/status.cgi")
cp.set("gip_tests", "schema_check_url",
web_server + "/data/cemon_processed_osg/%s.processed?which=%s")
cp.set("gip_tests", "validator_url",
web_server + "/data/cemon_processed_osg/%s.processed?which=%s")
if cp_get(cp, "gip_tests", "compare_excludes", "") == "":
compare_excludes = "GlueCEStateFreeJobSlots,GlueCEStateRunningJobs,GlueCEStateTotalJobs,GlueSiteLocation,GlueSAStateAvailableSpace,GlueSAStateUsedSpace"
cp.set("gip_tests", "compare_excludes", compare_excludes)
if cp_get(cp, "gip_tests", "enable_glite", "") == "":
cp.set("gip_tests", "enable_glite", "False")
if cp_get(cp, "gip_tests", "results_dir", "") == "":
results_dir = os.path.expandvars("$GIP_LOCATION/../apache/htdocs/")
cp.set("gip_tests", "results_dir", results_dir)
def parseNodes(cp):
"""
Parse the condor nodes.
@param cp: ConfigParser object for the GIP
@returns: A tuple consisting of the total, claimed, and unclaimed nodes.
"""
global _nodes_cache #pylint: disable-msg=W0603
if _nodes_cache:
return _nodes_cache
subtract = cp_getBoolean(cp, "condor", "subtract_owner", True)
log.debug("Parsing condor nodes.")
constraint = cp_get(cp, "condor", "status_constraint", "TRUE")
fp = condorCommand(condor_status, cp, {'constraint': constraint})
handler = ClassAdParser('Name', ['State'])
parseCondorXml(fp, handler)
total = 0
claimed = 0
unclaimed = 0
for info in handler.getClassAds().values():
total += 1
if 'State' not in info:
continue
if info['State'] == 'Claimed':
claimed += 1
elif info['State'] == 'Unclaimed':
unclaimed += 1
elif subtract and info['State'] == 'Owner':
total -= 1
log.info("There are %i total; %i claimed and %i unclaimed." % \
(total, claimed, unclaimed))
_nodes_cache = total, claimed, unclaimed
return total, claimed, unclaimed
def runTest(cp, cls, out=None, per_site=True):
"""
Given a test class, generate and run a test suite
@param cp: Site configuration
@type cp: ConfigParser
@param cls: Test class to use to generate a test suite. It is assumed
that the constructor for this class has signature cls(cp, site_name).
If per_site=False, then the signature is assumed to be cls().
@type cls: class
@keyword per_site: Set to true if there is one instance of the test class
per site.
@param out: A stream where the output from the test suite is logged
@type out: stream
"""
usexml = cp_getBoolean(cp, "gip_tests", "use_xml", default=False)
if per_site:
testSuite = generateTests(cp, cls, sys.argv[1:])
else:
testSuite = unittest.TestLoader().loadTestsFromTestCase(cls)
if usexml:
testRunner = GipUnittest.GipXmlTestRunner()
else:
if out is None:
#testRunner = unittest.TextTestRunner(verbosity=2)
testRunner = GipUnittest.GipTextTestRunner(verbosity=2)
else:
#testRunner = unittest.TextTestRunner(stream=out, verbosity=2)
testRunner = GipUnittest.GipTextTestRunner(stream=out, verbosity=2)
result = testRunner.run(testSuite)
sys.exit(not result.wasSuccessful())
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
osg_grid = cp_get(cp, "osg_dirs", "grid_dir", None)
if not osg_grid:
raise RuntimeError('grid_dir ($OSG_GRID) not defined!')
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {
'locationId': 'OSG_GRID',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'OSG_GRID',
'version': 1.0,
'path': osg_grid,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.error(e)
sys.stdout = sys.stderr
raise
def runTest(cp, cls, out=None, per_site=True):
"""
Given a test class, generate and run a test suite
@param cp: Site configuration
@type cp: ConfigParser
@param cls: Test class to use to generate a test suite. It is assumed
that the constructor for this class has signature cls(cp, site_name).
If per_site=False, then the signature is assumed to be cls().
@type cls: class
@keyword per_site: Set to true if there is one instance of the test class
per site.
@param out: A stream where the output from the test suite is logged
@type out: stream
"""
usexml = cp_getBoolean(cp, "gip_tests", "use_xml", default=False)
if per_site:
testSuite = generateTests(cp, cls, sys.argv[1:])
else:
testSuite = unittest.TestLoader().loadTestsFromTestCase(cls)
if usexml:
testRunner = GipUnittest.GipXmlTestRunner()
else:
if out is None:
#testRunner = unittest.TextTestRunner(verbosity=2)
testRunner = GipUnittest.GipTextTestRunner(verbosity=2)
else:
#testRunner = unittest.TextTestRunner(stream=out, verbosity=2)
testRunner = GipUnittest.GipTextTestRunner(stream=out, verbosity=2)
result = testRunner.run(testSuite)
sys.exit(not result.wasSuccessful())
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
# Get the timestamp in the two formats we wanted
epoch = str(time.time())
now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {'locationId': 'TIMESTAMP',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'TIMESTAMP',
'version': epoch,
'path': now,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.exception(e)
sys.stdout = sys.stderr
raise
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
# Get the timestamp in the two formats we wanted
epoch = str(time.time())
now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {
'locationId': 'TIMESTAMP',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'TIMESTAMP',
'version': epoch,
'path': now,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.exception(e)
sys.stdout = sys.stderr
raise
def parseNodes(cp):
"""
Parse the condor nodes.
@param cp: ConfigParser object for the GIP
@returns: A tuple consisting of the total, claimed, and unclaimed nodes.
"""
global _nodes_cache #pylint: disable-msg=W0603
if _nodes_cache:
return _nodes_cache
subtract = cp_getBoolean(cp, "condor", "subtract_owner", True)
log.debug("Parsing condor nodes.")
constraint = cp_get(cp, "condor", "status_constraint", "TRUE")
fp = condorCommand(condor_status, cp, {'constraint': constraint})
handler = ClassAdParser('Name', ['State'])
parseCondorXml(fp, handler)
total = 0
claimed = 0
unclaimed = 0
for info in handler.getClassAds().values():
total += 1
if 'State' not in info:
continue
if info['State'] == 'Claimed':
claimed += 1
elif info['State'] == 'Unclaimed':
unclaimed += 1
elif subtract and info['State'] == 'Owner':
total -= 1
log.info("There are %i total; %i claimed and %i unclaimed." % \
(total, claimed, unclaimed))
_nodes_cache = total, claimed, unclaimed
return total, claimed, unclaimed
def getCEList(cp, extraCEs=[]):
"""
Return a list of all the CE names at this site.
If WS-GRAM is installed, this might additionally return some WS-GRAM
entries (this feature is not yet implemented).
@param cp: Site configuration
@returns: List of strings containing all the local CE names.
"""
jobman = cp.get(ce, "job_manager").strip().lower()
hostnames = [cp.get(ce, 'name')]
hostnames += extraCEs
prefix = 'jobmanager'
port = 2119
if cp_getBoolean(cp, 'cream', 'enabled', False):
prefix = 'cream'
port = 8443
if cp_getBoolean(cp, 'htcondorce', 'enabled', False):
prefix = 'htcondorce'
port = getHTCondorCEPort()
ce_names = ['%s:%d/%s-%s-%%s' % (hostname, port, prefix, jobman) for hostname in hostnames]
ce_list = []
if jobman == 'pbs':
queue_entries = getPBSQueueList(cp)
elif jobman == 'lsf':
from gip.providers.lsf import bootstrapLSF
bootstrapLSF(cp)
queue_entries = getLSFQueueList(cp)
elif jobman == 'condor':
queue_entries = getCondorQueueList(cp)
elif jobman == 'slurm':
queue_entries = getSlurmQueueList(cp)
elif jobman == 'sge':
from gip.providers.sge import bootstrapSGE
from gip_common import addToPath
bootstrapSGE(cp)
addToPath(cp_get(cp, "sge", "sge_path", "."))
queue_entries = getSGEQueueList(cp)
else:
raise ValueError("Unknown job manager %s." % jobman)
for queue in queue_entries:
for ce_name in ce_names:
ce_list.append(ce_name % queue)
return ce_list
def main():
cp = config()
if not cp_getBoolean(cp, "cluster", "advertise_cluster", True):
return
try:
print_clusters(cp)
except Exception, e:
log.exception(e)
def main():
cp = config()
if not cp_getBoolean(cp, 'cluster', 'advertise_cluster', True):
return
try:
print_clusters(cp)
except Exception, e:
log.exception(e)
def main(cp = None, return_entries=False):
"""
Main method for the osg-info-wrapper script. This script safely runs the
plugin and provider modules, caching where necessary, and combines it with
the static data. It then outputs the final GLUE information for this site.
"""
log.debug("Starting up the osg-info-wrapper.")
if cp == None:
cp = config()
temp_dir = os.path.expandvars(cp_get(cp, "gip", "temp_dir", \
gipDir("$GIP_LOCATION/var/tmp", '/var/cache/gip')))
plugin_dir = os.path.expandvars(cp_get(cp, "gip", "plugin_dir", \
gipDir("$GIP_LOCATION/plugins", '/usr/libexec/gip/plugins')))
provider_dir = os.path.expandvars(cp_get(cp, "gip", "provider_dir", \
gipDir("$GIP_LOCATION/providers", '/usr/libexec/gip/providers')))
static_dir = os.path.expandvars(cp_get(cp, "gip", "static_dir", \
gipDir("$GIP_LOCATION/var/ldif", '/etc/gip/ldif.d')))
# Make sure that our directories exist.
create_if_not_exist(temp_dir, plugin_dir, provider_dir, static_dir)
# Load up our add, alter, and delete attributes
add_attributes = os.path.expandvars(cp_get(cp, "gip", \
"add_attributes", gipDir("$GIP_LOCATION/etc/add-attributes.conf",
'/etc/gip/add-attributes.conf')))
alter_attributes = os.path.expandvars(cp_get(cp, "gip", \
"alter_attributes", gipDir("$GIP_LOCATION/etc/alter-attributes.conf",
'/etc/gip/alter-attributes.conf')))
remove_attributes = os.path.expandvars(cp_get(cp, "gip", \
"remove_attributes", gipDir("$GIP_LOCATION/etc/remove-attributes.conf",
'/etc/gip/remove-attributes.conf')))
# Flush the cache if appropriate
do_flush_cache = cp_getBoolean(cp, "gip", "flush_cache", False)
if do_flush_cache:
log.info("Flushing cache upon request.")
flush_cache(temp_dir)
# Load up our parameters
freshness = cp_getInt(cp, "gip", "freshness", 300)
cache_ttl = cp_getInt(cp, "gip", "cache_ttl", 600)
response = cp_getInt(cp, "gip", "response", 240)
timeout = cp_getInt(cp, "gip", "timeout", 240)
try:
os.setpgrp()
except OSError, oe:
# If launched from a batch system (condor), we might not have perms
if oe.errno != 1:
raise
def main(cp=None, return_entries=False):
"""
Main method for the osg-info-wrapper script. This script safely runs the
plugin and provider modules, caching where necessary, and combines it with
the static data. It then outputs the final GLUE information for this site.
"""
log.debug("Starting up the osg-info-wrapper.")
if cp == None:
cp = config()
temp_dir = os.path.expandvars(cp_get(cp, "gip", "temp_dir", \
gipDir("$GIP_LOCATION/var/tmp", '/var/cache/gip')))
plugin_dir = os.path.expandvars(cp_get(cp, "gip", "plugin_dir", \
gipDir("$GIP_LOCATION/plugins", '/usr/libexec/gip/plugins')))
provider_dir = os.path.expandvars(cp_get(cp, "gip", "provider_dir", \
gipDir("$GIP_LOCATION/providers", '/usr/libexec/gip/providers')))
static_dir = os.path.expandvars(cp_get(cp, "gip", "static_dir", \
gipDir("$GIP_LOCATION/var/ldif", '/etc/gip/ldif.d')))
# Make sure that our directories exist.
create_if_not_exist(temp_dir, plugin_dir, provider_dir, static_dir)
# Load up our add, alter, and delete attributes
add_attributes = os.path.expandvars(cp_get(cp, "gip", \
"add_attributes", vdtDir("$VDT_LOCATION/gip/etc/add-attributes.conf",
'/etc/gip/add-attributes.conf')))
alter_attributes = os.path.expandvars(cp_get(cp, "gip", \
"alter_attributes", vdtDir("$VDT_LOCATION/gip/etc/alter-attributes.conf",
'/etc/gip/alter-attributes.conf')))
remove_attributes = os.path.expandvars(cp_get(cp, "gip", \
"remove_attributes", vdtDir("$VDT_LOCATION/gip/etc/remove-attributes.conf",
'/etc/gip/remove-attributes.conf')))
# Flush the cache if appropriate
do_flush_cache = cp_getBoolean(cp, "gip", "flush_cache", False)
if do_flush_cache:
log.info("Flushing cache upon request.")
flush_cache(temp_dir)
# Load up our parameters
freshness = cp_getInt(cp, "gip", "freshness", 300)
cache_ttl = cp_getInt(cp, "gip", "cache_ttl", 600)
response = cp_getInt(cp, "gip", "response", 240)
timeout = cp_getInt(cp, "gip", "timeout", 240)
try:
os.setpgrp()
except OSError, oe:
# If launched from a batch system (condor), we might not have perms
if oe.errno != 1:
raise
def interpolateConfig(cp):
grid = cp_get(cp, "site", "group", "")
if cp_getBoolean(cp, "gip_tests", "oim_aware", False):
sitelist_cmd = "wget -O - http://oim.grid.iu.edu/pub/resource/show.php?format=plain-text 2>/dev/null | grep \",%s,\" | grep \",CE\" | cut -f1 -d," % grid
sitelist = runCommand(sitelist_cmd).read().split()
sitelist = ",".join(sitelist)
cp.set("gip_tests", "site_names", sitelist)
else:
if cp_get(cp, "gip_tests", "site_names", "") == "":
cp.set("gip_tests", "site_names", cp_get(cp, "site", "name", ""))
if cp_get(cp, "gip_tests", "site_dns", "") == "":
host_parts = cp_get(cp, "ce", "name", "").split('.')
site_dns = "%s.%s" % (host_parts[:-2], host_parts[:-1])
cp.set("gip_tests", "site_dns", site_dns)
if cp_get(cp, "gip_tests", "required_site", "") == "":
cp.set("gip_tests", "required_sites", cp_get(cp, "gip_tests", "site_names", ""))
cp.set("gip_tests", "bdii_port", "2170")
cp.set("gip_tests", "egee_port", "2170")
cp.set("gip_tests", "interop_url", "http://oim.grid.iu.edu/publisher/get_osg_interop_bdii_ldap_list.php?grid=%s&format=html" % grid)
if strContains(grid, "ITB"):
cp.set("bdii", "endpoint", "ldap://is-itb.grid.iu.edu:2170")
cp.set("gip_tests", "bdii_addr", "is-itb.grid.iu.edu")
cp.set("gip_tests", "egee_bdii", "pps-bdii.cern.ch")
cp.set("gip_tests", "egee_bdii_conf_url", "http://egee-pre-production-service.web.cern.ch/egee-pre-production-service/bdii/pps-all-sites.conf")
web_server = "http://is-itb.grid.iu.edu"
else:
cp.set("gip_tests", "bdii_addr", "is.grid.iu.edu")
cp.set("gip_tests", "egee_bdii", "lcg-bdii.cern.ch")
cp.set("gip_tests", "egee_bdii_conf_url", "http://lcg-bdii-conf.cern.ch/bdii-conf/bdii.conf")
web_server = "http://is.grid.iu.edu"
cp.set("gip_tests", "update_url", web_server + "/cgi-bin/status.cgi")
cp.set("gip_tests", "schema_check_url", web_server + "/data/cemon_processed_osg/%s.processed?which=%s")
cp.set("gip_tests", "validator_url", web_server + "/data/cemon_processed_osg/%s.processed?which=%s")
if cp_get(cp, "gip_tests", "compare_excludes", "") == "":
compare_excludes="GlueCEStateFreeJobSlots,GlueCEStateRunningJobs,GlueCEStateTotalJobs,GlueSiteLocation,GlueSAStateAvailableSpace,GlueSAStateUsedSpace"
cp.set("gip_tests", "compare_excludes", compare_excludes)
if cp_get(cp, "gip_tests", "enable_glite", "") == "":
cp.set("gip_tests", "enable_glite", "False")
if cp_get(cp, "gip_tests", "results_dir", "") == "":
results_dir = os.path.expandvars("$GIP_LOCATION/../apache/htdocs/")
cp.set("gip_tests", "results_dir", results_dir)
def getTests(self):
test_dir = os.path.expandvars("$GIP_LOCATION/reporting")
test_list = ls(test_dir)
for test in test_list:
if test in self.crit:
self.critical_tests.append("%s/%s" % (test_dir, test))
else:
self.reports.append("%s/%s" % (test_dir, test))
if cp_getBoolean(self.cp, "gip_tests", "enable_glite"):
test_dir = os.path.expandvars("$GIP_LOCATION/reporting/glite_reports")
tmp_list = ls(test_dir)
for i in range(0, len(tmp_list)):
tmp_list[i] = "%s/%s" % (test_dir, tmp_list[i])
self.glite_tests.extend(tmp_list)
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only and 'VDT_LOCATION' in os.environ:
# get the VDT version
vdt_version_cmd = os.path.expandvars(
"$VDT_LOCATION/vdt/bin/") + 'vdt-version --no-wget'
vdt_version_out = runCommand(vdt_version_cmd).readlines()
gip_re = re.compile('Generic Information Provider\s+(.*?)\s*-.*')
gip_version = 'UNKNOWN'
for line in vdt_version_out:
m = gip_re.match(line)
if m:
gip_version = m.groups()[0]
break
gip_version += '; $Revision$'
# Get the timestamp in the two formats we wanted
now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {
'locationId': 'GIP_VERSION',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'GIP_VERSION',
'version': gip_version,
'path': now,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.exception(e)
sys.stdout = sys.stderr
raise
def getTests(self):
test_dir = os.path.expandvars("$GIP_LOCATION/reporting")
test_list = ls(test_dir)
for test in test_list:
if test in self.crit:
self.critical_tests.append("%s/%s" % (test_dir, test))
else:
self.reports.append("%s/%s" % (test_dir, test))
if cp_getBoolean(self.cp, "gip_tests", "enable_glite"):
test_dir = os.path.expandvars(
"$GIP_LOCATION/reporting/glite_reports")
tmp_list = ls(test_dir)
for i in range(0, len(tmp_list)):
tmp_list[i] = "%s/%s" % (test_dir, tmp_list[i])
self.glite_tests.extend(tmp_list)
def runTests(self):
self.getTests()
# perfom the reports
self.runList(self.reports, "reports")
# perfom the glite reports
self.runList(self.glite_tests, "glite")
# perfom the critical tests
self.runList(self.critical_tests, "critical")
self.writeResultsPage()
oim_plugin_enabled = cp_getBoolean(self.cp, "gip_tests", "enable_myosg_plugin", False)
if oim_plugin_enabled:
oim_plugin = os.path.expandvars("$GIP_LOCATION/reporting/plugins/OIM_XML_Aggregator.py")
cmd = '/bin/bash -c "%(source)s; %(plugin)s %(args)s "' % ({"source": self.source_cmd, "plugin": oim_plugin, "args": self.args})
runCommand(cmd)
def getCEList(cp, extraCEs=[]):
"""
Return a list of all the CE names at this site.
If WS-GRAM is installed, this might additionally return some WS-GRAM
entries (this feature is not yet implemented).
@param cp: Site configuration
@returns: List of strings containing all the local CE names.
"""
jobman = cp.get(ce, "job_manager").strip().lower()
hostnames = [cp.get(ce, 'name')]
hostnames += extraCEs
prefix = 'jobmanager'
port = 2119
if cp_getBoolean(cp, 'cream', 'enabled', False):
prefix = 'cream'
port = 8443
ce_names = [
'%s:%d/%s-%s-%%s' % (hostname, port, prefix, jobman)
for hostname in hostnames
]
ce_list = []
if jobman == 'pbs':
queue_entries = getPBSQueueList(cp)
elif jobman == 'lsf':
from gip.providers.lsf import bootstrapLSF
bootstrapLSF(cp)
queue_entries = getLSFQueueList(cp)
elif jobman == 'condor':
queue_entries = getCondorQueueList(cp)
elif jobman == 'slurm':
queue_entries = getSlurmQueueList(cp)
elif jobman == 'sge':
from gip.providers.sge import bootstrapSGE
from gip_common import addToPath
bootstrapSGE(cp)
addToPath(cp_get(cp, "sge", "sge_path", "."))
queue_entries = getSGEQueueList(cp)
else:
raise ValueError("Unknown job manager %s." % jobman)
for queue in queue_entries:
for ce_name in ce_names:
ce_list.append(ce_name % queue)
return ce_list
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only and 'VDT_LOCATION' in os.environ:
# get the VDT version
vdt_version_cmd = os.path.expandvars("$VDT_LOCATION/vdt/bin/") + 'vdt-version --no-wget'
vdt_version_out = runCommand(vdt_version_cmd).readlines()
gip_re = re.compile('Generic Information Provider\s+(.*?)\s*-.*')
gip_version = 'UNKNOWN'
for line in vdt_version_out:
m = gip_re.match(line)
if m:
gip_version = m.groups()[0]
break
gip_version += '; $Revision$'
# Get the timestamp in the two formats we wanted
now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {'locationId': 'GIP_VERSION',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'GIP_VERSION',
'version': gip_version,
'path': now,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.exception(e)
sys.stdout = sys.stderr
raise
def getSESpace(self, gb=False, total=False):
if cp_getBoolean(self._cp, self._section, 'use_df', False) or \
self.status == False:
# Let a configuration option override the use_df option.
space = cp_get(self._cp, self._section, 'space', '')
if space:
try:
used, free, tot = eval(space, {}, {})
used, free, tot = int(used), int(free), int(tot)
except:
used, free, tot = 0, 0, 0
else:
paths = sets.Set()
# Lookup SA paths only if there's a single SA.
# Otherwise, use the default path (otherwise we get a inf loop)
if self.sas:
for sa in self.getSAs():
path = sa['path']
paths.add(path)
else:
paths = [self.getPathForSA(space=None, \
section=self._section)]
used, free, tot = 0, 0, 0
for path in paths:
try:
stat_info = os.statvfs(path)
blocks = stat_info[statvfs.F_BLOCKS]
bsize = stat_info[statvfs.F_BSIZE]
avail = stat_info[statvfs.F_BFREE]
except Exception, e:
log.exception(e)
continue
used += (blocks - avail) * bsize / 1024.
free += avail * bsize / 1024.
tot += blocks * bsize / 1024.
if total:
if gb:
return int(used/1000.**2), int(free/1000.**2), \
int(tot/1000.**2)
else:
return int(used), int(free), int(tot)
else:
if gb:
return int(used / 1000.**2), int(free / 1000.**2)
else:
return int(used), int(free)
def getSESpace(self, gb=False, total=False):
if cp_getBoolean(self._cp, self._section, 'use_df', False) or \
self.status == False:
# Let a configuration option override the use_df option.
space = cp_get(self._cp, self._section, 'space', '')
if space:
try:
used, free, tot = eval(space, {}, {})
used, free, tot = int(used), int(free), int(tot)
except:
used, free, tot = 0, 0, 0
else:
paths = sets.Set()
# Lookup SA paths only if there's a single SA.
# Otherwise, use the default path (otherwise we get a inf loop)
if self.sas:
for sa in self.getSAs():
path = sa['path']
paths.add(path)
else:
paths = [self.getPathForSA(space=None, \
section=self._section)]
used, free, tot = 0, 0, 0
for path in paths:
try:
stat_info = os.statvfs(path)
blocks = stat_info[statvfs.F_BLOCKS]
bsize = stat_info[statvfs.F_BSIZE]
avail = stat_info[statvfs.F_BFREE]
except Exception, e:
log.exception(e)
continue
used += (blocks-avail) * bsize / 1024.
free += avail * bsize / 1024.
tot += blocks * bsize / 1024.
if total:
if gb:
return int(used/1000.**2), int(free/1000.**2), \
int(tot/1000.**2)
else:
return int(used), int(free), int(tot)
else:
if gb:
return int(used/1000.**2), int(free/1000.**2)
else:
return int(used), int(free)
def buildContactString(cp, batch, queue, ce_unique_id, log):
contact_string = cp_get(cp, batch, 'job_contact', ce_unique_id)
if contact_string.endswith("jobmanager-%s" % batch):
contact_string += "-%s" % queue
if cp_getBoolean(cp, 'cream', 'enabled', False) and not \
contact_string.endswith('cream-%s' % batch):
log.warning('CREAM CE enabled, but contact string in config.ini '
'does not end with "cream-%s"' % batch)
if contact_string.endswith('cream-%s' % batch):
contact_string += "-%s" % queue
if not contact_string.startswith('https://'):
contact_string = 'https://' + contact_string
return contact_string
def getSESections(cp):
"""
For each SE section (non-classic), determine the corresponding section
name.
Returns a dict; the SE unique ID is the key, the section is the value.
"""
se_map = {}
for sect in cp.sections():
if not sect.lower().startswith('se') or sect.lower() == 'se':
continue
if not cp_getBoolean(cp, sect, 'advertise_se', True):
continue
try:
se_map[cp.get(sect, 'unique_name')] = sect
except:
pass
return se_map
def getClassicSEList(cp):
"""
Return a list of all the ClassicSE's at this site
@param cp: Site configuration
@returns: List of all the ClassicSE's unique_ids
"""
classic_list = []
if cp_getBoolean(cp, "classic_se", "advertise_se", True):
classicSE = cp_get(cp, "classic_se", "unique_name", None)
if classicSE:
classic_list = [classicSE]
else:
siteUniqueID = cp_get(cp, "site", "unique_name", "UNKNOWN")
classicSE = siteUniqueID + "_classicSE"
classic_list = [classicSE]
return classic_list
def getSESections(cp):
"""
For each SE section (non-classic), determine the corresponding section
name.
Returns a dict; the SE unique ID is the key, the section is the value.
"""
se_map = {}
for sect in cp.sections():
if not sect.lower().startswith('se') or sect.lower() == 'se':
continue
if not cp_getBoolean(cp, sect, 'advertise_se', True):
continue
try:
se_map[cp.get(sect, 'unique_name')] = sect
except:
pass
return se_map
def getSESpace(cp, admin=None, gb=False, total=False, section=se):
"""
Return the amount of space available at the SE.
If se.dynamic_dcache=True, use dCache-based methods.
Otherwise, use classic SE methods (do a df on the SE mounts).
@param cp: Site configuration object
@keyword admin: If a dCache provider, the dCache admin objects
@keyword gb: Set to true to retun values in GB.
@keyword total: Also return totals
@return: used, free, total if total is True; otherwise, used, free.
In GB if GB=True; otherwise, in KB.
"""
if cp_getBoolean(cp, section, "dynamic_dcache", False):
return getdCacheSESpace(cp, admin, gb, total)
else:
return getClassicSESpace(cp, gb=gb, total=total)
def getClassicSEList(cp):
"""
Return a list of all the ClassicSE's at this site
@param cp: Site configuration
@returns: List of all the ClassicSE's unique_ids
"""
classic_list = []
if cp_getBoolean(cp, "classic_se", "advertise_se", True):
classicSE = cp_get(cp, "classic_se", "unique_name", None)
if classicSE:
classic_list = [classicSE]
else:
siteUniqueID = cp_get(cp, "site", "unique_name", "UNKNOWN")
classicSE = siteUniqueID + "_classicSE"
classic_list = [classicSE]
return classic_list
def getHTPCInfo(cp, batch, queue, log):
# return tuple: (non-Glue HTPC information, htpc_max_slots)
# where htpc_max_slots is the admin-provided "maximum number of slots per job"
htpcInfo = ('__GIP_DELETEME', 1) # defaults
if not cp_getBoolean(cp, batch, 'htpc_enabled', False):
log.info("HTPC is disabled for batch %s" % batch)
return htpcInfo
log.info("HTPC is enabled for batch %s" % batch)
whitelist = cp_getList(cp, batch, 'htpc_queues', [])
blacklist = cp_getList(cp, batch, 'htpc_blacklist_queues', [])
log.debug("HTPC whitelist: %s; HTPC blacklist %s: " % (whitelist, blacklist))
if '*' not in whitelist and queue not in whitelist:
log.info("HTPC Queue %s not in whitelist" % queue)
return htpcInfo
if queue in blacklist:
log.info("HTPC Queue %s in blacklist" % queue)
return htpcInfo
defaultRSL = cp_get(cp, batch, 'htpc_rsl', '')
log.debug("HTPC DefaultRSL: %s" % defaultRSL)
queueRSL = cp_get(cp, batch, 'htpc_rsl_%s' % queue, '')
if not queueRSL:
queueRSL = defaultRSL
if not queueRSL:
log.info("HTPC RSL not found for queue %s" % queue)
return htpcInfo
htpcMaxSlots = cp_getInt(cp, batch, 'htpc_max_slots', 1)
if htpcMaxSlots < 2:
log.info("HTPC max slots equal to 1 or not set!")
return htpcInfo
# acbr stuff?
return ('HTPCrsl: %s' % queueRSL, htpcMaxSlots)
def print_classicSE(cp):
"""
Emit the relevant GLUE entities for a ClassicSE.
"""
if not cp_getBoolean(cp, "classic_se", "advertise_se", True):
log.info("Not advertising a classic SE.")
return
else:
log.info("Advertising a classic SE.")
#if cp_getBoolean(cp, "se", "shares_fs_with_ce", False):
# log.info("Not advertising a classic SE because the SE shares a FS.")
# return
status = cp_get(cp, "classic_se", "status", "Production")
version = cp_get(cp, "classic_se", "version", "UNKNOWN")
try:
used, available, total = getClassicSESpace(cp, total=True, gb=True)
except Exception, e:
log.error("Unable to get SE space: %s" % str(e))
used, available, total = 0, 0, 0
def print_classicSE(cp):
"""
Emit the relevant GLUE entities for a ClassicSE.
"""
if not cp_getBoolean(cp, "classic_se", "advertise_se", True):
log.info("Not advertising a classic SE.")
return
else:
log.info("Advertising a classic SE.")
#if cp_getBoolean(cp, "se", "shares_fs_with_ce", False):
# log.info("Not advertising a classic SE because the SE shares a FS.")
# return
status = cp_get(cp, "classic_se", "status", "Production")
version = cp_get(cp, "classic_se", "version", "UNKNOWN")
try:
used, available, total = getClassicSESpace(cp, total=True, gb=True)
except Exception, e:
log.error("Unable to get SE space: %s" % str(e))
used, available, total = 0, 0, 0
def main():
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
job_manager = cp_get(cp, "ce", "job_manager", None)
if job_manager:
log.info("Using job manager %s" % job_manager)
else:
log.error("Job manager not specified!")
sys.exit(2)
if job_manager == 'pbs':
pbs_main()
elif job_manager == 'condor':
condor_main()
elif job_manager == 'sge':
sge_main()
elif job_manager == 'lsf':
lsf_main()
else:
log.error("Unknown job manager: %s." % job_manager)
sys.exit(1)
def runTests(self):
self.getTests()
# perfom the reports
self.runList(self.reports, "reports")
# perfom the glite reports
self.runList(self.glite_tests, "glite")
# perfom the critical tests
self.runList(self.critical_tests, "critical")
self.writeResultsPage()
oim_plugin_enabled = cp_getBoolean(self.cp, "gip_tests",
"enable_myosg_plugin", False)
if oim_plugin_enabled:
oim_plugin = os.path.expandvars(
"$GIP_LOCATION/reporting/plugins/OIM_XML_Aggregator.py")
cmd = '/bin/bash -c "%(source)s; %(plugin)s %(args)s "' % (
{
"source": self.source_cmd,
"plugin": oim_plugin,
"args": self.args
})
runCommand(cmd)
def main():
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
job_manager = cp_get(cp, "ce", "job_manager", None)
if job_manager:
log.info("Using job manager %s" % job_manager)
else:
log.error("Job manager not specified!")
sys.exit(2)
if job_manager == 'pbs':
pbs_main()
elif job_manager == 'condor':
condor_main()
elif job_manager == 'sge':
sge_main()
elif job_manager == 'lsf':
lsf_main()
elif job_manager == 'slurm':
slurm_main()
else:
log.error("Unknown job manager: %s." % job_manager)
sys.exit(1)
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only and 'VDT_LOCATION' in os.environ:
# get the VDT version
vdt_version_cmd = os.path.expandvars("$VDT_LOCATION/vdt/bin/") + 'vdt-version --brief'
vdt_version = runCommand(vdt_version_cmd).readlines()[0].strip()
if (vdt_version == ""): vdt_version = "OLD_VDT"
# Get the timestamp in the two formats we wanted
now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {'locationId': 'VDT_VERSION',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'VDT_VERSION',
'version': vdt_version,
'path': now,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.exception(e)
sys.stdout = sys.stderr
raise
def print_CE(cp):
slurmVersion = getLrmsInfo(cp)
queueInfo = getQueueInfo(cp)
ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
CE = getTemplate("GlueCE", "GlueCEUniqueID")
try:
excludeQueues = [i.strip() for i in cp_get(cp, "slurm", \
"queue_exclude", "").split(',')]
except:
excludeQueues = []
vo_queues = getVoQueues(cp)
for queue, info in queueInfo.items():
if queue in excludeQueues:
continue
info["lrmsVersion"] = slurmVersion
info["job_manager"] = "slurm"
# if no jobs are waiting in the queue, set the number of free slots
# to (job_slots - running), or the total number of free slots on the cluster,
# whichever is less.
info["queue"] = queue
info["ceName"] = ce_name
unique_id = buildCEUniqueID(cp, ce_name, 'slurm', queue)
ceImpl, ceImplVersion = getCEImpl(cp)
port = getPort(cp)
info['ceUniqueID'] = unique_id
if "job_slots" not in info:
log.error("no job_slots found for %s!" % queue)
if "priority" not in info:
info["priority"] = 0
if "max_running" not in info:
log.error("no max_running found for %s!" % queue)
if "max_wall" not in info:
info["max_wall"] = 1440
info["free_slots"] = 0
if info["wait"] == 0:
freeSlots = info["job_slots"] - info["running"]
if freeSlots > 0:
info["free_slots"] = freeSlots
ert, wrt = responseTimes(cp,
info.get("running", 0),
info.get("wait", 0),
max_job_time=info["max_wall"])
info['ert'] = ert
info['wrt'] = wrt
info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name)
info['hostName'] = cp_get(cp, ce, 'host_name', ce_name)
info['ceImpl'] = ceImpl
info['ceImplVersion'] = ceImplVersion
contact_string = buildContactString(cp, 'slurm', queue, unique_id, log)
info['contact_string'] = contact_string
info['app_dir'] = cp_get(cp, 'osg_dirs', 'app', "/UNKNOWN_APP")
info['data_dir'] = cp_get(cp, 'osg_dirs', 'data', "/UNKNOWN_DATA")
info['default_se'] = getDefaultSE(cp)
if 'max_waiting' not in info:
info['max_waiting'] = 999999
if 'max_queuable' in info:
info['max_total'] = info['max_queuable']
info['free_slots'] = min(info['free_slots'], info['max_queuable'])
else:
info['max_total'] = info['max_waiting'] + info['max_running']
info['free_slots'] = min(info['free_slots'], info['max_total'])
# Enforce invariants:
# max_total <= max_running
# free_slots <= max_running
info['max_total'] = min(info['max_total'], info['max_running'])
info['free_slots'] = min(info['free_slots'], info['max_running'])
info['assigned'] = info['job_slots']
# Enforce invariants:
# assigned <= max_running
info['assigned'] = min(info['assigned'], info['max_running'])
info['lrmsType'] = 'slurm'
info['preemption'] = cp_get(cp, 'slurm', 'preemption', '0')
acbr = ''
has_vo = False
for vo, queue2 in vo_queues:
if queue == queue2:
acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo
has_vo = True
if not has_vo:
continue
info['acbr'] = acbr[:-1]
info['bdii'] = cp.get('bdii', 'endpoint')
gramVersion = getGramVersion(cp)
info['gramVersion'] = gramVersion
info['port'] = port
info['waiting'] = info['wait']
info['referenceSI00'] = gip_cluster.getReferenceSI00(cp)
info['clusterUniqueID'] = getClusterID(cp)
extraCapabilities = ''
if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'
htpcRSL, maxSlots = getHTPCInfo(cp, 'slurm', queue, log)
info['max_slots'] = maxSlots
if maxSlots > 1:
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'
info['extraCapabilities'] = extraCapabilities
info['htpc'] = htpcRSL
printTemplate(CE, info)
return queueInfo
#!/usr/bin/env python
import os
import sys
if 'GIP_LOCATION' in os.environ:
sys.path.insert(0, os.path.expandvars("$GIP_LOCATION/lib/python"))
from gip_common import config, cp_getBoolean
from gip.providers.cese_bind import main
if __name__ == '__main__':
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
main()
myheld = sum([i.get('held', 0) for i in jinfo.values()], 0)
max_wall = cp_getInt(cp, "condor", "max_wall", 1440)
ert, wrt = responseTimes(cp,
myrunning,
myidle + myheld,
max_job_time=max_wall * 60)
referenceSI00 = gip_cluster.getReferenceSI00(cp)
contact_string = buildContactString(cp, 'condor', group, ce_unique_id,
log)
htpcRSL, maxSlots = getHTPCInfo(cp, 'condor', group, log)
extraCapabilities = ''
if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'
if maxSlots > 1:
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'
gramVersion = getGramVersion(cp)
ceImpl, ceImplVersion = getCEImpl(cp)
port = getPort(cp)
# Build all the GLUE CE entity information.
info = { \
"ceUniqueID" : ce_unique_id,
'contact_string' : contact_string,
"ceImpl" : ceImpl,
"ceImplVersion" : ceImplVersion,
def print_CE(cp):
SGEVersion = getLrmsInfo(cp)
queueInfo, _ = getQueueInfo(cp)
ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
ce_template = getTemplate("GlueCE", "GlueCEUniqueID")
queueList = getQueueList(cp)
vo_queues = getVoQueues(cp)
default_max_waiting = 999999
for queue in queueInfo.values():
if 'name' not in queue or queue['name'] not in queueList:
continue
if queue['name'] == 'waiting':
continue
unique_id = buildCEUniqueID(cp, ce_name, 'sge', queue['name'])
acbr = ''
for vo, queue2 in vo_queues:
if queue['name'] == queue2:
acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo
referenceSI00 = gip_cluster.getReferenceSI00(cp)
contact_string = buildContactString(cp, 'sge', queue['name'],
unique_id, log)
extraCapabilities = ''
if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'
htpcRSL, maxSlots = getHTPCInfo(cp, 'sge', queue, log)
if maxSlots > 1:
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'
gramVersion = getGramVersion(cp)
port = getPort(cp)
ceImpl, ceImplVersion = getCEImpl(cp)
max_wall = queue["max_wall"]
if cp.has_option("sge", "max_wall"):
max_wall = cp_getInt(cp, "sge", "max_wall", 1440)
info = { \
"ceUniqueID" : unique_id,
"ceName" : ce_name,
"ceImpl" : ceImpl,
"ceImplVersion" : ceImplVersion,
"clusterUniqueID" : getClusterID(cp),
"queue" : queue['name'],
"priority" : queue['priority'],
"lrmsType" : 'sge',
"lrmsVersion" : SGEVersion,
"job_manager" : "sge",
"job_slots" : queue["slots_total"],
"free_slots" : queue["slots_free"],
"running" : queue["slots_used"],
"status" : queue['status'],
"total" : queue['slots_used'] + queue['waiting'],
"ert" : 3600,
"wrt" : 3600,
"hostingCluster" : cp_get(cp, ce, 'hosting_cluster', ce_name),
"hostName" : cp_get(cp, ce, 'host_name', ce_name),
"contact_string" : contact_string,
"app_dir" : cp_get(cp, 'osg_dirs', 'app', "/OSG_APP_UNKNOWN"),
"data_dir" : cp_get(cp, 'osg_dirs', 'data', "/OSG_DATA_UNKNOWN"),
"default_se" : getDefaultSE(cp),
"max_running" : queue["slots_total"],
"max_wall" : max_wall,
"max_waiting" : default_max_waiting,
"max_slots" : maxSlots,
"max_total" : default_max_waiting + queue["slots_total"],
"assigned" : queue["slots_used"],
"preemption" : cp_get(cp, 'sge', 'preemption', '0'),
"acbr" : acbr[:-1],
"bdii": cp.get('bdii', 'endpoint'),
"gramVersion" : gramVersion,
"port" : port,
"waiting" : queue['waiting'],
"referenceSI00": referenceSI00,
'extraCapabilities' : extraCapabilities,
"htpc" : htpcRSL
}
printTemplate(ce_template, info)
return queueInfo
group = name
if len(all_group_info[name].get('vo', [])) == 1:
vo = all_group_info[name]['vo']
else:
vo = 'unknown'
else:
unknown_users.add(name)
continue
vo_jobs = group_jobs.setdefault(group, {})
# Add the information to the current dictionary.
my_info = vo_jobs.get(vo, {"running":0, "idle":0, "held":0, \
'max_running':0})
addIntInfo(my_info, info, "running", "RunningJobs")
if cp_getBoolean(cp, "condor", "count_flocked", True):
addIntInfo(my_info, info, "running", "FlockedJobs")
addIntInfo(my_info, info, "idle", "IdleJobs")
addIntInfo(my_info, info, "held", "HeldJobs")
addIntInfo(my_info, info, "max_running", "MaxJobsRunning")
vo_jobs[vo] = my_info
log.warning("The following users are non-grid users: %s" % \
", ".join(unknown_users))
log.info("Job information: %s." % group_jobs)
return group_jobs
_nodes_cache = []
def parseNodes(cp):
def getGroupInfo(vo_map, cp): #pylint: disable-msg=C0103,W0613
"""
Get the group info from condor
The return value is a dictionary; the key is the vo name, the values are
another dictionary of the form {'quota': integer, 'prio': integer}
@param vo_map: VoMapper object
@param cp: A ConfigParse object with the GIP config information
@returns: A dictionary whose keys are VO groups and values are the quota
and priority of the group.
"""
configDaemon = ''
if cp_getBoolean(cp, "condor", "query_only_local_condor", False):
log.info("Only querying local Condor -- ignoring use_collector, if set!")
else:
if cp_getBoolean(cp, "condor", "use_collector", False):
configDaemon = "-collector"
else:
configDaemon = "-negotiator"
fp = condorCommand(condor_group, cp, {'daemon' : configDaemon})
output = fp.read().split(',')
if fp.close():
log.info("No condor groups found.")
return {}
grouplist = []
for group in output:
grouplist.append(group.strip())
excludedGroups = cp_getList(cp, "condor", "exclude_groups", [])
log.debug("exclude_groups = %s" % excludedGroups)
for excludedGroup in excludedGroups:
excludedGroup = excludedGroup.strip()
try:
grouplist.remove(excludedGroup)
log.debug("Removed excluded group %s" % excludedGroup)
except ValueError:
if excludedGroup != 'default':
log.debug("Attempted to remove non-existent group %s" % excludedGroup)
if not grouplist:
log.info("No condor groups exist (after applying exclude_groups)")
return{}
retval = {}
if (not (grouplist[0].startswith('Not defined'))) and \
(len(grouplist[0]) > 0):
for group in grouplist:
quota = condorCommand(condor_quota, cp, \
{'group': group, 'daemon': configDaemon}).read().strip()
prio = condorCommand(condor_prio, cp, \
{'group': group, 'daemon': configDaemon}).read().strip()
vos = guessVO(cp, group)
if not vos:
continue
curInfo = {'quota': 0, 'prio': 0, 'vos': vos}
try:
curInfo['quota'] += int(quota)
except:
pass
try:
curInfo['prio'] += int(prio)
except:
pass
retval[group] = curInfo
log.debug("The condor groups are %s." % ', '.join(retval.keys()))
return retval
def main():
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
site_main()
if not se_only:
cluster_main()