def check_voview_2_ldif(self, entries, cp):
name_re = re.compile('([A-Za-z]+):default')
vos = voList(cp)
found_vos = dict([(vo, False) for vo in vos])
for entry in entries:
if 'GlueVOInfo' not in entry.objectClass:
continue
if 'GlueSALocalID=default' not in entry.glue['ChunkKey'] or \
'GlueSEUniqueID=dcache07.unl.edu' not in \
entry.glue['ChunkKey']:
continue
m = name_re.match(entry.glue['VOInfoLocalID'][0])
self.failUnless(m, msg="Unknown VOInfo entry!")
vo = m.groups()[0]
self.failIf(vo not in vos, msg="Unknown VO for view: %s." % vo)
found_vos[vo] = True
self.failUnless(entry.glue['VOInfoPath'][0] == \
'/user/%s' % vo, msg="Incorrect path, %s, for" \
" vo, %s" % (entry.glue['VOInfoPath'][0], vo))
self.failUnless(entry.glue['VOInfoLocalID'] == \
entry.glue['VOInfoName'])
self.failUnless(vo in entry.glue['VOInfoAccessControlBaseRule'])
for vo, status in found_vos.items():
if not status:
self.fail("VOView for %s missing." % vo)
def guessVO(cp, group):
"""
From the group name, guess my VO name
"""
mapper = VoMapper(cp)
bycp = determineGroupVOsFromConfig(cp, group, mapper)
vos = voList(cp, vo_map=mapper)
byname = sets.Set()
for vo in vos:
if group.find(vo) >= 0:
byname.add(vo)
altname = group.replace('group', '')
altname = altname.replace('-', '')
altname = altname.replace('_', '')
altname = altname.strip()
try:
bymapper = mapper[altname]
except:
bymapper = None
if bycp != None:
return bycp
elif bymapper:
return [bymapper]
elif byname:
return byname
else:
return [altname]
def check_srm_1_ldif(self, entries, cp):
vos = voList(cp)
found_srm = False
for entry in entries:
if 'GlueService' not in entry.objectClass:
continue
if 'httpg://srm.unl.edu:8443/srm/managerv2' not in \
entry.glue['ServiceName']:
continue
found_srm = True
for vo in vos:
self.failIf(vo not in entry.glue['ServiceAccessControlRule'])
self.failIf("VO:%s" % vo not in \
entry.glue['ServiceAccessControlRule'], msg="String `" \
"VO:%s` not in ACBR" % vo)
self.failUnless("OK" in entry.glue["ServiceStatus"])
self.failUnless(entry.glue['ServiceEndpoint'] == \
entry.glue['ServiceName'])
self.failUnless(entry.glue['ServiceAccessPointURL'] == \
entry.glue['ServiceName'])
self.failUnless(entry.glue['ServiceURI'] == \
entry.glue['ServiceName'])
self.failUnless("GlueSiteUniqueID=Nebraska" in \
entry.glue['ForeignKey'])
self.failUnless(entry.glue["ServiceType"][0] == 'SRM')
self.failUnless(entry.glue["ServiceVersion"][0] == '2.2.0')
self.failUnless(found_srm, msg="Could not find the SRM entity.")
def usersToVos(cp, qInfo, user_groups, vo_map):
"""
Given a queue's information, determine the users allowed to access the
queue.
@param qInfo: Information about a LSF queue; we assume that qInfo['USERS']
is a string of users/groups allowed to access the queue.
@param user_groups: A mapping of group name to a list of user names.
@param vo_map: A mapping of user name to vo name.
@returns: List of all the VOs allowed to access this queue.
"""
users = qInfo['USERS']
all_users = []
for user in users.split():
if user.endswith('/'):
all_users += user_groups.get(user[:-1], [])
elif user == 'all':
all_users.append(None)
else:
all_users.append(user)
all_vos = sets.Set()
if None in all_users:
return voList(cp, vo_map)
for user in all_users:
try:
vo = vo_map[user]
except:
continue
all_vos.add(vo)
return list(all_vos)
def check_srm_1_ldif(self, entries, cp):
vos = voList(cp)
found_srm = False
for entry in entries:
if 'GlueService' not in entry.objectClass:
continue
if 'httpg://srm.unl.edu:8443/srm/managerv2' not in \
entry.glue['ServiceName']:
continue
found_srm = True
for vo in vos:
self.failIf(vo not in entry.glue['ServiceAccessControlRule'])
self.failIf("VO:%s" % vo not in \
entry.glue['ServiceAccessControlRule'], msg="String `" \
"VO:%s` not in ACBR" % vo)
self.failUnless("OK" in entry.glue["ServiceStatus"])
self.failUnless(entry.glue['ServiceEndpoint'] == \
entry.glue['ServiceName'])
self.failUnless(entry.glue['ServiceAccessPointURL'] == \
entry.glue['ServiceName'])
self.failUnless(entry.glue['ServiceURI'] == \
entry.glue['ServiceName'])
self.failUnless("GlueSiteUniqueID=Nebraska" in \
entry.glue['ForeignKey'])
self.failUnless(entry.glue["ServiceType"][0] == 'SRM')
self.failUnless(entry.glue["ServiceVersion"][0] == '2.2.0')
self.failUnless(found_srm, msg="Could not find the SRM entity.")
def check_voview_2_ldif(self, entries, cp):
name_re = re.compile('([A-Za-z]+):default')
vos = voList(cp)
found_vos = dict([(vo, False) for vo in vos])
for entry in entries:
if 'GlueVOInfo' not in entry.objectClass:
continue
if 'GlueSALocalID=default' not in entry.glue['ChunkKey'] or \
'GlueSEUniqueID=dcache07.unl.edu' not in \
entry.glue['ChunkKey']:
continue
m = name_re.match(entry.glue['VOInfoLocalID'][0])
self.failUnless(m, msg="Unknown VOInfo entry!")
vo = m.groups()[0]
self.failIf(vo not in vos, msg="Unknown VO for view: %s." % vo)
found_vos[vo] = True
self.failUnless(entry.glue['VOInfoPath'][0] == \
'/user/%s' % vo, msg="Incorrect path, %s, for" \
" vo, %s" % (entry.glue['VOInfoPath'][0], vo))
self.failUnless(entry.glue['VOInfoLocalID'] == \
entry.glue['VOInfoName'])
self.failUnless(vo in entry.glue['VOInfoAccessControlBaseRule'])
for vo, status in found_vos.items():
if not status:
self.fail("VOView for %s missing." % vo)
def filter_sponsor(cp, text):
vo_list = voList(cp)
vo_map = dict([(i.lower(), i) for i in vo_list])
text = text.replace('"', '').replace("'", '')
entries = split_re.split(text)
results = []
if len(entries) == 1:
entry = entries[0]
if len(entry.split(':')) == 1:
entries[0] = entry.strip() + ":100"
for entry in entries:
try:
vo, number = entry.split(":")
number = float(number)
except:
log.warning("Entry for sponsor, `%s`, is not in <vo>:<value>" \
"format." % str(entry))
continue
if vo in vo_map:
vo = vo_map[vo]
elif (vo.startswith('uscms') or vo.startswith('usatlas')) and vo[2:] in vo_map:
vo = vo_map[vo[2:]]
elif vo.lower().startswith("local") or \
vo.lower().startswith("unknown"):
pass # Do not log warning in this case.
else:
log.warning("VO named `%s` does not match any VO in" \
" osg-user-vo-map.txt." % str(vo))
results.append("%s:%i" % (vo.lower(), int(number)))
return " ".join(results)
def filter_sponsor(cp, text):
vo_list = voList(cp)
vo_map = dict([(i.lower(), i) for i in vo_list])
text = text.replace('"', '').replace("'", '')
entries = split_re.split(text)
results = []
if len(entries) == 1:
entry = entries[0]
if len(entry.split(':')) == 1:
entries[0] = entry.strip() + ":100"
for entry in entries:
try:
vo, number = entry.split(":")
number = float(number)
except:
log.warning("Entry for sponsor, `%s`, is not in <vo>:<value>" \
"format." % str(entry))
continue
if vo in vo_map:
vo = vo_map[vo]
elif (vo.startswith('uscms')
or vo.startswith('usatlas')) and vo[2:] in vo_map:
vo = vo_map[vo[2:]]
elif vo.lower().startswith("local") or \
vo.lower().startswith("unknown"):
pass # Do not log warning in this case.
else:
log.warning("VO named `%s` does not match any VO in" \
" osg-user-vo-map.txt." % str(vo))
results.append("%s:%i" % (vo.lower(), int(number)))
return " ".join(results)
def guessVO(cp, group):
"""
From the group name, guess my VO name
"""
mapper = VoMapper(cp)
bycp = determineGroupVOsFromConfig(cp, group, mapper)
vos = voList(cp, vo_map=mapper)
byname = sets.Set()
for vo in vos:
if group.find(vo) >= 0:
byname.add(vo)
altname = group.replace('group', '')
altname = altname.replace('-', '')
altname = altname.replace('_', '')
altname = altname.strip()
try:
bymapper = mapper[altname]
except:
bymapper = None
if bycp != None:
return bycp
elif bymapper:
return [bymapper]
elif byname:
return byname
else:
return [altname]
def getVoQueues(cp):
"""
Determine the (vo, queue) tuples for this site. This allows for central
configuration of which VOs are advertised.
Sites will be able to blacklist queues they don't want to advertise,
whitelist certain VOs for a particular queue, and blacklist VOs from queues.
@param cp: Site configuration
@returns: A list of (vo, queue) tuples representing the queues each VO
is allowed to run in.
"""
voMap = VoMapper(cp)
try:
queue_exclude = [i.strip() for i in cp.get("pbs", "queue_exclude").\
split(',')]
except:
queue_exclude = []
vo_queues = []
queueInfo = getQueueInfo(cp)
rvf_info = parseRvf('pbs.rvf')
rvf_queue_list = rvf_info.get('queue', {}).get('Values', None)
if rvf_queue_list:
rvf_queue_list = rvf_queue_list.split()
log.info("The RVF lists the following queues: %s." % ', '.join( \
rvf_queue_list))
for queue, qinfo in queueInfo.items():
if rvf_queue_list and queue not in rvf_queue_list:
continue
if queue in queue_exclude:
continue
volist = sets.Set(voList(cp, voMap))
try:
whitelist = [i.strip() for i in cp.get("pbs", "%s_whitelist" % \
queue).split(',')]
except:
whitelist = []
whitelist = sets.Set(whitelist)
try:
blacklist = [i.strip() for i in cp.get("pbs", "%s_blacklist" % \
queue).split(',')]
except:
blacklist = []
blacklist = sets.Set(blacklist)
if 'users' in qinfo or 'groups' in qinfo:
acl_vos = parseAclInfo(queue, qinfo, voMap)
volist.intersection_update(acl_vos)
# Force any VO in the whitelist to show up in the volist, even if it
# isn't in the acl_users / acl_groups
for vo in whitelist:
if vo not in volist:
volist.add(vo)
# Apply white and black lists
for vo in volist:
if (vo in blacklist or "*" in blacklist) and ((len(whitelist) == 0)\
or vo not in whitelist):
continue
vo_queues.append((vo, queue))
return vo_queues
def getVoQueues(cp):
"""
Determine the (vo, queue) tuples for this site. This allows for central
configuration of which VOs are advertised.
Sites will be able to blacklist queues they don't want to advertise,
whitelist certain VOs for a particular queue, and blacklist VOs from queues.
@param cp: Site configuration
@returns: A list of (vo, queue) tuples representing the queues each VO
is allowed to run in.
"""
voMap = VoMapper(cp)
try:
queue_exclude = [i.strip() for i in cp.get("pbs", "queue_exclude").\
split(',')]
except:
queue_exclude = []
vo_queues = []
queueInfo = getQueueInfo(cp)
rvf_info = parseRvf('pbs.rvf')
rvf_queue_list = rvf_info.get('queue', {}).get('Values', None)
if rvf_queue_list:
rvf_queue_list = rvf_queue_list.split()
log.info("The RVF lists the following queues: %s." % ', '.join( \
rvf_queue_list))
for queue, qinfo in queueInfo.items():
if rvf_queue_list and queue not in rvf_queue_list:
continue
if queue in queue_exclude:
continue
volist = sets.Set(voList(cp, voMap))
try:
whitelist = [i.strip() for i in cp.get("pbs", "%s_whitelist" % \
queue).split(',')]
except:
whitelist = []
whitelist = sets.Set(whitelist)
try:
blacklist = [i.strip() for i in cp.get("pbs", "%s_blacklist" % \
queue).split(',')]
except:
blacklist = []
blacklist = sets.Set(blacklist)
if 'users' in qinfo or 'groups' in qinfo:
acl_vos = parseAclInfo(queue, qinfo, voMap)
volist.intersection_update(acl_vos)
# Force any VO in the whitelist to show up in the volist, even if it
# isn't in the acl_users / acl_groups
for vo in whitelist:
if vo not in volist:
volist.add(vo)
# Apply white and black lists
for vo in volist:
if (vo in blacklist or "*" in blacklist) and ((len(whitelist) == 0)\
or vo not in whitelist):
continue
vo_queues.append((vo, queue))
return vo_queues
def main():
log.info('Starting CREAM service provider')
try:
cp = config()
serviceID = buildServiceID(cp)
siteID = cp_get(cp, "site", "unique_name", 'UNKNOWN_SITE')
serviceName = '%s-CREAM' % siteID
creamVersion = getCreamVersion(cp)
endpoint = 'https://%s:8443/ce-cream/services' % cp_get(cp, "ce", "name", 'UNKNOWN_CE')
allVOs = voList(cp)
acbr = ''
owner = ''
log.debug('CREAM VOs are %s' % allVOs)
if not allVOs:
log.error("No VOs supported!")
acbr = '__GIP_DELETEME'
else:
acbr = '\n'.join(['GlueServiceAccessControlBaseRule: %s\n' \
'GlueServiceAccessControlBaseRule: VO:%s' % (vo, vo) for vo in allVOs])
owner = '\n' + '\n'.join(['GlueServiceOwner: %s' % vo for vo in allVOs]) # owner needs an extra prepended newline
pid = -1
startTime = 'Not Applicable'
serviceStatus = 'Not OK'
serviceStatusInfo = 'Could not find tomcat process'
try:
(startTime, pid) = getStartTimeAndPid(cp)
serviceStatus = 'OK'
serviceStatusInfo = 'Tomcat (%d) is running' % pid
except:
log.error('Could not locate tomcat process (pgrep -f "org.apache.catalina.startup.Bootstrap start"'
' probably failed to return any output!)')
info = {'serviceID': serviceID,
'serviceType': 'org.glite.ce.CREAM',
'serviceName': serviceName,
'version': creamVersion,
'endpoint': endpoint,
'semantics': 'https://edms.cern.ch/document/595770',
'owner': owner,
'url': '__GIP_DELETEME', # deprecated
'uri': '__GIP_DELETEME', # deprecated
'status': serviceStatus,
'statusInfo': serviceStatusInfo,
'wsdl': 'http://grid.pd.infn.it/cream/wsdl/org.glite.ce-cream_service.wsdl',
'startTime': startTime,
'siteID': siteID,
'acbr': acbr
}
template = getTemplate("GlueService", "GlueServiceUniqueID")
printTemplate(template, info)
except Exception, e:
sys.stdout = sys.stderr
log.error(e)
raise
def getVoQueues(cp):
voMap = VoMapper(cp)
try:
queue_exclude = [
i.strip() for i in cp.get("sge", "queue_exclude").split(',')
]
except:
queue_exclude = []
# SGE has a special "waiting" queue -- ignore it.
queue_exclude.append('waiting')
vo_queues = []
queue_list, q = getQueueInfo(cp)
rvf_info = parseRvf('sge.rvf')
rvf_queue_list = rvf_info.get('queue', {}).get('Values', None)
if rvf_queue_list:
rvf_queue_list = rvf_queue_list.split()
log.info("The RVF lists the following queues: %s." % ', '.join( \
rvf_queue_list))
else:
log.warning("Unable to load a RVF file for SGE.")
for queue, qinfo in queue_list.items():
if rvf_queue_list and queue not in rvf_queue_list:
continue
if queue in queue_exclude:
continue
volist = sets.Set(voList(cp, voMap))
try:
whitelist = [
i.strip()
for i in cp.get("sge", "%s_whitelist" % queue).split(',')
]
except:
whitelist = []
whitelist = sets.Set(whitelist)
try:
blacklist = [
i.strip()
for i in cp.get("sge", "%s_blacklist" % queue).split(',')
]
except:
blacklist = []
blacklist = sets.Set(blacklist)
if 'user_list' in qinfo:
acl_vos = parseAclInfo(queue, qinfo, voMap)
if acl_vos:
volist.intersection_update(acl_vos)
for vo in volist:
if (vo in blacklist or "*" in blacklist) and ((len(whitelist) == 0)\
or vo not in whitelist):
continue
vo_queues.append((vo, queue))
return vo_queues
def test_voList(self):
"""
Make sure voList does indeed load up the correct VOs.
"""
cp = ConfigParser.ConfigParser()
cp.add_section("vo")
cp.set("vo", "vo_blacklist", "ilc")
cp.set("vo", "vo_whitelist", "vo1")
cp.set("vo","user_vo_map", "test_configs/fermigrid-osg-user-vo-map.txt")
vos = voList(cp)
vos = sets.Set(vos)
diff = vos.symmetric_difference(fermigrid_vos)
self.failIf(diff, msg="Difference between voList output and desired " \
"output: %s." % ', '.join(diff))
def test_many_sa(self):
vos = voList(self.cp)
for sa in self.sas:
print sa
rules = sa.glue['SAAccessControlBaseRule']
for rule in rules:
if rule.startswith("VO:"):
rule = rule[3:]
if rule in vos:
vos.remove(rule)
else:
self.fail("Unknown VO supported: %s." % rule)
self.failIf(vos, msg="VOs with no classicSE support: %s" % \
', '.join(vos))
def test_many_sa(self):
vos = voList(self.cp)
for sa in self.sas:
print sa
rules = sa.glue['SAAccessControlBaseRule']
for rule in rules:
if rule.startswith("VO:"):
rule = rule[3:]
if rule in vos:
vos.remove(rule)
else:
self.fail("Unknown VO supported: %s." % rule)
self.failIf(vos, msg="VOs with no classicSE support: %s" % \
', '.join(vos))
def getVoQueues(cp):
voMap = VoMapper(cp)
try:
queue_exclude = [i.strip() for i in cp.get("sge",
"queue_exclude").split(',')]
except:
queue_exclude = []
# SGE has a special "waiting" queue -- ignore it.
queue_exclude.append('waiting')
vo_queues = []
queue_list, q = getQueueInfo(cp)
rvf_info = parseRvf('sge.rvf')
rvf_queue_list = rvf_info.get('queue', {}).get('Values', None)
if rvf_queue_list:
rvf_queue_list = rvf_queue_list.split()
log.info("The RVF lists the following queues: %s." % ', '.join( \
rvf_queue_list))
else:
log.warning("Unable to load a RVF file for SGE.")
for queue, qinfo in queue_list.items():
if rvf_queue_list and queue not in rvf_queue_list:
continue
if queue in queue_exclude:
continue
volist = sets.Set(voList(cp, voMap))
try:
whitelist = [i.strip() for i in cp.get("sge",
"%s_whitelist" % queue).split(',')]
except:
whitelist = []
whitelist = sets.Set(whitelist)
try:
blacklist = [i.strip() for i in cp.get("sge",
"%s_blacklist" % queue).split(',')]
except:
blacklist = []
blacklist = sets.Set(blacklist)
if 'user_list' in qinfo:
acl_vos = parseAclInfo(queue, qinfo, voMap)
if acl_vos:
volist.intersection_update(acl_vos)
for vo in volist:
if (vo in blacklist or "*" in blacklist) and ((len(whitelist) == 0)\
or vo not in whitelist):
continue
vo_queues.append((vo, queue))
return vo_queues
def test_one_sa(self):
vos = voList(self.cp)
len_vos = len(vos)
has_special_sa = False
for sa in self.sas:
if len(sa.glue['SAAccessControlBaseRule']) == len_vos:
has_special_sa = True
for vo in vos:
has_vo_entry = False
for entry in sa.glue['SAAccessControlBaseRule']:
if entry.find(vo) >= 0:
has_vo_entry = True
break
self.failUnless(has_vo_entry, "VO %s isn't in ACBR." % vo)
self.failUnless(has_special_sa, "Can't find group SA for classicSE")
def test_one_sa(self):
vos = voList(self.cp)
len_vos = len(vos)
has_special_sa = False
for sa in self.sas:
if len(sa.glue['SAAccessControlBaseRule']) == len_vos:
has_special_sa = True
for vo in vos:
has_vo_entry = False
for entry in sa.glue['SAAccessControlBaseRule']:
if entry.find(vo) >= 0:
has_vo_entry = True
break
self.failUnless(has_vo_entry, "VO %s isn't in ACBR." % vo)
self.failUnless(has_special_sa, "Can't find group SA for classicSE")
def getApplicationsV1(cp):
"""
Retrieves the applications in the new "v1" format; it looks in
- $OSG_APP/etc/grid3-locations.txt
- $OSG_APP/etc/<vo>/locations-v1.txt
Here, $OSG_APP/etc may be overridden by the config variable
gip.software_dir.
One directory is created per VO, owned by root, world-writable, and set to
sticky. The VO names will be determined by the current methods (using
voList). GIP will then scan the directory $OSG_APP/etc/<vo> for all the
VO names known to GIP, and read only a file named locations-v1.txt. It is
assumed that locations-v1.txt is the same format as the grid3-locations.txt.
"""
app_dir = cp_get(cp, "osg_dirs", "app", "/UNKNOWN")
base_path = os.path.join(app_dir, "etc")
base_path = cp_get(cp, "gip", "software_dir", base_path)
locations = {}
for vo in voList(cp):
vo_dir = os.path.join(base_path, vo)
vo_dir = os.path.expandvars(vo_dir)
vo_path = os.path.join(vo_dir, 'locations-v1.txt')
if not os.path.exists(vo_path):
continue
try:
fp = open(vo_path, 'r')
except:
log.warning("Unable to read VO application file: %s" % vo_path)
continue
for line in fp:
line = line.strip()
info = line.split()
# Skip blank lines and comments
if len(line) == 0 or info[0].startswith('#'):
continue
if len(info) != 3:
log.warning("Invalid line: %s" % line)
continue
if info[1].startswith('#') or info[1].startswith('$'):
info[1] = 'UNDEFINED'
info = {'locationName': info[0], 'version': info[1], 'path':info[2]}
info['locationId'] = info['locationName']
locations = _addLocation(locations, info)
return locations
def getVoQueues(queueInfo, cp):
"""
Determine the (vo, queue) tuples for this site. This allows for central
configuration of which VOs are advertised.
Sites will be able to blacklist queues they don't want to advertise,
whitelist certain VOs for a particular queue, and blacklist VOs from queues.
@param cp: Site configuration
@returns: A list of (vo, queue) tuples representing the queues each VO
is allowed to run in.
"""
voMap = VoMapper(cp)
try:
queue_exclude = [i.strip() for i in cp.get("lsf", "queue_exclude").\
split(',')]
except:
queue_exclude = []
rvf_info = parseRvf('lsf.rvf')
rvf_queue_list = rvf_info.get('queue', {}).get('Values', None)
if rvf_queue_list:
rvf_queue_list = rvf_queue_list.split()
log.info("The RVF lists the following queues: %s." % ', '.join( \
rvf_queue_list))
vo_queues= []
for queue in queueInfo:
if queue in queue_exclude:
continue
if rvf_queue_list and queue not in rvf_queue_list:
continue
try:
whitelist = [i.strip() for i in cp.get("lsf", "%s_whitelist" % \
queue).split(',')]
except:
whitelist = []
try:
blacklist = [i.strip() for i in cp.get("lsf", "%s_blacklist" % \
queue).split(',')]
except:
blacklist = []
for vo in queueInfo[queue].get('vos', voList(cp, voMap)):
if (vo in blacklist or "*" in blacklist) and ((len(whitelist) == 0)\
or vo not in whitelist):
continue
vo_queues.append((vo, queue))
return vo_queues
def getApplicationsV1(cp):
"""
Retrieves the applications in the new "v1" format; it looks in
- $OSG_APP/etc/grid3-locations.txt
- $OSG_APP/etc/<vo>/locations-v1.txt
Here, $OSG_APP/etc may be overridden by the config variable
gip.software_dir.
One directory is created per VO, owned by root, world-writable, and set to
sticky. The VO names will be determined by the current methods (using
voList). GIP will then scan the directory $OSG_APP/etc/<vo> for all the
VO names known to GIP, and read only a file named locations-v1.txt. It is
assumed that locations-v1.txt is the same format as the grid3-locations.txt.
"""
app_dir = cp_get(cp, "osg_dirs", "app", "/UNKNOWN")
base_path = os.path.join(app_dir, "etc")
base_path = cp_get(cp, "gip", "software_dir", base_path)
locations = {}
for vo in voList(cp):
vo_dir = os.path.join(base_path, vo)
vo_dir = os.path.expandvars(vo_dir)
vo_path = os.path.join(vo_dir, 'locations-v1.txt')
if not os.path.exists(vo_path):
continue
try:
fp = open(vo_path, 'r')
except:
log.warning("Unable to read VO application file: %s" % vo_path)
continue
for line in fp:
line = line.strip()
info = line.split()
# Skip blank lines and comments
if len(line) == 0 or info[0].startswith('#'):
continue
if len(info) != 3:
log.warning("Invalid line: %s" % line)
continue
if info[1].startswith('#') or info[1].startswith('$'):
info[1] = 'UNDEFINED'
info = {'locationName': info[0], 'version': info[1], 'path':info[2]}
info['locationId'] = info['locationName']
locations = _addLocation(locations, info)
return locations
def determineGroupVOsFromConfig(cp, group, voMap):
"""
Given a group name and the config object, determine the VOs which are
allowed in that group; this is based solely on the config files.
"""
# This is the old behavior. Base everything on (groupname)_vos
bycp = cp_get(cp, "condor", "%s_vos" % group, None)
if bycp:
return [i.strip() for i in bycp.split(',')]
# This is the new behavior. Base everything on (groupname)_blacklist and
# (groupname)_whitelist. Done to mimic the PBS configuration.
volist = sets.Set(voList(cp, voMap))
try:
whitelist = [i.strip() for i in cp.get("condor", "%s_whitelist" % \
group).split(',')]
except:
whitelist = []
whitelist = sets.Set(whitelist)
try:
blacklist = [i.strip() for i in cp.get("condor", "%s_blacklist" % \
group).split(',')]
except:
blacklist = []
blacklist = sets.Set(blacklist)
# Return None if there's no explicit white/black list setting.
if len(whitelist) == 0 and len(blacklist) == 0:
return None
# Force any VO in the whitelist to show up in the volist, even if it
# isn't in the acl_users / acl_groups
for vo in whitelist:
if vo not in volist:
volist.add(vo)
# Apply white and black lists
results = sets.Set()
for vo in volist:
if (vo in blacklist or "*" in blacklist) and ((len(whitelist) == 0)\
or vo not in whitelist):
continue
results.add(vo)
return list(results)
def determineGroupVOsFromConfig(cp, group, voMap):
"""
Given a group name and the config object, determine the VOs which are
allowed in that group; this is based solely on the config files.
"""
# This is the old behavior. Base everything on (groupname)_vos
bycp = cp_get(cp, "condor", "%s_vos" % group, None)
if bycp:
return [i.strip() for i in bycp.split(',')]
# This is the new behavior. Base everything on (groupname)_blacklist and
# (groupname)_whitelist. Done to mimic the PBS configuration.
volist = sets.Set(voList(cp, voMap))
try:
whitelist = [i.strip() for i in cp.get("condor", "%s_whitelist" % \
group).split(',')]
except:
whitelist = []
whitelist = sets.Set(whitelist)
try:
blacklist = [i.strip() for i in cp.get("condor", "%s_blacklist" % \
group).split(',')]
except:
blacklist = []
blacklist = sets.Set(blacklist)
# Return None if there's no explicit white/black list setting.
if len(whitelist) == 0 and len(blacklist) == 0:
return None
# Force any VO in the whitelist to show up in the volist, even if it
# isn't in the acl_users / acl_groups
for vo in whitelist:
if vo not in volist:
volist.add(vo)
# Apply white and black lists
results = sets.Set()
for vo in volist:
if (vo in blacklist or "*" in blacklist) and ((len(whitelist) == 0)\
or vo not in whitelist):
continue
results.add(vo)
return list(results)
def check_sa_1_ldif(self, entries, cp):
found_se = False
for entry in entries:
if not ('GlueSA' in entry.objectClass and \
entry.glue.get('ChunkKey', [''])[0] == \
'GlueSEUniqueID=srm.unl.edu' and
entry.glue.get('SALocalID', [''])[0] == 'default'):
continue
found_se = True
self.failUnless(entry.glue['SARoot'][0] == '/')
self.failUnless(entry.glue['SAPath'][0] == '/pnfs/unl.edu/data4/')
self.failUnless(entry.glue['SAType'][0] == 'permanent')
self.failUnless(entry.glue['SARetentionPolicy'][0] == 'replica')
self.failUnless(entry.glue['SAAccessLatency'][0] == 'online')
self.failUnless(entry.glue['SAExpirationMode'][0] == 'neverExpire')
self.failUnless(entry.glue['SACapability'][0] == 'file storage')
self.failUnless(entry.glue['SAPolicyFileLifeTime'][0] == \
'permanent')
for vo in voList(cp):
self.failUnless(vo in entry.glue['SAAccessControlBaseRule'])
self.failUnless("VO:%s" % vo in \
entry.glue['SAAccessControlBaseRule'])
self.failUnless(found_se, msg="GlueSA entry for srm.unl.edu missing.")
def check_sa_1_ldif(self, entries, cp):
found_se = False
for entry in entries:
if not ('GlueSA' in entry.objectClass and \
entry.glue.get('ChunkKey', [''])[0] == \
'GlueSEUniqueID=srm.unl.edu' and
entry.glue.get('SALocalID', [''])[0] == 'default'):
continue
found_se = True
self.failUnless(entry.glue['SARoot'][0] == '/')
self.failUnless(entry.glue['SAPath'][0] == '/pnfs/unl.edu/data4/')
self.failUnless(entry.glue['SAType'][0] == 'permanent')
self.failUnless(entry.glue['SARetentionPolicy'][0] == 'replica')
self.failUnless(entry.glue['SAAccessLatency'][0] == 'online')
self.failUnless(entry.glue['SAExpirationMode'][0] == 'neverExpire')
self.failUnless(entry.glue['SACapability'][0] == 'file storage')
self.failUnless(entry.glue['SAPolicyFileLifeTime'][0] == \
'permanent')
for vo in voList(cp):
self.failUnless(vo in entry.glue['SAAccessControlBaseRule'])
self.failUnless("VO:%s" % vo in \
entry.glue['SAAccessControlBaseRule'])
self.failUnless(found_se, msg="GlueSA entry for srm.unl.edu missing.")
total_nodes, claimed, unclaimed = 0, 0, 0
vo_map = VoMapper(cp)
# Determine the information about the current jobs in queue
try:
jobs_info = getJobsInfo(vo_map, cp)
except Exception, e:
log.exception(e)
jobs_info = {
'default':
dict([(vo, {
'running': 0,
'idle': 0,
'held': 0
}) for vo in voList(cp)])
}
# Determine the group information, if there are any Condor groups
try:
groupInfo = getGroupInfo(vo_map, cp)
except Exception, e:
log.exception(e)
# Default to no groups.
groupInfo = {}
log.debug("Group Info: %s" % str(groupInfo))
# Accumulate the entire statistics, instead of breaking down by VO.
running, idle, held = 0, 0, 0
for group, ginfo in jobs_info.items():
# Get the node information for condor
try:
total_nodes, claimed, unclaimed = parseNodes(cp)
except Exception, e:
log.exception(e)
total_nodes, claimed, unclaimed = 0, 0, 0
vo_map = VoMapper(cp)
# Determine the information about the current jobs in queue
try:
jobs_info = getJobsInfo(vo_map, cp)
except Exception, e:
log.exception(e)
jobs_info = {'default': dict([(vo, {'running': 0, 'idle': 0,
'held': 0}) for vo in voList(cp)])}
# Determine the group information, if there are any Condor groups
try:
groupInfo = getGroupInfo(vo_map, cp)
except Exception, e:
log.exception(e)
# Default to no groups.
groupInfo = {}
log.debug("Group Info: %s" % str(groupInfo))
# Accumulate the entire statistics, instead of breaking down by VO.
running, idle, held = 0, 0, 0
for group, ginfo in jobs_info.items():
for vo, info in ginfo.items():
@param cp: Site configuration
@returns: List of strings containing the queue names.
"""
doPath(cp)
vo_map = VoMapper(cp)
# Determine the group information, if there are any Condor groups
try:
groupInfo = getGroupInfo(vo_map, cp)
except Exception, e:
log.exception(e)
# Default to no groups.
groupInfo = {}
# filter out queues that don't match a VO
allVos = sets.Set(voList(cp))
for group in groupInfo.keys():
vos = sets.Set(groupInfo[group]['vos'])
if not vos.intersection(allVos):
log.debug('Filtering out %s in getQueueList -- no matching VO' % groupInfo[group]['vos'])
del groupInfo[group]
# Set up the "default" group with all the VOs which aren't already in a
# group
if not defaultGroupIsExcluded(cp):
groupInfo['default'] = {'prio': 999999, 'quota': 999999, 'vos': sets.Set()}
all_group_vos = []
for val in groupInfo.values():
all_group_vos.extend(val['vos'])
defaultVoList = voList(cp, vo_map=vo_map)
def print_VOViewLocal(cp):
"""
Print the GLUE VOView entity; shows the VO's view of the condor batch
system.
Config options used:
* ce.name. The human-readable name of the ce.
* condor.status. The status of condor; defaults to "Production"
* osg_dirs.app. The $OSG_APP directory; defaults to "/Unknown"
* osg_dirs.data. The $OSG_DATA directory; defaults to "/Unknown"
* se.name. The human-readable name of the closest SE.
@param cp: The GIP configuration object
@type cp: ConfigParser.ConfigParser
"""
VOView = getTemplate("GlueCE", "GlueVOViewLocalID")
ce_name = cp_get(cp, "ce", "name", "")
#status = cp_get(cp, "condor", "status", "Production")
#condorVersion = getLrmsInfo(cp)
total_nodes, _, unclaimed = parseNodes(cp)
vo_map = VoMapper(cp)
jobs_info = getJobsInfo(vo_map, cp)
groupInfo = getGroupInfo(vo_map, cp)
# Add in the default group
all_group_vos = []
total_assigned = 0
for key, val in groupInfo.items():
if key == 'default':
continue
all_group_vos.extend(val['vos'])
total_assigned += val.get('quota', 0)
all_vos = sets.Set(voList(cp))
defaultVoList = [i for i in all_vos if i not in all_group_vos]
if 'default' not in groupInfo:
groupInfo['default'] = {}
groupInfo['default']['vos'] = defaultVoList
if total_nodes > total_assigned:
log.info("There are %i assigned job slots out of %i total; assigning" \
" the rest to the default group." % (total_assigned, total_nodes))
groupInfo['default']['quota'] = total_nodes-total_assigned
else:
log.warning("More assigned nodes (%i) than actual nodes (%i)!" % \
(total_assigned, total_nodes))
if defaultGroupIsExcluded(cp):
if groupInfo.has_key('default'):
del groupInfo['default']
for group in groupInfo:
jinfo = jobs_info.get(group, {})
vos = sets.Set(groupInfo[group].get('vos', [group]))
vos.update(jinfo.keys())
vos.intersection_update(all_vos)
# Enforce invariants
# VO_FREE_SLOTS <= CE_FREE_SLOTS
# VO_FREE_SLOTS <= CE_ASSIGNED - VO_RUNNING
# This code determines CE_ASSIGNED
ginfo = groupInfo[group]
if ginfo.get("quota", 0) > 0:
assigned = ginfo.get("quota", 0)
else:
assigned = total_nodes
log.debug("All VOs for %s: %s" % (group, ", ".join(vos)))
ce_unique_id = buildCEUniqueID(cp, ce_name, 'condor', group)
max_wall = cp_getInt(cp, "condor", "max_wall", 1440)
myrunning = sum([i.get('running', 0) for i in jinfo.values()], 0)
assigned = max(assigned, myrunning)
for vo in vos:
acbr = 'VO:%s' % vo
info = jinfo.get(vo.lower(), {"running": 0, "idle": 0, "held": 0})
ert, wrt = responseTimes(cp, info["running"], info["idle"] + \
info["held"], max_job_time=max_wall*60)
free = min(unclaimed, assigned-myrunning,
assigned-int(info['running']))
free = int(free)
waiting = int(info["idle"]) + int(info["held"])
if waiting > cp_getInt(cp, 'condor', 'idle_slack', '10'):
free = 0
info = {"vo" : vo,
"acbr" : acbr,
"ceUniqueID" : ce_unique_id,
"voLocalID" : vo,
"ce_name" : ce_name,
"job_manager" : 'condor',
"queue" : vo,
"running" : info["running"],
# Held jobs are included as "waiting" since the definition is:
# Number of jobs that are in a state different than running
"waiting" : waiting,
"total" : info["running"] + info["idle"] + info["held"],
"free_slots" : free,
"job_slots" : int(total_nodes),
"ert" : ert,
"wrt" : wrt,
"default_se" : getDefaultSE(cp),
'app' : cp_get(cp, 'osg_dirs', 'app', '/Unknown'),
"data" : cp_get(cp, "osg_dirs", "data", "/Unknown"),
}
printTemplate(VOView, info)
Returns a list of all the queue names that are supported.
@param cp: Site configuration
@returns: List of strings containing the queue names.
"""
vo_map = VoMapper(cp)
# Determine the group information, if there are any Condor groups
try:
groupInfo = getGroupInfo(vo_map, cp)
except Exception, e:
log.exception(e)
# Default to no groups.
groupInfo = {}
# filter out queues that don't match a VO
allVos = sets.Set(voList(cp))
for group in groupInfo.keys():
vos = sets.Set(groupInfo[group]['vos'])
if not vos.intersection(allVos):
log.debug('Filtering out %s in getQueueList -- no matching VO' % groupInfo[group]['vos'])
del groupInfo[group]
# Set up the "default" group with all the VOs which aren't already in a
# group
if not defaultGroupIsExcluded(cp):
groupInfo['default'] = {'prio': 999999, 'quota': 999999, 'vos': sets.Set()}
all_group_vos = []
for val in groupInfo.values():
all_group_vos.extend(val['vos'])
defaultVoList = voList(cp, vo_map=vo_map)
def main():
log.info('Starting CREAM service provider')
try:
cp = config()
serviceID = buildServiceID(cp)
siteID = cp_get(cp, "site", "unique_name", 'UNKNOWN_SITE')
serviceName = '%s-CREAM' % siteID
creamVersion = getCreamVersion(cp)
endpoint = 'https://%s:8443/ce-cream/services' % cp_get(
cp, "ce", "name", 'UNKNOWN_CE')
allVOs = voList(cp)
acbr = ''
owner = ''
log.debug('CREAM VOs are %s' % allVOs)
if not allVOs:
log.error("No VOs supported!")
acbr = '__GIP_DELETEME'
else:
acbr = '\n'.join(['GlueServiceAccessControlBaseRule: %s\n' \
'GlueServiceAccessControlBaseRule: VO:%s' % (vo, vo) for vo in allVOs])
owner = '\n' + '\n'.join(
['GlueServiceOwner: %s' % vo
for vo in allVOs]) # owner needs an extra prepended newline
pid = -1
startTime = 'Not Applicable'
serviceStatus = 'Not OK'
serviceStatusInfo = 'Could not find tomcat process'
try:
(startTime, pid) = getStartTimeAndPid(cp)
serviceStatus = 'OK'
serviceStatusInfo = 'Tomcat (%d) is running' % pid
except:
log.error(
'Could not locate tomcat process (pgrep -f "org.apache.catalina.startup.Bootstrap start"'
' probably failed to return any output!)')
info = {
'serviceID': serviceID,
'serviceType': 'org.glite.ce.CREAM',
'serviceName': serviceName,
'version': creamVersion,
'endpoint': endpoint,
'semantics': 'https://edms.cern.ch/document/595770',
'owner': owner,
'url': '__GIP_DELETEME', # deprecated
'uri': '__GIP_DELETEME', # deprecated
'status': serviceStatus,
'statusInfo': serviceStatusInfo,
'wsdl':
'http://grid.pd.infn.it/cream/wsdl/org.glite.ce-cream_service.wsdl',
'startTime': startTime,
'siteID': siteID,
'acbr': acbr
}
template = getTemplate("GlueService", "GlueServiceUniqueID")
printTemplate(template, info)
except Exception, e:
sys.stdout = sys.stderr
log.error(e)
raise
def print_VOViewLocal(cp):
"""
Print the GLUE VOView entity; shows the VO's view of the condor batch
system.
Config options used:
* ce.name. The human-readable name of the ce.
* condor.status. The status of condor; defaults to "Production"
* osg_dirs.app. The $OSG_APP directory; defaults to "/Unknown"
* osg_dirs.data. The $OSG_DATA directory; defaults to "/Unknown"
* se.name. The human-readable name of the closest SE.
@param cp: The GIP configuration object
@type cp: ConfigParser.ConfigParser
"""
VOView = getTemplate("GlueCE", "GlueVOViewLocalID")
ce_name = cp_get(cp, "ce", "name", "")
#status = cp_get(cp, "condor", "status", "Production")
#condorVersion = getLrmsInfo(cp)
total_nodes, _, unclaimed = parseNodes(cp)
vo_map = VoMapper(cp)
jobs_info = getJobsInfo(vo_map, cp)
groupInfo = getGroupInfo(vo_map, cp)
# Add in the default group
all_group_vos = []
total_assigned = 0
for key, val in groupInfo.items():
if key == 'default':
continue
all_group_vos.extend(val['vos'])
total_assigned += val.get('quota', 0)
all_vos = sets.Set(voList(cp))
defaultVoList = [i for i in all_vos if i not in all_group_vos]
if 'default' not in groupInfo:
groupInfo['default'] = {}
groupInfo['default']['vos'] = defaultVoList
if total_nodes > total_assigned:
log.info("There are %i assigned job slots out of %i total; assigning" \
" the rest to the default group." % (total_assigned, total_nodes))
groupInfo['default']['quota'] = total_nodes - total_assigned
else:
log.warning("More assigned nodes (%i) than actual nodes (%i)!" % \
(total_assigned, total_nodes))
if defaultGroupIsExcluded(cp):
if groupInfo.has_key('default'):
del groupInfo['default']
for group in groupInfo:
jinfo = jobs_info.get(group, {})
vos = sets.Set(groupInfo[group].get('vos', [group]))
vos.update(jinfo.keys())
vos.intersection_update(all_vos)
# Enforce invariants
# VO_FREE_SLOTS <= CE_FREE_SLOTS
# VO_FREE_SLOTS <= CE_ASSIGNED - VO_RUNNING
# This code determines CE_ASSIGNED
ginfo = groupInfo[group]
if ginfo.get("quota", 0) > 0:
assigned = ginfo.get("quota", 0)
else:
assigned = total_nodes
log.debug("All VOs for %s: %s" % (group, ", ".join(vos)))
ce_unique_id = buildCEUniqueID(cp, ce_name, 'condor', group)
max_wall = cp_getInt(cp, "condor", "max_wall", 1440)
myrunning = sum([i.get('running', 0) for i in jinfo.values()], 0)
assigned = max(assigned, myrunning)
for vo in vos:
acbr = 'VO:%s' % vo
info = jinfo.get(vo.lower(), {"running": 0, "idle": 0, "held": 0})
ert, wrt = responseTimes(cp, info["running"], info["idle"] + \
info["held"], max_job_time=max_wall*60)
free = min(unclaimed, assigned - myrunning,
assigned - int(info['running']))
free = int(free)
waiting = int(info["idle"]) + int(info["held"])
if waiting > cp_getInt(cp, 'condor', 'idle_slack', '10'):
free = 0
info = {
"vo": vo,
"acbr": acbr,
"ceUniqueID": ce_unique_id,
"voLocalID": vo,
"ce_name": ce_name,
"job_manager": 'condor',
"queue": vo,
"running": info["running"],
# Held jobs are included as "waiting" since the definition is:
# Number of jobs that are in a state different than running
"waiting": waiting,
"total": info["running"] + info["idle"] + info["held"],
"free_slots": free,
"job_slots": int(total_nodes),
"ert": ert,
"wrt": wrt,
"default_se": getDefaultSE(cp),
'app': cp_get(cp, 'osg_dirs', 'app', '/Unknown'),
"data": cp_get(cp, "osg_dirs", "data", "/Unknown"),
}
printTemplate(VOView, info)
