def handle_providers(entries, providers):
"""
Add the output from the providers to the list of the GIP entries.
This will match the DNs; if two DNs are repeated, then one will be thrown
out
@param entries: A list of LdapData objects
@param providers: A list of provider information dictionaries.
@returns: The altered entries list.
"""
provider_entries = []
for _, p_info in providers.items():
if 'output' in p_info:
fp = cStringIO.StringIO(p_info['output'])
provider_entries += read_ldap(fp, multi=True)
remove_entries = []
# Calculate all the duplicate entries, build a list of the ones
# to remove.
for entry in entries:
for p_entry in provider_entries:
if compareDN(entry, p_entry):
remove_entries.append(entry)
for entry in sets.Set(remove_entries):
log.debug("Removing entry %s" % entry)
try:
entries.remove(entry)
except ValueError:
pass
# Now add all the new entries from the providers
for p_entry in provider_entries:
entries.append(p_entry)
return entries
def setUpLDAP(self, multi=False, filename=None):
if filename != None:
self.filename = filename
command = "$GIP_LOCATION/providers/site_topology.py --config=%s" % self.filename
stdout = os.popen(command)
entries = read_ldap(stdout, multi=multi)
return entries
def handle_providers(entries, providers):
"""
Add the output from the providers to the list of the GIP entries.
This will match the DNs; if two DNs are repeated, then one will be thrown
out
@param entries: A list of LdapData objects
@param providers: A list of provider information dictionaries.
@returns: The altered entries list.
"""
provider_entries = []
for provider, p_info in providers.items():
if 'output' in p_info:
fp = cStringIO.StringIO(p_info['output'])
provider_entries += read_ldap(fp, multi=True)
remove_entries = []
# Calculate all the duplicate entries, build a list of the ones
# to remove.
for entry in entries:
for p_entry in provider_entries:
if compareDN(entry, p_entry):
remove_entries.append(entry)
for entry in sets.Set(remove_entries):
log.debug("Removing entry %s" % entry)
try:
entries.remove(entry)
except ValueError:
pass
# Now add all the new entries from the providers
for p_entry in provider_entries:
entries.append(p_entry)
return entries
def test_max_queuable_22(self):
"""
Regression test for the max_queuable attribute. Ticket #22.
"""
os.environ['GIP_TESTING'] = '1'
path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg-info-provider-pbs.py --config=test_configs/red.conf")
fd = os.popen(path)
entries = read_ldap(fd)
self.failUnless(fd.close() == None)
has_default_ce = False
has_lcgadmin_ce = False
for entry in entries:
if 'GlueCE' in entry.objectClass and \
entry.glue['CEUniqueID'] == 'red.unl.edu:2119/jobmanager' \
'-pbs-default':
self.failUnless(entry.glue['CEPolicyMaxTotalJobs'] == '2000')
has_default_ce = True
if 'GlueCE' in entry.objectClass and \
entry.glue['CEUniqueID'] == 'red.unl.edu:2119/jobmanager' \
'-pbs-lcgadmin':
self.failUnless(entry.glue['CEPolicyMaxWaitingJobs'] == '183')
has_lcgadmin_ce = True
self.failUnless(has_default_ce,
msg="Default queue's CE was not found!")
self.failUnless(has_default_ce,
msg="lcgadmin queue's CE was not found!")
def test_max_queuable_22(self):
"""
Regression test for the max_queuable attribute. Ticket #22.
"""
os.environ['GIP_TESTING'] = '1'
path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg-info-provider-pbs.py --config=test_configs/red.conf")
fd = os.popen(path)
entries = read_ldap(fd)
self.failUnless(fd.close() == None)
has_default_ce = False
has_lcgadmin_ce = False
for entry in entries:
if 'GlueCE' in entry.objectClass and \
entry.glue['CEUniqueID'] == 'red.unl.edu:2119/jobmanager' \
'-pbs-default':
self.failUnless(entry.glue['CEPolicyMaxTotalJobs'] == '2000')
has_default_ce = True
if 'GlueCE' in entry.objectClass and \
entry.glue['CEUniqueID'] == 'red.unl.edu:2119/jobmanager' \
'-pbs-lcgadmin':
self.failUnless(entry.glue['CEPolicyMaxWaitingJobs'] == '183')
has_lcgadmin_ce = True
self.failUnless(has_default_ce, msg="Default queue's CE was not found!")
self.failUnless(has_default_ce, msg="lcgadmin queue's CE was not found!")
def setUpLDAP(self, multi=False, filename=None):
if filename != None:
self.filename = filename
command = "$GIP_LOCATION/providers/site_topology.py --config=%s" % \
self.filename
stdout = os.popen(command)
entries = read_ldap(stdout, multi=multi)
return entries
def run_compareDN(self, string1, string2, should_match = False):
orig_entries = []
fp = cStringIO.StringIO(string1)
orig_entries += read_ldap(fp, multi=True)
other_entries = []
fp = cStringIO.StringIO(string2)
other_entries += read_ldap(fp, multi=True)
result = compareDN(orig_entries[0], other_entries[0])
if should_match:
self.failIf(not result, msg="compareDN returned False when it should have "\
"returned True.\n\nDN1: %s \nDN2: %s" % \
(prettyDN(orig_entries[0].dn),prettyDN(other_entries[0].dn)))
else:
self.failIf(result, msg="compareDN returned True when it should have "\
"returned False.\n\nDN1: %s \nDN2: %s" % \
(prettyDN(orig_entries[0].dn),prettyDN(other_entries[0].dn)))
return result
def test_condorq_parsing(self):
"""
Test the condor_q -xml parsing for Condor format changes
"""
# Condor 7.3.2 and above
os.environ['GIP_TESTING'] = 'suffix=glow'
path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
"condor.py --config=test_configs/glow_condor.conf")
fd = os.popen(path)
entries = read_ldap(fd, multi=True)
self.assertEquals(fd.close(), None)
# Condor 7.3.1 and less
os.environ['GIP_TESTING'] = 'suffix=glow-condor72'
path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
"condor.py --config=test_configs/glow_condor.conf")
fd = os.popen(path)
entries = read_ldap(fd, multi=True)
self.assertEquals(fd.close(), None)
def test_site_entries(self):
"""
Checks the %s information.
""" % site
# Switch commands over to the site ones:
os.environ['GIP_TESTING'] = 'suffix=%s' % site
path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg-info-provider-pbs.py --config=test_configs/%s.conf" % site)
fd = os.popen(path)
entries = read_ldap(fd)
self.assertEquals(fd.close(), None)
def test_site_entries(self):
"""
Checks the %s information.
""" % site
# Switch commands over to the site ones:
os.environ['GIP_TESTING'] = 'suffix=%s' % site
path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg-info-provider-pbs.py --config=test_configs/%s.conf" % site)
fd = os.popen(path)
entries = read_ldap(fd)
self.assertEquals(fd.close(), None)
def test_condorq_parsing(self):
"""
Test the condor_q -xml parsing for Condor format changes
"""
# Condor 7.3.2 and above
os.environ['GIP_TESTING'] = 'suffix=glow'
path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
"condor.py --config=test_configs/glow_condor.conf")
fd = os.popen(path)
entries = read_ldap(fd, multi=True)
self.assertEquals(fd.close(), None)
# Condor 7.3.1 and less
os.environ['GIP_TESTING'] = 'suffix=glow-condor72'
path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
"condor.py --config=test_configs/glow_condor.conf")
fd = os.popen(path)
entries = read_ldap(fd, multi=True)
self.assertEquals(fd.close(), None)
def run_compareDN(self, string1, string2, should_match=False):
orig_entries = []
fp = cStringIO.StringIO(string1)
orig_entries += read_ldap(fp, multi=True)
other_entries = []
fp = cStringIO.StringIO(string2)
other_entries += read_ldap(fp, multi=True)
result = compareDN(orig_entries[0], other_entries[0])
if should_match:
self.failIf(not result, msg="compareDN returned False when it should have "\
"returned True.\n\nDN1: %s \nDN2: %s" % \
(prettyDN(orig_entries[0].dn),prettyDN(other_entries[0].dn)))
else:
self.failIf(result, msg="compareDN returned True when it should have "\
"returned False.\n\nDN1: %s \nDN2: %s" % \
(prettyDN(orig_entries[0].dn),prettyDN(other_entries[0].dn)))
return result
def test_condor_hierarchical_group(self):
"""
Make sure that condor can read and publish hierarchical groups too.
"""
ce = 'fnpcfg1.fnal.gov:2119/jobmanager-condor-group_nysgrid.sub1'
os.environ['GIP_TESTING'] = 'suffix=fnal'
path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
"condor.py --config=test_configs/fnal_condor.conf")
fd = os.popen(path)
entries = read_ldap(fd, multi=True)
entry = self.check_for_ce(ce, entries)
self.failUnless('33' in entry.glue['CEStateRunningJobs'], msg="Did not"\
" get hierarchical groups.")
def setUpLDAP(self, filename=None):
if filename != None:
self.filename = filename
os.environ['GIP_TESTING'] = "1"
cp = config(self.filename)
self.ces = getCEList(cp)
self.ses = getSEList(cp)
cese_provider_path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg_info_cesebind.py --config %s" % self.filename)
print >> sys.stderr, "Used command", cese_provider_path
fd = os.popen(cese_provider_path)
self.entries = read_ldap(fd, multi=True)
self.exit_status = fd.close()
def test_condor_hierarchical_group(self):
"""
Make sure that condor can read and publish hierarchical groups too.
"""
ce = 'fnpcfg1.fnal.gov:2119/jobmanager-condor-group_nysgrid.sub1'
os.environ['GIP_TESTING'] = 'suffix=fnal'
path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
"condor.py --config=test_configs/fnal_condor.conf")
fd = os.popen(path)
entries = read_ldap(fd, multi=True)
entry = self.check_for_ce(ce, entries)
self.failUnless('33' in entry.glue['CEStateRunningJobs'], msg="Did not"\
" get hierarchical groups.")
def test_multi_schedd_output(self):
"""
Make sure that condor submitter accounting groups spread over multiple
schedds (and hence in multiple ClassAds) are aggregated.
"""
ce = 'fnpcfg1.fnal.gov:2119/jobmanager-condor-group_nysgrid'
os.environ['GIP_TESTING'] = 'suffix=fnal'
path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
"condor.py --config=test_configs/fnal_condor.conf")
fd = os.popen(path)
entries = read_ldap(fd, multi=True)
entry = self.check_for_ce(ce, entries)
self.failUnless('25' in entry.glue['CEStateRunningJobs'], msg="Did not"\
" aggregate multiple schedd's properly.")
def test_contact_string(self):
os.environ['GIP_TESTING'] = '1'
path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg-info-provider-sge.py --config=test_configs/pf-sge.conf")
fd = os.popen(path)
entries = read_ldap(fd)
self.failUnless(fd.close() == None)
for entry in entries:
if 'GlueCE' in entry.objectClass:
contact_string = entry.glue['CEInfoContactString']
self.failIf(contact_string == "", "Contact string is missing")
self.failIf(contact_string.endswith("jobmanager-sge"), \
"Contact string must include the queue.")
def test_contact_string(self):
os.environ['GIP_TESTING'] = '1'
path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg-info-provider-pbs.py --config=test_configs/red.conf")
fd = os.popen(path)
entries = read_ldap(fd)
self.failUnless(fd.close() == None)
for entry in entries:
if 'GlueCE' in entry.objectClass:
contact_string = entry.glue['CEInfoContactString']
self.failIf(contact_string == "", "Contact string is missing")
self.failIf(contact_string.endswith("jobmanager-pbs"), \
"Contact string must include the queue.")
def test_multi_schedd_output(self):
"""
Make sure that condor submitter accounting groups spread over multiple
schedds (and hence in multiple ClassAds) are aggregated.
"""
ce = 'fnpcfg1.fnal.gov:2119/jobmanager-condor-group_nysgrid'
os.environ['GIP_TESTING'] = 'suffix=fnal'
path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
"condor.py --config=test_configs/fnal_condor.conf")
fd = os.popen(path)
entries = read_ldap(fd, multi=True)
entry = self.check_for_ce(ce, entries)
self.failUnless('25' in entry.glue['CEStateRunningJobs'], msg="Did not"\
" aggregate multiple schedd's properly.")
def test_mwt2_uc__entries(self):
"""
Checks the LBL information, as they don't have PBSPro
"""
# Switch commands over to the LBL ones:
old_commands = dict(gip_testing.commands)
try:
os.environ['GIP_TESTING'] = 'suffix=mwt2'
path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg-info-provider-pbs.py --config=test_configs/red.conf")
fd = os.popen(path)
entries = read_ldap(fd)
self.assertEquals(fd.close(), None)
finally:
gip_testing.commands = old_commands
def test_mwt2_uc__entries(self):
"""
Checks the LBL information, as they don't have PBSPro
"""
# Switch commands over to the LBL ones:
old_commands = dict(gip_testing.commands)
try:
os.environ['GIP_TESTING'] = 'suffix=mwt2'
path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg-info-provider-pbs.py --config=test_configs/red.conf")
fd = os.popen(path)
entries = read_ldap(fd)
self.assertEquals(fd.close(), None)
finally:
gip_testing.commands = old_commands
def run_test_config(self, testconfig):
os.environ['GIP_TESTING'] = "1"
cfg_name = os.path.join("test_configs", testconfig)
if not os.path.exists(cfg_name):
self.fail(msg="Test configuration %s does not exist." % cfg_name)
cp = config(cfg_name)
providers_path = cp_get(cp, "gip", "provider_dir", "$GIP_LOCATION/" \
"providers")
providers_path = os.path.expandvars(providers_path)
se_provider_path = os.path.join(providers_path, 'storage_element.py')
cmd = se_provider_path + " --config %s" % cfg_name
print cmd
fd = os.popen(cmd)
entries = read_ldap(fd, multi=True)
self.failIf(fd.close(), msg="Run of storage element provider failed!")
return entries, cp
def run_test_config(self, testconfig):
os.environ['GIP_TESTING'] = "1"
cfg_name = os.path.join("test_configs", testconfig)
if not os.path.exists(cfg_name):
self.fail(msg="Test configuration %s does not exist." % cfg_name)
cp = config(cfg_name)
providers_path = cp_get(cp, "gip", "provider_dir", "$GIP_LOCATION/" \
"providers")
providers_path = os.path.expandvars(providers_path)
se_provider_path = os.path.join(providers_path,
'storage_element.py')
cmd = se_provider_path + " --config %s" % cfg_name
print cmd
fd = os.popen(cmd)
entries = read_ldap(fd, multi=True)
self.failIf(fd.close(), msg="Run of storage element provider failed!")
return entries, cp
def handle_plugins(entries, plugins):
# Make a list of all the plugin GLUE entries
plugin_entries = []
for _, plugin_info in plugins.items():
if 'output' in plugin_info:
fp = cStringIO.StringIO(plugin_info['output'])
plugin_entries += read_ldap(fp, multi=True)
# Merge all the plugin entries into the main stream.
for p_entry in plugin_entries:
log.debug("Plugin contents:\n%s" % p_entry)
for entry in entries:
for p_entry in plugin_entries:
if compareDN(entry, p_entry):
for glue, value in p_entry.glue.items():
entry.glue[glue] = value
for key, value in p_entry.nonglue.items():
entry.nonglue[key] = value
return entries
def test_red_entries(self):
"""
Make sure that VOLocal gets the correct queue information.
"""
os.environ['GIP_TESTING'] = '1'
path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg-info-provider-pbs.py --config=test_configs/red.conf")
fd = os.popen(path)
entries = read_ldap(fd)
self.failUnless(fd.close() == None)
for entry in entries:
if 'GlueVOView' in entry.objectClass and \
entry.glue['VOViewLocalID'] == 'cms' and \
entry.glue['ChunkKey']=='GlueCEUniqueID=red.unl.edu:2119/'\
'jobmanager-pbs-cms':
self.failUnless(entry.glue['CEStateRunningJobs'] == '203')
self.failUnless(entry.glue['CEStateWaitingJobs'] == '1')
self.failUnless(entry.glue['CEStateTotalJobs'] == '204')
def test_red_entries(self):
"""
Make sure that VOLocal gets the correct queue information.
"""
os.environ['GIP_TESTING'] = '1'
path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg-info-provider-pbs.py --config=test_configs/red.conf")
fd = os.popen(path)
entries = read_ldap(fd)
self.failUnless(fd.close() == None)
for entry in entries:
if 'GlueVOView' in entry.objectClass and \
entry.glue['VOViewLocalID'] == 'cms' and \
entry.glue['ChunkKey']=='GlueCEUniqueID=red.unl.edu:2119/'\
'jobmanager-pbs-cms':
self.failUnless(entry.glue['CEStateRunningJobs'] == '203')
self.failUnless(entry.glue['CEStateWaitingJobs'] == '1')
self.failUnless(entry.glue['CEStateTotalJobs'] == '204')
def handle_plugins(entries, plugins):
# Make a list of all the plugin GLUE entries
plugin_entries = []
for plugin, plugin_info in plugins.items():
if 'output' in plugin_info:
fp = cStringIO.StringIO(plugin_info['output'])
plugin_entries += read_ldap(fp, multi=True)
# Merge all the plugin entries into the main stream.
for p_entry in plugin_entries:
log.debug("Plugin contents:\n%s" % p_entry)
for entry in entries:
for p_entry in plugin_entries:
if compareDN(entry, p_entry):
for glue, value in p_entry.glue.items():
entry.glue[glue] = value
for key, value in p_entry.nonglue.items():
entry.nonglue[key] = value
return entries
def test_max_running(self):
"""
Regression test for max_running. Ensure that the number of free slots
& total reported is never greater than the # of max running.
"""
os.environ['GIP_TESTING'] = '1'
path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg-info-provider-pbs.py --config=test_configs/red.conf")
fd = os.popen(path)
entries = read_ldap(fd)
self.failUnless(fd.close() == None)
has_dzero_ce = False
for entry in entries:
if 'GlueCE' in entry.objectClass and \
entry.glue['CEUniqueID'] == 'red.unl.edu:2119/jobmanager' \
'-pbs-dzero':
self.failUnless(entry.glue['CEPolicyMaxRunningJobs'] == '158')
self.failUnless(entry.glue['CEPolicyMaxTotalJobs'] == '158')
self.failUnless(entry.glue['CEStateFreeJobSlots'] == '158')
has_dzero_ce = True
self.failUnless(has_dzero_ce, msg="dzero queue's CE was not found!")
def test_max_queuable_26(self):
"""
Regression test for max_queuable. Ensure that the number of free slots
reported is never greater than the # of max queuable
"""
os.environ['GIP_TESTING'] = '1'
path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg-info-provider-pbs.py --config=test_configs/red.conf")
fd = os.popen(path)
entries = read_ldap(fd)
self.failUnless(fd.close() == None)
has_lcgadmin_ce = False
for entry in entries:
if 'GlueCE' in entry.objectClass and \
entry.glue['CEUniqueID'] == 'red.unl.edu:2119/jobmanager' \
'-pbs-lcgadmin':
self.failUnless(entry.glue['CEPolicyMaxWaitingJobs'] == '183')
self.failUnless(entry.glue['CEStateFreeCPUs'] == '4')
self.failUnless(entry.glue['CEStateFreeJobSlots'] == '4')
has_lcgadmin_ce = True
self.failUnless(has_lcgadmin_ce, msg="lcgadmin queue's CE was not found!")
def setUp(self):
cp = ConfigParser.ConfigParser()
self.cp = cp
cp.add_section("site")
cp.set("site", "name", "Nebraska")
cp.set("site", "unique_name", "red.unl.edu")
cp.add_section("bdii")
cp.set("bdii", "endpoint", "ldap://is.grid.iu.edu:2170/")
cp.add_section("classic_se")
cp.set("classic_se", "advertise_se", "True")
cp.set("classic_se", "host", "red.unl.edu")
cp.set("classic_se", "port", "2811")
cp.set("classic_se", "name", "T2_Nebraska_classicSE")
cp.set("classic_se", "unique_name", "red.unl.edu_se")
cp.set("classic_se", "default", "/opt/osg/data/$VO")
cp.set("classic_se", "space",
"%i, %i, %i" % (1000**2, 10 * 1000**2, 11 * 1000**2))
cp.add_section("vo")
cp.set("vo", "user_vo_map", "test_configs/red-osg-user-vo-map.txt")
self.output = cStringIO.StringIO()
old_stdout = sys.stdout
sys.stdout = self.output
try:
print_classicSE(cp)
finally:
sys.stdout = old_stdout
self.output.seek(0)
self.entries = read_ldap(self.output, multi=True)
ses = []
sas = []
for entry in self.entries:
if 'GlueSE' in entry.objectClass:
ses.append(entry)
if 'GlueSA' in entry.objectClass:
sas.append(entry)
self.ses = ses
self.sas = sas
def setUp(self):
cp = ConfigParser.ConfigParser()
self.cp = cp
cp.add_section("site")
cp.set("site", "name", "Nebraska")
cp.set("site", "unique_name", "red.unl.edu")
cp.add_section("bdii")
cp.set("bdii", "endpoint", "ldap://is.grid.iu.edu:2170/")
cp.add_section("classic_se")
cp.set("classic_se", "advertise_se", "True")
cp.set("classic_se", "host", "red.unl.edu")
cp.set("classic_se", "port", "2811")
cp.set("classic_se", "name", "T2_Nebraska_classicSE")
cp.set("classic_se", "unique_name", "red.unl.edu_se")
cp.set("classic_se", "default", "/opt/osg/data/$VO")
cp.set("classic_se", "space", "%i, %i, %i" % (1000**2, 10*1000**2,
11*1000**2))
cp.add_section("vo")
cp.set("vo", "user_vo_map", "test_configs/red-osg-user-vo-map.txt")
self.output = cStringIO.StringIO()
old_stdout = sys.stdout
sys.stdout = self.output
try:
print_classicSE(cp)
finally:
sys.stdout = old_stdout
self.output.seek(0)
self.entries = read_ldap(self.output, multi=True)
ses = []
sas = []
for entry in self.entries:
if 'GlueSE' in entry.objectClass:
ses.append(entry)
if 'GlueSA' in entry.objectClass:
sas.append(entry)
self.ses = ses
self.sas = sas
def test_output_pf(self):
"""
Test the sample output from prairiefire.unl.edu. Should represent
a "simple" Condor setup, no groups or priorities.
"""
os.environ['GIP_TESTING'] = '1'
path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
"condor.py --config=test_configs/condor_test.conf")
fd = os.popen(path)
entries = read_ldap(fd)
self.assertEquals(fd.close(), None)
has_ce = False
ce_name = socket.gethostname()
for entry in entries:
if 'GlueCE' in entry.objectClass:
has_ce = True
self.assertEquals(entry.glue['CEStateTotalJobs'], '6')
self.assertEquals(entry.glue['CEStateRunningJobs'], '4')
self.assertEquals(entry.glue['CEStateFreeCPUs'], '77')
self.assertEquals(entry.glue['CEPolicyAssignedJobSlots'], '81')
self.assertEquals(entry.glue['CEUniqueID'], \
'%s:2119/jobmanager-condor-default' % ce_name)
self.assertEquals(has_ce, True)
def test_output_pf(self):
"""
Test the sample output from prairiefire.unl.edu. Should represent
a "simple" Condor setup, no groups or priorities.
"""
os.environ['GIP_TESTING'] = '1'
path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
"condor.py --config=test_configs/condor_test.conf")
fd = os.popen(path)
entries = read_ldap(fd)
self.assertEquals(fd.close(), None)
has_ce = False
ce_name = socket.gethostname()
for entry in entries:
if 'GlueCE' in entry.objectClass:
has_ce = True
self.assertEquals(entry.glue['CEStateTotalJobs'], '6')
self.assertEquals(entry.glue['CEStateRunningJobs'], '4')
self.assertEquals(entry.glue['CEStateFreeCPUs'], '77')
self.assertEquals(entry.glue['CEPolicyAssignedJobSlots'], '81')
self.assertEquals(entry.glue['CEUniqueID'], \
'%s:2119/jobmanager-condor-default' % ce_name)
self.assertEquals(has_ce, True)
def test_lbl_entries(self):
"""
Checks the LBL information, as they don't have PBSPro
"""
# Switch commands over to the LBL ones:
old_commands = dict(gip_testing.commands)
try:
os.environ['GIP_TESTING'] = 'suffix=lbl'
path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg-info-provider-pbs.py --config=test_configs/red.conf")
fd = os.popen(path)
entries = read_ldap(fd)
self.assertEquals(fd.close(), None)
finally:
gip_testing.commands = old_commands
for entry in entries:
if 'GlueCE' in entry.objectClass:
self.assertEquals(entry.glue['CEStateTotalJobs'], '6')
self.assertEquals(entry.glue['CEStateRunningJobs'], '6')
self.assertEquals(entry.glue['CEStateFreeCPUs'], '51')
self.assertEquals(entry.glue['CEStateFreeJobSlots'], '51')
self.assertEquals(entry.glue['CEPolicyAssignedJobSlots'], '60')
self.assertEquals(entry.glue['CEUniqueID'], \
'red.unl.edu:2119/jobmanager-pbs-batch')
def test_lbl_entries(self):
"""
Checks the LBL information, as they don't have PBSPro
"""
# Switch commands over to the LBL ones:
old_commands = dict(gip_testing.commands)
try:
os.environ['GIP_TESTING'] = 'suffix=lbl'
path = os.path.expandvars("$GIP_LOCATION/libexec/" \
"osg-info-provider-pbs.py --config=test_configs/red.conf")
fd = os.popen(path)
entries = read_ldap(fd)
self.assertEquals(fd.close(), None)
finally:
gip_testing.commands = old_commands
for entry in entries:
if 'GlueCE' in entry.objectClass:
self.assertEquals(entry.glue['CEStateTotalJobs'], '6')
self.assertEquals(entry.glue['CEStateRunningJobs'], '6')
self.assertEquals(entry.glue['CEStateFreeCPUs'], '51')
self.assertEquals(entry.glue['CEStateFreeJobSlots'], '51')
self.assertEquals(entry.glue['CEPolicyAssignedJobSlots'], '60')
self.assertEquals(entry.glue['CEUniqueID'], \
'red.unl.edu:2119/jobmanager-pbs-batch')
def test_groups_output(self):
"""
Look at the group output from UCSD.
- Check to make sure a GlueCE is defined for each group.
- Check to make sure the black/white lists are obeyed.
- Check to make sure that the FreeSlots are right for GlueCE.
(FreeSlots should not be more than group quota).
- Check to make sure that the FreeSlots are right for GlueVOView.
(FreeSlots should not be more than group quota).
"""
os.environ['GIP_TESTING'] = 'suffix=ucsd'
path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
"condor.py --config=test_configs/ucsd_condor.conf")
fd = os.popen(path)
entries = read_ldap(fd, multi=True)
tmpl = "osg-gw-2.t2.ucsd.edu:2119/jobmanager-condor-%s"
# This CE should *not* be present -> verifies blacklists work.
try:
entry = self.check_for_ce(tmpl % "group_cdf", entries)
did_fail = False
except:
did_fail = True
self.failUnless(did_fail, msg="CE %s is present, but shouldn't be" % \
(tmpl % "group_cdf"))
# Check lcgadmin group. Verifies whitelist is being used.
entry = self.check_for_ce(tmpl % "group_lcgadmin", entries)
self.failUnless('VO:atlas' in entry.glue['CEAccessControlBaseRule'],
msg="ATLAS is not allowed in group_lcgadmin")
self.failUnless('VO:cms' in entry.glue['CEAccessControlBaseRule'],
msg="CMS is not allowed in group_lcgadmin")
# Check cmsprod group. Verifies that the mapper is being used.
entry = self.check_for_ce(tmpl % "group_cmsprod", entries)
self.failUnless('VO:cms' in entry.glue['CEAccessControlBaseRule'],
msg="CMS is not allowed in group_cmsprod")
# Check ligo group. Verifies that whitelist overrides the mapper.
entry = self.check_for_ce(tmpl % "group_ligo", entries)
self.failUnless('VO:fmri' in entry.glue['CEAccessControlBaseRule'],
msg="FMRI is not allowed in group_cmsprod")
self.failIf('VO:ligo' in entry.glue['CEAccessControlBaseRule'],
msg="LIGO is allowed in group_cmsprod")
for entry in entries:
if 'GlueCE' not in entry.objectClass:
continue
total = int(entry.glue['CEPolicyAssignedJobSlots'][0])
assigned = int(entry.glue['CEPolicyAssignedJobSlots'][0])
running = int(entry.glue['CEStateRunningJobs'][0])
free = int(entry.glue['CEStateFreeJobSlots'][0])
self.failUnless(total <= assigned, msg="Failed invariant: " \
"TOTAL <= CE_ASSIGNED")
self.failUnless(running <= total, msg="Failed invariant: " \
"RUNNING <= TOTAL")
self.failUnless(running <= assigned, msg="Failed invariant: " \
"RUNNING <= CE_ASSIGNED")
self.failUnless(free <= assigned - running, msg="Failed invariant" \
": CE_FREE_SLOTS <= CE_ASSIGNED - RUNNING")
unique_id = entry.glue['CEUniqueID']
ce_entry = entry
for entry in entries:
if 'GlueVOView' not in entry.objectClass:
continue
chunk = 'GlueCEUniqueID=%s' % unique_id
if chunk not in entry.glue['ChunkKey']:
continue
vo_free = int(entry.glue['CEStateFreeJobSlots'][0])
running = int(entry.glue['CEStateRunningJobs'][0])
self.failUnless(vo_free <= free, msg="Failed invariant: " \
"VO_FREE_SLOTS <= CE_FREE_SLOTS")
self.failUnless(vo_free <= assigned - running, msg="Failed " \
"invariant: VO_FREE_SLOTS <= CE_ASSIGNED - VO_RUNNING")
def testGipOutput(self):
"""
GIP Validator
This check tests the ldif as reported by cemon for:
* The following stanzas mustappear at least once:
o GlueCEUniqueID
o GlueVOViewLocalID
o GlueSubClusterUniqueID
o GlueClusterUniqueID
o GlueCESEBindSEUniqueID
o GlueCESEBindGroupCEUniqueID
o GlueLocationLocalID
o GlueServiceUniqueID
o GlueSEUniqueID
o GlueSEAccessProtocolLocalID
o GlueSEControlProtocolLocalID
o GlueSALocalID
* The GlueSiteUniqueID and GlueSiteName must be the same for EGEE compatibility
* The CE Stanza for the following conditions:
o CEInfoDefaultSE != UNAVAILABLE
o CEPolicyMaxCPUTime != 0
o CEInfoTotalCPUs != 0
o CEStateEstimatedResponseTime != 0
o CEStateWorstResponseTime != 0
o CEPolicyMaxWallClockTime != 0
* The ldiff must have newlines appended after every key value combination
* The all foriegn keys and chunk keys must have corresponding stanzas
* All entries must conform to the attribute:value format
* Test SRM ads for the following:
o endpoint type is SRM
o Version is 1.1.0 or 2.2.0 (1.1 or 2.2 generate warnings)
o Site unique ID is not blank
o Site unique ID is actual unique ID used for this site.
o If dCache, make sure that the /srm/managervX string is correct.
* Test DN ads for the following:
o o=grid appears once
o mds-vo-name=local appears once
o mds-vo-name=<site> appears once
o they appear in the correct order
"""
if self.type.lower() == "url": # we want info from the web status page
fqdn = getFQDNBySiteName(self.cp, self.site)
url = cp_get(self.cp, "gip_tests", "validator_url",
"") % (fqdn, self.site)
fd = urllib2.urlopen(url)
# line = fd.readline()
# if "html" in fd.readline():
# self.assertEquals(0, 1, "CEMon not reporting for site %s" % self.site)
# fd = urllib2.urlopen(url)
elif self.type.lower(
) == "gipinfo": # We want info from the gip_info script
path = os.path.expandvars("$GIP_LOCATION/bin/gip_info")
fd = os.popen(path)
else: # assume we want to read from the bdii
fd = query_bdii(self.cp,
query="",
base="mds-vo-name=%s,mds-vo-name=local,o=grid" %
self.site)
self.entries = read_ldap(fd)
self.site_id = self.getSiteUniqueID()
self.test_existence_all()
self.test_egee_site_unique_id()
self.test_ce()
self.test_site()
self.test_missing_newlines()
self.test_missing_values()
self.test_dn()
self.test_srm()
check_cache(plugins, temp_dir, freshness)
# Launch the providers and plugins
pids = launch_modules(providers, provider_dir, temp_dir, timeout)
pids += launch_modules(plugins, plugin_dir, temp_dir, timeout)
# Wait for the results
wait_children(pids, response)
# Load results from the cache
check_cache(providers, temp_dir, cache_ttl)
check_cache(plugins, temp_dir, cache_ttl)
# Create LDAP entries out of the static info
static_fp = cStringIO.StringIO(static_info)
entries = read_ldap(static_fp, multi=True)
# Apply output from the providers
entries = handle_providers(entries, providers)
# Apply output from the plugins
entries = handle_plugins(entries, plugins)
# Finally, apply our special cases
entries = handle_add_attributes(entries, add_attributes)
entries = handle_alter_attributes(entries, alter_attributes)
entries = handle_remove_attributes(entries, remove_attributes)
# Return the LDAP or print it out.
if return_entries:
return entries
def test_groups_output(self):
"""
Look at the group output from UCSD.
- Check to make sure a GlueCE is defined for each group.
- Check to make sure the black/white lists are obeyed.
- Check to make sure that the FreeSlots are right for GlueCE.
(FreeSlots should not be more than group quota).
- Check to make sure that the FreeSlots are right for GlueVOView.
(FreeSlots should not be more than group quota).
"""
os.environ['GIP_TESTING'] = 'suffix=ucsd'
path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
"condor.py --config=test_configs/ucsd_condor.conf")
fd = os.popen(path)
entries = read_ldap(fd, multi=True)
tmpl = "osg-gw-2.t2.ucsd.edu:2119/jobmanager-condor-%s"
# This CE should *not* be present -> verifies blacklists work.
try:
entry = self.check_for_ce(tmpl % "group_cdf", entries)
did_fail = False
except:
did_fail = True
self.failUnless(did_fail, msg="CE %s is present, but shouldn't be" % \
(tmpl % "group_cdf"))
# Check lcgadmin group. Verifies whitelist is being used.
entry = self.check_for_ce(tmpl % "group_lcgadmin", entries)
self.failUnless('VO:atlas' in entry.glue['CEAccessControlBaseRule'],
msg="ATLAS is not allowed in group_lcgadmin")
self.failUnless('VO:cms' in entry.glue['CEAccessControlBaseRule'],
msg="CMS is not allowed in group_lcgadmin")
# Check cmsprod group. Verifies that the mapper is being used.
entry = self.check_for_ce(tmpl % "group_cmsprod", entries)
self.failUnless('VO:cms' in entry.glue['CEAccessControlBaseRule'],
msg="CMS is not allowed in group_cmsprod")
# Check ligo group. Verifies that whitelist overrides the mapper.
entry = self.check_for_ce(tmpl % "group_ligo", entries)
self.failUnless('VO:fmri' in entry.glue['CEAccessControlBaseRule'],
msg="FMRI is not allowed in group_cmsprod")
self.failIf('VO:ligo' in entry.glue['CEAccessControlBaseRule'],
msg="LIGO is allowed in group_cmsprod")
for entry in entries:
if 'GlueCE' not in entry.objectClass:
continue
total = int(entry.glue['CEPolicyAssignedJobSlots'][0])
assigned = int(entry.glue['CEPolicyAssignedJobSlots'][0])
running = int(entry.glue['CEStateRunningJobs'][0])
free = int(entry.glue['CEStateFreeJobSlots'][0])
self.failUnless(total <= assigned, msg="Failed invariant: " \
"TOTAL <= CE_ASSIGNED")
self.failUnless(running <= total, msg="Failed invariant: " \
"RUNNING <= TOTAL")
self.failUnless(running <= assigned, msg="Failed invariant: " \
"RUNNING <= CE_ASSIGNED")
self.failUnless(free <= assigned - running, msg="Failed invariant" \
": CE_FREE_SLOTS <= CE_ASSIGNED - RUNNING")
unique_id = entry.glue['CEUniqueID']
ce_entry = entry
for entry in entries:
if 'GlueVOView' not in entry.objectClass:
continue
chunk = 'GlueCEUniqueID=%s' % unique_id
if chunk not in entry.glue['ChunkKey']:
continue
vo_free = int(entry.glue['CEStateFreeJobSlots'][0])
running = int(entry.glue['CEStateRunningJobs'][0])
self.failUnless(vo_free <= free, msg="Failed invariant: " \
"VO_FREE_SLOTS <= CE_FREE_SLOTS")
self.failUnless(vo_free <= assigned - running, msg="Failed " \
"invariant: VO_FREE_SLOTS <= CE_ASSIGNED - VO_RUNNING")
check_cache(plugins, temp_dir, freshness)
# Launch the providers and plugins
pids = launch_modules(providers, provider_dir, temp_dir, timeout)
pids += launch_modules(plugins, plugin_dir, temp_dir, timeout)
# Wait for the results
wait_children(pids, response)
# Load results from the cache
check_cache(providers, temp_dir, cache_ttl)
check_cache(plugins, temp_dir, cache_ttl)
# Create LDAP entries out of the static info
static_fp = cStringIO.StringIO(static_info)
entries = read_ldap(static_fp, multi=True)
# Apply output from the providers
entries = handle_providers(entries, providers)
# Apply output from the plugins
entries = handle_plugins(entries, plugins)
# Finally, apply our special cases
entries = handle_add_attributes(entries, add_attributes)
entries = handle_alter_attributes(entries, alter_attributes)
entries = handle_remove_attributes(entries, remove_attributes)
# Return the LDAP or print it out.
if return_entries:
return entries
def testGipOutput(self):
"""
GIP Validator
This check tests the ldif as reported by cemon for:
* The following stanzas mustappear at least once:
o GlueCEUniqueID
o GlueVOViewLocalID
o GlueSubClusterUniqueID
o GlueClusterUniqueID
o GlueCESEBindSEUniqueID
o GlueCESEBindGroupCEUniqueID
o GlueLocationLocalID
o GlueServiceUniqueID
o GlueSEUniqueID
o GlueSEAccessProtocolLocalID
o GlueSEControlProtocolLocalID
o GlueSALocalID
* The GlueSiteUniqueID and GlueSiteName must be the same for EGEE compatibility
* The CE Stanza for the following conditions:
o CEInfoDefaultSE != UNAVAILABLE
o CEPolicyMaxCPUTime != 0
o CEInfoTotalCPUs != 0
o CEStateEstimatedResponseTime != 0
o CEStateWorstResponseTime != 0
o CEPolicyMaxWallClockTime != 0
* The ldiff must have newlines appended after every key value combination
* The all foriegn keys and chunk keys must have corresponding stanzas
* All entries must conform to the attribute:value format
* Test SRM ads for the following:
o endpoint type is SRM
o Version is 1.1.0 or 2.2.0 (1.1 or 2.2 generate warnings)
o Site unique ID is not blank
o Site unique ID is actual unique ID used for this site.
o If dCache, make sure that the /srm/managervX string is correct.
* Test DN ads for the following:
o o=grid appears once
o mds-vo-name=local appears once
o mds-vo-name=<site> appears once
o they appear in the correct order
"""
if self.type.lower() == "url": # we want info from the web status page
fqdn = getFQDNBySiteName(self.cp, self.site)
url = cp_get(self.cp, "gip_tests", "validator_url", "") % (fqdn, self.site)
fd = urllib2.urlopen(url)
# line = fd.readline()
# if "html" in fd.readline():
# self.assertEquals(0, 1, "CEMon not reporting for site %s" % self.site)
# fd = urllib2.urlopen(url)
elif self.type.lower() == "gipinfo": # We want info from the gip_info script
path = os.path.expandvars("$GIP_LOCATION/bin/gip_info")
fd = os.popen(path)
else: # assume we want to read from the bdii
fd = query_bdii(self.cp, query="", base="mds-vo-name=%s,mds-vo-name=local,o=grid" % self.site)
self.entries = read_ldap(fd)
self.site_id = self.getSiteUniqueID()
self.test_existence_all()
self.test_egee_site_unique_id()
self.test_ce()
self.test_site()
self.test_missing_newlines()
self.test_missing_values()
self.test_dn()
self.test_srm()