Пример #1
0
def handle_providers(entries, providers):
    """
    Add the output from the providers to the list of the GIP entries.

    This will match the DNs; if two DNs are repeated, then one will be thrown
    out

    @param entries: A list of LdapData objects
    @param providers: A list of provider information dictionaries.
    @returns: The altered entries list.
    """
    provider_entries = []
    for _, p_info in providers.items():
        if 'output' in p_info:
            fp = cStringIO.StringIO(p_info['output'])
            provider_entries += read_ldap(fp, multi=True)
    remove_entries = []
    # Calculate all the duplicate entries, build a list of the ones
    # to remove.
    for entry in entries:
        for p_entry in provider_entries:
            if compareDN(entry, p_entry):
                remove_entries.append(entry)

    for entry in sets.Set(remove_entries):
        log.debug("Removing entry %s" % entry)
        try:
            entries.remove(entry)
        except ValueError:
            pass
    # Now add all the new entries from the providers
    for p_entry in provider_entries:
        entries.append(p_entry)
    return entries
Пример #2
0
 def setUpLDAP(self, multi=False, filename=None):
     if filename != None:
         self.filename = filename
     command = "$GIP_LOCATION/providers/site_topology.py --config=%s" % self.filename
     stdout = os.popen(command)
     entries = read_ldap(stdout, multi=multi)
     return entries
Пример #3
0
def handle_providers(entries, providers):
    """
    Add the output from the providers to the list of the GIP entries.

    This will match the DNs; if two DNs are repeated, then one will be thrown
    out

    @param entries: A list of LdapData objects
    @param providers: A list of provider information dictionaries.
    @returns: The altered entries list.
    """
    provider_entries = []
    for provider, p_info in providers.items():
        if 'output' in p_info:
            fp = cStringIO.StringIO(p_info['output'])
            provider_entries += read_ldap(fp, multi=True)
    remove_entries = []
    # Calculate all the duplicate entries, build a list of the ones
    # to remove.
    for entry in entries:
        for p_entry in provider_entries:
            if compareDN(entry, p_entry):
                remove_entries.append(entry)

    for entry in sets.Set(remove_entries):
        log.debug("Removing entry %s" % entry)  
        try:
              entries.remove(entry)
        except ValueError:
              pass
    # Now add all the new entries from the providers
    for p_entry in provider_entries:
        entries.append(p_entry)
    return entries
Пример #4
0
 def test_max_queuable_22(self):
     """
     Regression test for the max_queuable attribute.  Ticket #22.
     """
     os.environ['GIP_TESTING'] = '1'
     path = os.path.expandvars("$GIP_LOCATION/libexec/" \
         "osg-info-provider-pbs.py --config=test_configs/red.conf")
     fd = os.popen(path)
     entries = read_ldap(fd)
     self.failUnless(fd.close() == None)
     has_default_ce = False
     has_lcgadmin_ce = False
     for entry in entries:
         if 'GlueCE' in entry.objectClass and \
                 entry.glue['CEUniqueID'] == 'red.unl.edu:2119/jobmanager' \
                 '-pbs-default':
             self.failUnless(entry.glue['CEPolicyMaxTotalJobs'] == '2000')
             has_default_ce = True
         if 'GlueCE' in entry.objectClass and \
                 entry.glue['CEUniqueID'] == 'red.unl.edu:2119/jobmanager' \
                 '-pbs-lcgadmin':
             self.failUnless(entry.glue['CEPolicyMaxWaitingJobs'] == '183')
             has_lcgadmin_ce = True
     self.failUnless(has_default_ce,
                     msg="Default queue's CE was not found!")
     self.failUnless(has_default_ce,
                     msg="lcgadmin queue's CE was not found!")
Пример #5
0
 def test_max_queuable_22(self):
     """
     Regression test for the max_queuable attribute.  Ticket #22.
     """
     os.environ['GIP_TESTING'] = '1'
     path = os.path.expandvars("$GIP_LOCATION/libexec/" \
         "osg-info-provider-pbs.py --config=test_configs/red.conf")
     fd = os.popen(path)
     entries = read_ldap(fd)
     self.failUnless(fd.close() == None)
     has_default_ce = False
     has_lcgadmin_ce = False
     for entry in entries:
         if 'GlueCE' in entry.objectClass and \
                 entry.glue['CEUniqueID'] == 'red.unl.edu:2119/jobmanager' \
                 '-pbs-default':
             self.failUnless(entry.glue['CEPolicyMaxTotalJobs'] == '2000')
             has_default_ce = True
         if 'GlueCE' in entry.objectClass and \
                 entry.glue['CEUniqueID'] == 'red.unl.edu:2119/jobmanager' \
                 '-pbs-lcgadmin':
             self.failUnless(entry.glue['CEPolicyMaxWaitingJobs'] == '183')
             has_lcgadmin_ce = True
     self.failUnless(has_default_ce, msg="Default queue's CE was not found!")
     self.failUnless(has_default_ce, msg="lcgadmin queue's CE was not found!")
Пример #6
0
 def setUpLDAP(self, multi=False, filename=None):
     if filename != None:
         self.filename = filename
     command = "$GIP_LOCATION/providers/site_topology.py --config=%s" % \
         self.filename
     stdout = os.popen(command)
     entries = read_ldap(stdout, multi=multi)
     return entries
Пример #7
0
 def run_compareDN(self, string1, string2, should_match = False):
     orig_entries = []
     fp = cStringIO.StringIO(string1)
     orig_entries += read_ldap(fp, multi=True)
     
     other_entries = []
     fp = cStringIO.StringIO(string2)
     other_entries += read_ldap(fp, multi=True)
     
     result = compareDN(orig_entries[0], other_entries[0])
     if should_match:
         self.failIf(not result, msg="compareDN returned False when it should have "\
                 "returned True.\n\nDN1: %s \nDN2: %s" % \
                 (prettyDN(orig_entries[0].dn),prettyDN(other_entries[0].dn)))
     else:
         self.failIf(result, msg="compareDN returned True when it should have "\
                 "returned False.\n\nDN1: %s \nDN2: %s" % \
                 (prettyDN(orig_entries[0].dn),prettyDN(other_entries[0].dn)))
         
     return result
Пример #8
0
    def test_condorq_parsing(self):
        """
        Test the condor_q -xml parsing for Condor format changes
        """

        # Condor 7.3.2 and above
        os.environ['GIP_TESTING'] = 'suffix=glow'
        path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
            "condor.py --config=test_configs/glow_condor.conf")
        fd = os.popen(path)
        entries = read_ldap(fd, multi=True)
        self.assertEquals(fd.close(), None)

        # Condor 7.3.1 and less
        os.environ['GIP_TESTING'] = 'suffix=glow-condor72'
        path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
            "condor.py --config=test_configs/glow_condor.conf")
        fd = os.popen(path)
        entries = read_ldap(fd, multi=True)
        self.assertEquals(fd.close(), None)
Пример #9
0
 def test_site_entries(self):
     """
     Checks the %s information.
     """ % site
     # Switch commands over to the site ones:
     os.environ['GIP_TESTING'] = 'suffix=%s' % site
     path = os.path.expandvars("$GIP_LOCATION/libexec/" \
         "osg-info-provider-pbs.py --config=test_configs/%s.conf" % site)
     fd = os.popen(path)
     entries = read_ldap(fd)
     self.assertEquals(fd.close(), None)
Пример #10
0
 def test_site_entries(self):
     """
     Checks the %s information.
     """ % site
     # Switch commands over to the site ones:
     os.environ['GIP_TESTING'] = 'suffix=%s' % site
     path = os.path.expandvars("$GIP_LOCATION/libexec/" \
         "osg-info-provider-pbs.py --config=test_configs/%s.conf" % site)
     fd = os.popen(path)
     entries = read_ldap(fd)
     self.assertEquals(fd.close(), None)
Пример #11
0
    def test_condorq_parsing(self):
        """
        Test the condor_q -xml parsing for Condor format changes
        """

        # Condor 7.3.2 and above
        os.environ['GIP_TESTING'] = 'suffix=glow'
        path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
            "condor.py --config=test_configs/glow_condor.conf")
        fd = os.popen(path)
        entries = read_ldap(fd, multi=True)
        self.assertEquals(fd.close(), None)

        # Condor 7.3.1 and less
        os.environ['GIP_TESTING'] = 'suffix=glow-condor72'
        path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
            "condor.py --config=test_configs/glow_condor.conf")
        fd = os.popen(path)
        entries = read_ldap(fd, multi=True)
        self.assertEquals(fd.close(), None)
Пример #12
0
    def run_compareDN(self, string1, string2, should_match=False):
        orig_entries = []
        fp = cStringIO.StringIO(string1)
        orig_entries += read_ldap(fp, multi=True)

        other_entries = []
        fp = cStringIO.StringIO(string2)
        other_entries += read_ldap(fp, multi=True)

        result = compareDN(orig_entries[0], other_entries[0])
        if should_match:
            self.failIf(not result, msg="compareDN returned False when it should have "\
                    "returned True.\n\nDN1: %s \nDN2: %s" % \
                    (prettyDN(orig_entries[0].dn),prettyDN(other_entries[0].dn)))
        else:
            self.failIf(result, msg="compareDN returned True when it should have "\
                    "returned False.\n\nDN1: %s \nDN2: %s" % \
                    (prettyDN(orig_entries[0].dn),prettyDN(other_entries[0].dn)))

        return result
Пример #13
0
 def test_condor_hierarchical_group(self):
     """
     Make sure that condor can read and publish hierarchical groups too.
     """
     ce = 'fnpcfg1.fnal.gov:2119/jobmanager-condor-group_nysgrid.sub1'
     os.environ['GIP_TESTING'] = 'suffix=fnal'
     path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
         "condor.py --config=test_configs/fnal_condor.conf")
     fd = os.popen(path)
     entries = read_ldap(fd, multi=True)
     entry = self.check_for_ce(ce, entries)
     self.failUnless('33' in entry.glue['CEStateRunningJobs'], msg="Did not"\
         " get hierarchical groups.")
Пример #14
0
 def setUpLDAP(self, filename=None):
     if filename != None:
         self.filename = filename
     os.environ['GIP_TESTING'] = "1"
     cp = config(self.filename)
     self.ces = getCEList(cp)
     self.ses = getSEList(cp)
     cese_provider_path = os.path.expandvars("$GIP_LOCATION/libexec/" \
         "osg_info_cesebind.py --config %s" % self.filename)
     print >> sys.stderr, "Used command", cese_provider_path
     fd = os.popen(cese_provider_path)
     self.entries = read_ldap(fd, multi=True)
     self.exit_status = fd.close()
Пример #15
0
 def test_condor_hierarchical_group(self):
     """
     Make sure that condor can read and publish hierarchical groups too.
     """
     ce = 'fnpcfg1.fnal.gov:2119/jobmanager-condor-group_nysgrid.sub1'
     os.environ['GIP_TESTING'] = 'suffix=fnal'
     path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
         "condor.py --config=test_configs/fnal_condor.conf")
     fd = os.popen(path)
     entries = read_ldap(fd, multi=True)
     entry = self.check_for_ce(ce, entries)
     self.failUnless('33' in entry.glue['CEStateRunningJobs'], msg="Did not"\
         " get hierarchical groups.")
Пример #16
0
 def test_multi_schedd_output(self):
     """
     Make sure that condor submitter accounting groups spread over multiple
     schedds (and hence in multiple ClassAds) are aggregated.
     """
     ce = 'fnpcfg1.fnal.gov:2119/jobmanager-condor-group_nysgrid'
     os.environ['GIP_TESTING'] = 'suffix=fnal'
     path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
         "condor.py --config=test_configs/fnal_condor.conf")
     fd = os.popen(path)
     entries = read_ldap(fd, multi=True)
     entry = self.check_for_ce(ce, entries)
     self.failUnless('25' in entry.glue['CEStateRunningJobs'], msg="Did not"\
         " aggregate multiple schedd's properly.")
Пример #17
0
 def test_contact_string(self):
     os.environ['GIP_TESTING'] = '1'
     path = os.path.expandvars("$GIP_LOCATION/libexec/" \
         "osg-info-provider-sge.py --config=test_configs/pf-sge.conf")
     fd = os.popen(path)
     entries = read_ldap(fd)
     self.failUnless(fd.close() == None)
     
     for entry in entries:
         if 'GlueCE' in entry.objectClass:
             contact_string = entry.glue['CEInfoContactString']
             self.failIf(contact_string == "", "Contact string is missing")
             self.failIf(contact_string.endswith("jobmanager-sge"), \
                 "Contact string must include the queue.")
Пример #18
0
    def test_contact_string(self):
        os.environ['GIP_TESTING'] = '1'
        path = os.path.expandvars("$GIP_LOCATION/libexec/" \
            "osg-info-provider-pbs.py --config=test_configs/red.conf")
        fd = os.popen(path)
        entries = read_ldap(fd)
        self.failUnless(fd.close() == None)

        for entry in entries:
            if 'GlueCE' in entry.objectClass:
                contact_string = entry.glue['CEInfoContactString']
                self.failIf(contact_string == "", "Contact string is missing")
                self.failIf(contact_string.endswith("jobmanager-pbs"), \
                    "Contact string must include the queue.")
Пример #19
0
 def test_multi_schedd_output(self):
     """
     Make sure that condor submitter accounting groups spread over multiple
     schedds (and hence in multiple ClassAds) are aggregated.
     """
     ce = 'fnpcfg1.fnal.gov:2119/jobmanager-condor-group_nysgrid'
     os.environ['GIP_TESTING'] = 'suffix=fnal'
     path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
         "condor.py --config=test_configs/fnal_condor.conf")
     fd = os.popen(path)
     entries = read_ldap(fd, multi=True)
     entry = self.check_for_ce(ce, entries)
     self.failUnless('25' in entry.glue['CEStateRunningJobs'], msg="Did not"\
         " aggregate multiple schedd's properly.")
Пример #20
0
 def test_mwt2_uc__entries(self):
     """
     Checks the LBL information, as they don't have PBSPro
     """
     # Switch commands over to the LBL ones:
     old_commands = dict(gip_testing.commands)
     try:
         os.environ['GIP_TESTING'] = 'suffix=mwt2'
         path = os.path.expandvars("$GIP_LOCATION/libexec/" \
             "osg-info-provider-pbs.py --config=test_configs/red.conf")
         fd = os.popen(path)
         entries = read_ldap(fd)
         self.assertEquals(fd.close(), None)
     finally:
         gip_testing.commands = old_commands
Пример #21
0
 def test_mwt2_uc__entries(self):
     """
     Checks the LBL information, as they don't have PBSPro
     """
     # Switch commands over to the LBL ones:
     old_commands = dict(gip_testing.commands)
     try:
         os.environ['GIP_TESTING'] = 'suffix=mwt2'
         path = os.path.expandvars("$GIP_LOCATION/libexec/" \
             "osg-info-provider-pbs.py --config=test_configs/red.conf")
         fd = os.popen(path)
         entries = read_ldap(fd)
         self.assertEquals(fd.close(), None)
     finally:
         gip_testing.commands = old_commands
Пример #22
0
 def run_test_config(self, testconfig):
     os.environ['GIP_TESTING'] = "1"
     cfg_name = os.path.join("test_configs", testconfig)
     if not os.path.exists(cfg_name):
         self.fail(msg="Test configuration %s does not exist." % cfg_name)
     cp = config(cfg_name)
     providers_path = cp_get(cp, "gip", "provider_dir", "$GIP_LOCATION/" \
                             "providers")
     providers_path = os.path.expandvars(providers_path)
     se_provider_path = os.path.join(providers_path, 'storage_element.py')
     cmd = se_provider_path + " --config %s" % cfg_name
     print cmd
     fd = os.popen(cmd)
     entries = read_ldap(fd, multi=True)
     self.failIf(fd.close(), msg="Run of storage element provider failed!")
     return entries, cp
Пример #23
0
 def run_test_config(self, testconfig):
     os.environ['GIP_TESTING'] = "1"
     cfg_name = os.path.join("test_configs", testconfig)
     if not os.path.exists(cfg_name):
         self.fail(msg="Test configuration %s does not exist." % cfg_name)
     cp = config(cfg_name)
     providers_path = cp_get(cp, "gip", "provider_dir", "$GIP_LOCATION/" \
                             "providers")
     providers_path = os.path.expandvars(providers_path)
     se_provider_path = os.path.join(providers_path,
                                     'storage_element.py')
     cmd = se_provider_path + " --config %s" % cfg_name
     print cmd
     fd = os.popen(cmd)
     entries = read_ldap(fd, multi=True)
     self.failIf(fd.close(), msg="Run of storage element provider failed!")
     return entries, cp
Пример #24
0
def handle_plugins(entries, plugins):
    # Make a list of all the plugin GLUE entries
    plugin_entries = []
    for _, plugin_info in plugins.items():
        if 'output' in plugin_info:
            fp = cStringIO.StringIO(plugin_info['output'])
            plugin_entries += read_ldap(fp, multi=True)
    # Merge all the plugin entries into the main stream.
    for p_entry in plugin_entries:
        log.debug("Plugin contents:\n%s" % p_entry)
    for entry in entries:
        for p_entry in plugin_entries:
            if compareDN(entry, p_entry):
                for glue, value in p_entry.glue.items():
                    entry.glue[glue] = value
                for key, value in p_entry.nonglue.items():
                    entry.nonglue[key] = value
    return entries
Пример #25
0
 def test_red_entries(self):
     """
     Make sure that VOLocal gets the correct queue information.
     """
     os.environ['GIP_TESTING'] = '1'
     path = os.path.expandvars("$GIP_LOCATION/libexec/" \
         "osg-info-provider-pbs.py --config=test_configs/red.conf")
     fd = os.popen(path)
     entries = read_ldap(fd)
     self.failUnless(fd.close() == None)
     for entry in entries:
         if 'GlueVOView' in entry.objectClass and \
                 entry.glue['VOViewLocalID'] == 'cms' and \
                 entry.glue['ChunkKey']=='GlueCEUniqueID=red.unl.edu:2119/'\
                 'jobmanager-pbs-cms':
             self.failUnless(entry.glue['CEStateRunningJobs'] == '203')
             self.failUnless(entry.glue['CEStateWaitingJobs'] == '1')
             self.failUnless(entry.glue['CEStateTotalJobs'] == '204')
Пример #26
0
 def test_red_entries(self):
     """
     Make sure that VOLocal gets the correct queue information.
     """
     os.environ['GIP_TESTING'] = '1'
     path = os.path.expandvars("$GIP_LOCATION/libexec/" \
         "osg-info-provider-pbs.py --config=test_configs/red.conf")
     fd = os.popen(path)
     entries = read_ldap(fd)
     self.failUnless(fd.close() == None)
     for entry in entries:
         if 'GlueVOView' in entry.objectClass and \
                 entry.glue['VOViewLocalID'] == 'cms' and \
                 entry.glue['ChunkKey']=='GlueCEUniqueID=red.unl.edu:2119/'\
                 'jobmanager-pbs-cms':
             self.failUnless(entry.glue['CEStateRunningJobs'] == '203')
             self.failUnless(entry.glue['CEStateWaitingJobs'] == '1')
             self.failUnless(entry.glue['CEStateTotalJobs'] == '204')
Пример #27
0
def handle_plugins(entries, plugins):
    # Make a list of all the plugin GLUE entries
    plugin_entries = []
    for plugin, plugin_info in plugins.items():
        if 'output' in plugin_info:
            fp = cStringIO.StringIO(plugin_info['output'])
            plugin_entries += read_ldap(fp, multi=True)
    # Merge all the plugin entries into the main stream.
    for p_entry in plugin_entries:
        log.debug("Plugin contents:\n%s" % p_entry)
    for entry in entries:
        for p_entry in plugin_entries:
            if compareDN(entry, p_entry):
                for glue, value in p_entry.glue.items():
                    entry.glue[glue] = value
                for key, value in p_entry.nonglue.items():
                    entry.nonglue[key] = value
    return entries
Пример #28
0
 def test_max_running(self):
     """
     Regression test for max_running.  Ensure that the number of free slots
     & total reported is never greater than the # of max running.
     """
     os.environ['GIP_TESTING'] = '1'
     path = os.path.expandvars("$GIP_LOCATION/libexec/" \
         "osg-info-provider-pbs.py --config=test_configs/red.conf")
     fd = os.popen(path)
     entries = read_ldap(fd)
     self.failUnless(fd.close() == None)
     has_dzero_ce = False
     for entry in entries:
         if 'GlueCE' in entry.objectClass and \
                 entry.glue['CEUniqueID'] == 'red.unl.edu:2119/jobmanager' \
                 '-pbs-dzero':
             self.failUnless(entry.glue['CEPolicyMaxRunningJobs'] == '158')
             self.failUnless(entry.glue['CEPolicyMaxTotalJobs'] == '158')
             self.failUnless(entry.glue['CEStateFreeJobSlots'] == '158')
             has_dzero_ce = True
     self.failUnless(has_dzero_ce, msg="dzero queue's CE was not found!")
Пример #29
0
 def test_max_queuable_26(self):
     """
     Regression test for max_queuable.  Ensure that the number of free slots
     reported is never greater than the # of max queuable
     """
     os.environ['GIP_TESTING'] = '1'
     path = os.path.expandvars("$GIP_LOCATION/libexec/" \
         "osg-info-provider-pbs.py --config=test_configs/red.conf")
     fd = os.popen(path)
     entries = read_ldap(fd)
     self.failUnless(fd.close() == None)
     has_lcgadmin_ce = False
     for entry in entries:
         if 'GlueCE' in entry.objectClass and \
                 entry.glue['CEUniqueID'] == 'red.unl.edu:2119/jobmanager' \
                 '-pbs-lcgadmin':
             self.failUnless(entry.glue['CEPolicyMaxWaitingJobs'] == '183')
             self.failUnless(entry.glue['CEStateFreeCPUs'] == '4')
             self.failUnless(entry.glue['CEStateFreeJobSlots'] == '4')
             has_lcgadmin_ce = True
     self.failUnless(has_lcgadmin_ce, msg="lcgadmin queue's CE was not found!")
Пример #30
0
    def setUp(self):
        cp = ConfigParser.ConfigParser()
        self.cp = cp
        cp.add_section("site")
        cp.set("site", "name", "Nebraska")
        cp.set("site", "unique_name", "red.unl.edu")
        cp.add_section("bdii")
        cp.set("bdii", "endpoint", "ldap://is.grid.iu.edu:2170/")
        cp.add_section("classic_se")
        cp.set("classic_se", "advertise_se", "True")
        cp.set("classic_se", "host", "red.unl.edu")
        cp.set("classic_se", "port", "2811")
        cp.set("classic_se", "name", "T2_Nebraska_classicSE")
        cp.set("classic_se", "unique_name", "red.unl.edu_se")
        cp.set("classic_se", "default", "/opt/osg/data/$VO")
        cp.set("classic_se", "space",
               "%i, %i, %i" % (1000**2, 10 * 1000**2, 11 * 1000**2))
        cp.add_section("vo")
        cp.set("vo", "user_vo_map", "test_configs/red-osg-user-vo-map.txt")

        self.output = cStringIO.StringIO()
        old_stdout = sys.stdout
        sys.stdout = self.output
        try:
            print_classicSE(cp)
        finally:
            sys.stdout = old_stdout
        self.output.seek(0)
        self.entries = read_ldap(self.output, multi=True)
        ses = []
        sas = []
        for entry in self.entries:
            if 'GlueSE' in entry.objectClass:
                ses.append(entry)
            if 'GlueSA' in entry.objectClass:
                sas.append(entry)
        self.ses = ses
        self.sas = sas
Пример #31
0
    def setUp(self):
        cp = ConfigParser.ConfigParser()
        self.cp = cp
        cp.add_section("site")
        cp.set("site", "name", "Nebraska")
        cp.set("site", "unique_name", "red.unl.edu")
        cp.add_section("bdii")
        cp.set("bdii", "endpoint", "ldap://is.grid.iu.edu:2170/")
        cp.add_section("classic_se")
        cp.set("classic_se", "advertise_se", "True")
        cp.set("classic_se", "host", "red.unl.edu")
        cp.set("classic_se", "port", "2811")
        cp.set("classic_se", "name", "T2_Nebraska_classicSE")
        cp.set("classic_se", "unique_name", "red.unl.edu_se")
        cp.set("classic_se", "default", "/opt/osg/data/$VO")
        cp.set("classic_se", "space", "%i, %i, %i" % (1000**2, 10*1000**2,
            11*1000**2))
        cp.add_section("vo")
        cp.set("vo", "user_vo_map", "test_configs/red-osg-user-vo-map.txt")

        self.output = cStringIO.StringIO()
        old_stdout = sys.stdout
        sys.stdout = self.output
        try:
            print_classicSE(cp)
        finally:
            sys.stdout = old_stdout
        self.output.seek(0)
        self.entries = read_ldap(self.output, multi=True)
        ses = []
        sas = []
        for entry in self.entries:
            if 'GlueSE' in entry.objectClass:
                ses.append(entry)
            if 'GlueSA' in entry.objectClass:
                sas.append(entry)
        self.ses = ses
        self.sas = sas
Пример #32
0
 def test_output_pf(self):
     """
     Test the sample output from prairiefire.unl.edu.  Should represent
     a "simple" Condor setup, no groups or priorities.
     """
     os.environ['GIP_TESTING'] = '1'
     path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
         "condor.py --config=test_configs/condor_test.conf")
     fd = os.popen(path)
     entries = read_ldap(fd)
     self.assertEquals(fd.close(), None)
     has_ce = False
     ce_name = socket.gethostname()
     for entry in entries:
         if 'GlueCE' in entry.objectClass:
             has_ce = True
             self.assertEquals(entry.glue['CEStateTotalJobs'], '6')
             self.assertEquals(entry.glue['CEStateRunningJobs'], '4')
             self.assertEquals(entry.glue['CEStateFreeCPUs'], '77')
             self.assertEquals(entry.glue['CEPolicyAssignedJobSlots'], '81')
             self.assertEquals(entry.glue['CEUniqueID'], \
                 '%s:2119/jobmanager-condor-default' % ce_name)
     self.assertEquals(has_ce, True)
Пример #33
0
 def test_output_pf(self):
     """
     Test the sample output from prairiefire.unl.edu.  Should represent
     a "simple" Condor setup, no groups or priorities.
     """
     os.environ['GIP_TESTING'] = '1'
     path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
         "condor.py --config=test_configs/condor_test.conf")
     fd = os.popen(path)
     entries = read_ldap(fd)
     self.assertEquals(fd.close(), None)
     has_ce = False
     ce_name = socket.gethostname()
     for entry in entries:
         if 'GlueCE' in entry.objectClass:
             has_ce = True
             self.assertEquals(entry.glue['CEStateTotalJobs'], '6')
             self.assertEquals(entry.glue['CEStateRunningJobs'], '4')
             self.assertEquals(entry.glue['CEStateFreeCPUs'], '77')
             self.assertEquals(entry.glue['CEPolicyAssignedJobSlots'], '81')
             self.assertEquals(entry.glue['CEUniqueID'], \
                 '%s:2119/jobmanager-condor-default' % ce_name)
     self.assertEquals(has_ce, True)
Пример #34
0
 def test_lbl_entries(self):
     """
     Checks the LBL information, as they don't have PBSPro
     """
     # Switch commands over to the LBL ones:
     old_commands = dict(gip_testing.commands)
     try:
         os.environ['GIP_TESTING'] = 'suffix=lbl'
         path = os.path.expandvars("$GIP_LOCATION/libexec/" \
             "osg-info-provider-pbs.py --config=test_configs/red.conf")
         fd = os.popen(path)
         entries = read_ldap(fd)
         self.assertEquals(fd.close(), None)
     finally:
         gip_testing.commands = old_commands
     for entry in entries:
         if 'GlueCE' in entry.objectClass:
             self.assertEquals(entry.glue['CEStateTotalJobs'], '6')
             self.assertEquals(entry.glue['CEStateRunningJobs'], '6')
             self.assertEquals(entry.glue['CEStateFreeCPUs'], '51')
             self.assertEquals(entry.glue['CEStateFreeJobSlots'], '51')
             self.assertEquals(entry.glue['CEPolicyAssignedJobSlots'], '60')
             self.assertEquals(entry.glue['CEUniqueID'], \
                 'red.unl.edu:2119/jobmanager-pbs-batch')
Пример #35
0
 def test_lbl_entries(self):
     """
     Checks the LBL information, as they don't have PBSPro
     """
     # Switch commands over to the LBL ones:
     old_commands = dict(gip_testing.commands)
     try:
         os.environ['GIP_TESTING'] = 'suffix=lbl'
         path = os.path.expandvars("$GIP_LOCATION/libexec/" \
             "osg-info-provider-pbs.py --config=test_configs/red.conf")
         fd = os.popen(path)
         entries = read_ldap(fd)
         self.assertEquals(fd.close(), None)
     finally:
         gip_testing.commands = old_commands
     for entry in entries:
         if 'GlueCE' in entry.objectClass:
             self.assertEquals(entry.glue['CEStateTotalJobs'], '6')
             self.assertEquals(entry.glue['CEStateRunningJobs'], '6')
             self.assertEquals(entry.glue['CEStateFreeCPUs'], '51')
             self.assertEquals(entry.glue['CEStateFreeJobSlots'], '51')
             self.assertEquals(entry.glue['CEPolicyAssignedJobSlots'], '60')
             self.assertEquals(entry.glue['CEUniqueID'], \
                 'red.unl.edu:2119/jobmanager-pbs-batch')
Пример #36
0
    def test_groups_output(self):
        """
        Look at the group output from UCSD.
           - Check to make sure a GlueCE is defined for each group.
           - Check to make sure the black/white lists are obeyed.
           - Check to make sure that the FreeSlots are right for GlueCE.
               (FreeSlots should not be more than group quota).
           - Check to make sure that the FreeSlots are right for GlueVOView.
               (FreeSlots should not be more than group quota).
        """
        os.environ['GIP_TESTING'] = 'suffix=ucsd'
        path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
            "condor.py --config=test_configs/ucsd_condor.conf")
        fd = os.popen(path)
        entries = read_ldap(fd, multi=True)
        tmpl = "osg-gw-2.t2.ucsd.edu:2119/jobmanager-condor-%s"
        # This CE should *not* be present -> verifies blacklists work.
        try:
            entry = self.check_for_ce(tmpl % "group_cdf", entries)
            did_fail = False
        except:
            did_fail = True
        self.failUnless(did_fail, msg="CE %s is present, but shouldn't be" % \
            (tmpl % "group_cdf"))

        # Check lcgadmin group.  Verifies whitelist is being used.
        entry = self.check_for_ce(tmpl % "group_lcgadmin", entries)
        self.failUnless('VO:atlas' in entry.glue['CEAccessControlBaseRule'],
                        msg="ATLAS is not allowed in group_lcgadmin")
        self.failUnless('VO:cms' in entry.glue['CEAccessControlBaseRule'],
                        msg="CMS is not allowed in group_lcgadmin")

        # Check cmsprod group.  Verifies that the mapper is being used.
        entry = self.check_for_ce(tmpl % "group_cmsprod", entries)
        self.failUnless('VO:cms' in entry.glue['CEAccessControlBaseRule'],
                        msg="CMS is not allowed in group_cmsprod")

        # Check ligo group.  Verifies that whitelist overrides the mapper.
        entry = self.check_for_ce(tmpl % "group_ligo", entries)
        self.failUnless('VO:fmri' in entry.glue['CEAccessControlBaseRule'],
                        msg="FMRI is not allowed in group_cmsprod")
        self.failIf('VO:ligo' in entry.glue['CEAccessControlBaseRule'],
                    msg="LIGO is allowed in group_cmsprod")

        for entry in entries:
            if 'GlueCE' not in entry.objectClass:
                continue
            total = int(entry.glue['CEPolicyAssignedJobSlots'][0])
            assigned = int(entry.glue['CEPolicyAssignedJobSlots'][0])
            running = int(entry.glue['CEStateRunningJobs'][0])
            free = int(entry.glue['CEStateFreeJobSlots'][0])
            self.failUnless(total <= assigned, msg="Failed invariant: " \
                "TOTAL <= CE_ASSIGNED")
            self.failUnless(running <= total, msg="Failed invariant: " \
                "RUNNING <= TOTAL")
            self.failUnless(running <= assigned, msg="Failed invariant: " \
                "RUNNING <= CE_ASSIGNED")
            self.failUnless(free <= assigned - running, msg="Failed invariant" \
                ": CE_FREE_SLOTS <= CE_ASSIGNED - RUNNING")
            unique_id = entry.glue['CEUniqueID']
            ce_entry = entry
            for entry in entries:
                if 'GlueVOView' not in entry.objectClass:
                    continue
                chunk = 'GlueCEUniqueID=%s' % unique_id
                if chunk not in entry.glue['ChunkKey']:
                    continue
                vo_free = int(entry.glue['CEStateFreeJobSlots'][0])
                running = int(entry.glue['CEStateRunningJobs'][0])
                self.failUnless(vo_free <= free, msg="Failed invariant: " \
                    "VO_FREE_SLOTS <= CE_FREE_SLOTS")
                self.failUnless(vo_free <= assigned - running, msg="Failed " \
                    "invariant: VO_FREE_SLOTS <= CE_ASSIGNED - VO_RUNNING")
Пример #37
0
    def testGipOutput(self):
        """
GIP Validator
This check tests the ldif as reported by cemon for:

    * The following stanzas mustappear at least once:
        o GlueCEUniqueID
        o GlueVOViewLocalID
        o GlueSubClusterUniqueID
        o GlueClusterUniqueID
        o GlueCESEBindSEUniqueID
        o GlueCESEBindGroupCEUniqueID
        o GlueLocationLocalID
        o GlueServiceUniqueID
        o GlueSEUniqueID
        o GlueSEAccessProtocolLocalID
        o GlueSEControlProtocolLocalID
        o GlueSALocalID
    * The GlueSiteUniqueID and GlueSiteName must be the same for EGEE compatibility
    * The CE Stanza for the following conditions:
        o CEInfoDefaultSE != UNAVAILABLE
        o CEPolicyMaxCPUTime != 0
        o CEInfoTotalCPUs != 0
        o CEStateEstimatedResponseTime != 0
        o CEStateWorstResponseTime != 0
        o CEPolicyMaxWallClockTime != 0
    * The ldiff must have newlines appended after every key value combination
    * The all foriegn keys and chunk keys must have corresponding stanzas
    * All entries must conform to the attribute:value format
    * Test SRM ads for the following:
        o endpoint type is SRM
        o Version is 1.1.0 or 2.2.0 (1.1 or 2.2 generate warnings)
        o Site unique ID is not blank
        o Site unique ID is actual unique ID used for this site.
        o If dCache, make sure that the /srm/managervX string is correct.
    * Test DN ads for the following:
        o o=grid appears once
        o mds-vo-name=local appears once
        o mds-vo-name=<site> appears once
        o they appear in the correct order
        """
        if self.type.lower() == "url":  # we want info from the web status page
            fqdn = getFQDNBySiteName(self.cp, self.site)
            url = cp_get(self.cp, "gip_tests", "validator_url",
                         "") % (fqdn, self.site)
            fd = urllib2.urlopen(url)


#            line = fd.readline()
#            if "html" in fd.readline():
#                self.assertEquals(0, 1, "CEMon not reporting for site %s" % self.site)
#            fd = urllib2.urlopen(url)
        elif self.type.lower(
        ) == "gipinfo":  # We want info from the gip_info script
            path = os.path.expandvars("$GIP_LOCATION/bin/gip_info")
            fd = os.popen(path)
        else:  # assume we want to read from the bdii
            fd = query_bdii(self.cp,
                            query="",
                            base="mds-vo-name=%s,mds-vo-name=local,o=grid" %
                            self.site)

        self.entries = read_ldap(fd)
        self.site_id = self.getSiteUniqueID()

        self.test_existence_all()
        self.test_egee_site_unique_id()
        self.test_ce()
        self.test_site()
        self.test_missing_newlines()
        self.test_missing_values()
        self.test_dn()
        self.test_srm()
Пример #38
0
    check_cache(plugins, temp_dir, freshness)

    # Launch the providers and plugins
    pids = launch_modules(providers, provider_dir, temp_dir, timeout)
    pids += launch_modules(plugins, plugin_dir, temp_dir, timeout)

    # Wait for the results
    wait_children(pids, response)

    # Load results from the cache
    check_cache(providers, temp_dir, cache_ttl)
    check_cache(plugins, temp_dir, cache_ttl)

    # Create LDAP entries out of the static info
    static_fp = cStringIO.StringIO(static_info)
    entries = read_ldap(static_fp, multi=True)

    # Apply output from the providers
    entries = handle_providers(entries, providers)

    # Apply output from the plugins
    entries = handle_plugins(entries, plugins)

    # Finally, apply our special cases
    entries = handle_add_attributes(entries, add_attributes)
    entries = handle_alter_attributes(entries, alter_attributes)
    entries = handle_remove_attributes(entries, remove_attributes)

    # Return the LDAP or print it out.
    if return_entries:
        return entries
Пример #39
0
    def test_groups_output(self):
        """
        Look at the group output from UCSD.
           - Check to make sure a GlueCE is defined for each group.
           - Check to make sure the black/white lists are obeyed.
           - Check to make sure that the FreeSlots are right for GlueCE.
               (FreeSlots should not be more than group quota).
           - Check to make sure that the FreeSlots are right for GlueVOView.
               (FreeSlots should not be more than group quota).
        """
        os.environ['GIP_TESTING'] = 'suffix=ucsd'
        path = os.path.expandvars("$GIP_LOCATION/libexec/osg_info_provider_" \
            "condor.py --config=test_configs/ucsd_condor.conf")
        fd = os.popen(path)
        entries = read_ldap(fd, multi=True)
        tmpl = "osg-gw-2.t2.ucsd.edu:2119/jobmanager-condor-%s"
        # This CE should *not* be present -> verifies blacklists work.
        try:
            entry = self.check_for_ce(tmpl % "group_cdf", entries)
            did_fail = False
        except:
            did_fail = True
        self.failUnless(did_fail, msg="CE %s is present, but shouldn't be" % \
            (tmpl % "group_cdf"))

        # Check lcgadmin group.  Verifies whitelist is being used.
        entry = self.check_for_ce(tmpl % "group_lcgadmin", entries)
        self.failUnless('VO:atlas' in entry.glue['CEAccessControlBaseRule'],
            msg="ATLAS is not allowed in group_lcgadmin")
        self.failUnless('VO:cms' in entry.glue['CEAccessControlBaseRule'],
            msg="CMS is not allowed in group_lcgadmin")

        # Check cmsprod group.  Verifies that the mapper is being used.
        entry = self.check_for_ce(tmpl % "group_cmsprod", entries)
        self.failUnless('VO:cms' in entry.glue['CEAccessControlBaseRule'],
            msg="CMS is not allowed in group_cmsprod")
        
        # Check ligo group.  Verifies that whitelist overrides the mapper.
        entry = self.check_for_ce(tmpl % "group_ligo", entries)
        self.failUnless('VO:fmri' in entry.glue['CEAccessControlBaseRule'],
            msg="FMRI is not allowed in group_cmsprod")
        self.failIf('VO:ligo' in entry.glue['CEAccessControlBaseRule'],
            msg="LIGO is allowed in group_cmsprod")

        for entry in entries:
            if 'GlueCE' not in entry.objectClass:
                continue
            total = int(entry.glue['CEPolicyAssignedJobSlots'][0])
            assigned = int(entry.glue['CEPolicyAssignedJobSlots'][0])
            running = int(entry.glue['CEStateRunningJobs'][0])
            free = int(entry.glue['CEStateFreeJobSlots'][0])
            self.failUnless(total <= assigned, msg="Failed invariant: " \
                "TOTAL <= CE_ASSIGNED")
            self.failUnless(running <= total, msg="Failed invariant: " \
                "RUNNING <= TOTAL")
            self.failUnless(running <= assigned, msg="Failed invariant: " \
                "RUNNING <= CE_ASSIGNED")
            self.failUnless(free <= assigned - running, msg="Failed invariant" \
                ": CE_FREE_SLOTS <= CE_ASSIGNED - RUNNING")
            unique_id = entry.glue['CEUniqueID']
            ce_entry = entry
            for entry in entries:
                if 'GlueVOView' not in entry.objectClass:
                    continue
                chunk = 'GlueCEUniqueID=%s' % unique_id
                if chunk not in entry.glue['ChunkKey']:
                    continue
                vo_free = int(entry.glue['CEStateFreeJobSlots'][0])
                running = int(entry.glue['CEStateRunningJobs'][0])
                self.failUnless(vo_free <= free, msg="Failed invariant: " \
                    "VO_FREE_SLOTS <= CE_FREE_SLOTS")
                self.failUnless(vo_free <= assigned - running, msg="Failed " \
                    "invariant: VO_FREE_SLOTS <= CE_ASSIGNED - VO_RUNNING")
Пример #40
0
    check_cache(plugins, temp_dir, freshness)

    # Launch the providers and plugins
    pids = launch_modules(providers, provider_dir, temp_dir, timeout)
    pids += launch_modules(plugins, plugin_dir, temp_dir, timeout)

    # Wait for the results
    wait_children(pids, response)

    # Load results from the cache
    check_cache(providers, temp_dir, cache_ttl)
    check_cache(plugins, temp_dir, cache_ttl)

    # Create LDAP entries out of the static info
    static_fp = cStringIO.StringIO(static_info)
    entries = read_ldap(static_fp, multi=True)

    # Apply output from the providers
    entries = handle_providers(entries, providers)

    # Apply output from the plugins
    entries = handle_plugins(entries, plugins)

    # Finally, apply our special cases
    entries = handle_add_attributes(entries, add_attributes)
    entries = handle_alter_attributes(entries, alter_attributes)
    entries = handle_remove_attributes(entries, remove_attributes)

    # Return the LDAP or print it out.
    if return_entries:
        return entries
Пример #41
0
    def testGipOutput(self):
        """
GIP Validator
This check tests the ldif as reported by cemon for:

    * The following stanzas mustappear at least once:
        o GlueCEUniqueID
        o GlueVOViewLocalID
        o GlueSubClusterUniqueID
        o GlueClusterUniqueID
        o GlueCESEBindSEUniqueID
        o GlueCESEBindGroupCEUniqueID
        o GlueLocationLocalID
        o GlueServiceUniqueID
        o GlueSEUniqueID
        o GlueSEAccessProtocolLocalID
        o GlueSEControlProtocolLocalID
        o GlueSALocalID
    * The GlueSiteUniqueID and GlueSiteName must be the same for EGEE compatibility
    * The CE Stanza for the following conditions:
        o CEInfoDefaultSE != UNAVAILABLE
        o CEPolicyMaxCPUTime != 0
        o CEInfoTotalCPUs != 0
        o CEStateEstimatedResponseTime != 0
        o CEStateWorstResponseTime != 0
        o CEPolicyMaxWallClockTime != 0
    * The ldiff must have newlines appended after every key value combination
    * The all foriegn keys and chunk keys must have corresponding stanzas
    * All entries must conform to the attribute:value format
    * Test SRM ads for the following:
        o endpoint type is SRM
        o Version is 1.1.0 or 2.2.0 (1.1 or 2.2 generate warnings)
        o Site unique ID is not blank
        o Site unique ID is actual unique ID used for this site.
        o If dCache, make sure that the /srm/managervX string is correct.
    * Test DN ads for the following:
        o o=grid appears once
        o mds-vo-name=local appears once
        o mds-vo-name=<site> appears once
        o they appear in the correct order
        """
        if self.type.lower() == "url": # we want info from the web status page
            fqdn = getFQDNBySiteName(self.cp, self.site)
            url = cp_get(self.cp, "gip_tests", "validator_url", "") % (fqdn, self.site)
            fd = urllib2.urlopen(url)
#            line = fd.readline()
#            if "html" in fd.readline():
#                self.assertEquals(0, 1, "CEMon not reporting for site %s" % self.site)
#            fd = urllib2.urlopen(url)
        elif self.type.lower() == "gipinfo": # We want info from the gip_info script
            path = os.path.expandvars("$GIP_LOCATION/bin/gip_info")
            fd = os.popen(path)
        else: # assume we want to read from the bdii
            fd = query_bdii(self.cp, query="", base="mds-vo-name=%s,mds-vo-name=local,o=grid" % self.site)

        self.entries = read_ldap(fd)
        self.site_id = self.getSiteUniqueID()

        self.test_existence_all()
        self.test_egee_site_unique_id()
        self.test_ce()
        self.test_site()
        self.test_missing_newlines()
        self.test_missing_values()
        self.test_dn()
        self.test_srm()