Example #1
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only:
            # Load up the template for GlueService
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueService
            template = getTemplate("GlueService", "GlueServiceUniqueID")
            if not cp_getBoolean(cp, "site", "advertise_gums", True):
                log.info("Not advertising authorization service.")
                return
            else:
                log.info("Advertising authorization service.")
    
            try:
                authfile = open('/etc/grid-security/gsi-authz.conf', 'r')
                authlines = authfile.readlines()
                authmod = re.compile('^(?!#)(.*)libprima_authz_module')
                for line in authlines:
                    m = authmod.match(line)
                    if m:
                        publish_gums(cp, template)
                        return
            except IOError:
                log.info("/etc/grid-security/gsi-authz.conf not found; assuming gridmap file authorization")
            
            publish_gridmap_file(cp, template)                

    except Exception, e:
        # Log error, then report it via stderr.
        log.exception(e)
        sys.stdout = sys.stderr
        raise
Example #2
0
def main():
    """
    The primary wrapper function for emitting GLUE for storage elements.
    """
    cp = config()

    # Handle full-fledged SEs
    found_se = False
    for section in cp.sections():
        # need to search for sections with "_" because if you are only
        # advertising a classic SE, then you do NOT want to run the
        # handle_SE function or you will get duplicate and incorrect info
        if section.lower().startswith("se_"):
            advertise_se = cp_getBoolean(cp, section, "advertise_se", True)
            if advertise_se:
                handle_SE(cp, section)
            found_se = True
    if found_se == False and 'se' in cp.sections():
        handle_SE(cp, 'se')

    # Handle the "classic" SE.
    se_only = cp_getBoolean(cp, "gip", "se_only", False)
    if not se_only:
        try:
            print_classicSE(cp)
        except Exception, e:
            log.exception(e)
Example #3
0
def getSEList(cp, classicSEs=True):
    """
    Return a list of all the SE's at this site.

    @param cp: Site configuration.
    @keyword classicSEs: Return list should contain classicSEs; default is True.
    @returns: List of strings containing all the local SE unique_ids.
    """
    simple = cp.getboolean(cesebind, 'simple')
    se_list = []
    if simple:
        try:
            if cp_getBoolean(cp, se, 'advertise_se', True):
                se_list = [cp.get(se, 'unique_name')]
        except:
            pass
        for sect in cp.sections():
            if not sect.lower().startswith('se') or sect.lower() == 'se':
                continue
            if not cp_getBoolean(cp, sect, 'advertise_se', True):
                continue
            try:
                se_list += [cp.get(sect, 'unique_name')]
            except:
                pass
    else:
        try:
            se_list = list(split_re.split(cp.get(cesebind, 'se_list')))
        except:
            se_list = getSEList(cp, classicSEs=classicSEs)
    if classicSEs:
        se_list.extend(getClassicSEList(cp))
    se_list = list(Set(se_list))
    return se_list
Example #4
0
def main():
    """
    The primary wrapper function for emitting GLUE for storage elements.
    """
    cp = config()

    # Handle full-fledged SEs
    found_se = False
    for section in cp.sections():
        # need to search for sections with "_" because if you are only
        # advertising a classic SE, then you do NOT want to run the
        # handle_SE function or you will get duplicate and incorrect info
        if section.lower().startswith("se_"):
            advertise_se = cp_getBoolean(cp, section, "advertise_se", True)
            if advertise_se:
                handle_SE(cp, section)
            found_se = True
    if found_se == False and 'se' in cp.sections():
        handle_SE(cp, 'se')

    # Handle the "classic" SE.
    se_only = cp_getBoolean(cp, "gip", "se_only", False)
    if not se_only:
        try:
            print_classicSE(cp)
        except Exception, e:
            log.exception(e)
Example #5
0
def getDefaultSE(cp):
    global _defaultSE
    if _defaultSE:
        return _defaultSE
    default_se = cp_get(cp, "se", "name", "UNKNOWN")
    # if [se] name: ??? is "UNAVAILABLE" or not set, then try to get the
    # default_se
    if default_se == "UNKNOWN" or default_se == "UNAVAILABLE":
        default_se = cp_get(cp, "se", "default_se", "UNAVAILABLE")
    # if it is still UNAVAILABLE or not set, check to see if the classic SE
    # is being advertised and use that
    if default_se == "UNAVAILABLE" and cp_getBoolean(cp, "classic_se",
            "advertise_se", True):
        fallback_name = cp_get(cp, "site", "unique_name", "UNKNOWN") + \
            "_classicSE"
        default_se = cp_get(cp, "classic_se", "name", fallback_name)

    current_se = None
    for sect in cp.sections():
        if not sect.lower().startswith('se'):
            continue
        try:
            current_se = cp.get(sect, 'name')
        except:
            continue
        if cp_getBoolean(cp, sect, "default", False):
            _defaultSE = current_se
            return current_se
    if default_se == 'UNKNOWN' and current_se:
        _defaultSE = current_se
        return current_se
    _defaultSE = default_se
    return default_se
Example #6
0
def getSEList(cp, classicSEs=True):
    """
    Return a list of all the SE's at this site.

    @param cp: Site configuration.
    @keyword classicSEs: Return list should contain classicSEs; default is True.
    @returns: List of strings containing all the local SE unique_ids.
    """
    simple = cp.getboolean(cesebind, 'simple')
    se_list = []
    if simple:
        try:
            if cp_getBoolean(cp, se, 'advertise_se', True):
                se_list = [cp.get(se, 'unique_name')]
        except:
            pass
        for sect in cp.sections():
            if not sect.lower().startswith('se') or sect.lower() == 'se':
                continue
            if not cp_getBoolean(cp, sect, 'advertise_se', True):
                continue
            try:
                se_list += [cp.get(sect, 'unique_name')]
            except:
                pass
    else:
        try:
            se_list = list(split_re.split(cp.get(cesebind, 'se_list')))
        except:
            se_list = getSEList(cp, classicSEs=classicSEs)
    if classicSEs:
        se_list.extend(getClassicSEList(cp))
    se_list = list(Set(se_list))
    return se_list
Example #7
0
def getPort(cp):
    port = 2119
    if cp_getBoolean(cp, 'cream', 'enabled', False):
        port = 8443
    if cp_getBoolean(cp, 'htcondorce', 'enabled', False):
        port = getHTCondorCEPort()
    return port
Example #8
0
def getGramVersion(cp):
    gramVersion = '\n' + 'GlueCEInfoGRAMVersion: 5.0'
    if cp_getBoolean(cp, 'cream', 'enabled', False):
        gramVersion = ''
    if cp_getBoolean(cp, 'htcondorce', 'enabled', False):
        gramVersion = ''

    return gramVersion
Example #9
0
def _generateSubClusterHelper(cp, section):
    """
    Private helper function for generateSubClusters; do not use.
    
    Implementation note: We create a bunch of variables but never
       appear to use them - we actually do return the local dictionary
       at the end of the function.  Just being syntactically lazy.
    """
    #pylint: disable-msg=W0612
    # Names
    subCluster = cp_get(cp, section, "name", cluster)
    subClusterUniqueID = cp_get(cp, section, "unique_name", subCluster)
    clusterUniqueID = getClusterID(cp)

    # Host statistics
    clockSpeed = cp_getInt(cp, section, "cpu_speed_mhz", 999999999)
    cpuCount = cp_getInt(cp, section, "cpus_per_node", 2)
    model = cp_get(cp, section, "cpu_model", 'UNDEFINEDVALUE')
    platform = cp_get(cp, section, "platform", "UNKNOWN")
    vendor = cp_get(cp, section, "cpu_vendor", 'UNDEFINEDVALUE')
    cores_per_cpu = cp_getInt(cp, section, "cores_per_cpu", 2)
    si2k = cp_getInt(cp, section, "SI00", 2000)
    sf2k = cp_getInt(cp, section, "SF00", 2000)
    hepspec = cp_getInt(cp, section, "HEPSPEC", 0)
    ram = cp_getInt(cp, section, "ram_size", 1000*cpuCount*cores_per_cpu)
    cores = cp_getInt(cp, section, "total_cores", 999999999)
    if cores_per_cpu != 0:
        cpus = cp_getInt(cp, section, "total_cpus", cores/cores_per_cpu)
    else:
        cpus = 0
    virtualMem = ram + cp_getInt(cp, section, "swap_size", 0)
    inboundIP = cp_getBoolean(cp, section, "inbound_network", False)
    outboundIP = cp_getBoolean(cp, section, "outbound_network", True)
    inboundIP = ldap_boolean(inboundIP)
    outboundIP = ldap_boolean(outboundIP)

    # OS Stats
    osName, osRelease, osVersion = getRelease()

    # Temp directories
    default_tmp = cp_get(cp, "osg_dirs", "tmp", cp_get(cp, "osg_dirs", "data", \
             "/tmp"))
    wn_tmp = cp_get(cp, "osg_dirs", "wn_tmp", "/tmp")
    tmp = cp_get(cp, section, "tmp", default_tmp)
    if notDefined(tmp):
        tmp = default_tmp

    app_attr = 'GlueHostApplicationSoftwareRunTimeEnvironment'
    apps = getApplications(cp)
    applications = '\n'.join(['%s: %s' % (app_attr, i['locationId']) for i in \
        apps if i['locationId']])
    applications += '\n'
        
    # BDII stuff
    bdii = cp_get(cp, "bdii", "endpoint", "ldap://is.grid.iu.edu:2170")

    return locals()
Example #10
0
def getCEImpl(cp):
    ceImpl = 'Globus'
    ceImplVersion = cp_get(cp, ce, 'globus_version', '4.0.6')    
    if cp_getBoolean(cp, 'cream', 'enabled', False):
        ceImpl = 'CREAM'
        ceImplVersion = getOSGVersion(cp)
    if cp_getBoolean(cp, 'htcondorce', 'enabled', False):
        ceImpl = 'HTCondorCE'
        ceImplVersion = getHTCondorCEVersion(cp)
    return (ceImpl, ceImplVersion)
Example #11
0
def buildCEUniqueID(cp, ce_name, batch, queue):
    ce_prefix = 'jobmanager'
    if cp_getBoolean(cp, 'cream', 'enabled', False):
        ce_prefix = 'cream'
    if cp_getBoolean(cp, 'htcondorce', 'enabled', False):
        ce_prefix = 'htcondorce'

    port = getPort(cp)
    ce_unique_id = '%s:%d/%s-%s-%s' % (ce_name, port, ce_prefix, batch, queue)
    return ce_unique_id
Example #12
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only:
            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            osg_grid = cp_get(cp, "osg_dirs", "grid_dir", None)
    
            if not osg_grid:
                raise RuntimeError('grid_dir ($OSG_GRID) not defined!')
                
            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {'locationId':   'OSG_GRID',
                        'subClusterId': subClusterId,
                        'clusterId':    cluster_id,
                        'locationName': 'OSG_GRID',
                        'version':      1.0,
                        'path':         osg_grid,
                       }
        
                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)
            
    except Exception, e:
        # Log error, then report it via stderr.
        log.error(e)
        sys.stdout = sys.stderr
        raise
Example #13
0
def handle_SE(cp, section):
    """
    Run a provider for one SE.
    """
    # if you only have a classic SE, there will still be a [se] section
    # with the default_se option.  This will result in a GlueService
    # stanza being written with garbage information in it. So, check
    # for default_se = "UNKNOWN" or "UNAVAILABLE" and return if it does
    #
    # default_se is set in the [Storage] section in config.ini and is
    # required by configure-osg.py
    default_se = cp_get(cp, "se", "default_se", "UNKNOWN")
    if default_se == "UNAVAILABLE" or default_se == "UNKNOWN": return

    impl = cp_get(cp, section, "implementation", "UNKNOWN")
    provider_impl = cp_get(cp, section, "provider_implementation", "UNKNOWN")
    if provider_impl == "UNKNOWN" and not cp_getBoolean(cp, section,
            "dynamic_dcache", False):
        provider_impl = 'static'
    se_class, cp = determine_provider(provider_impl, impl, cp)
    se = se_class(cp, section=section)
    log.info("Outputting SE %s; implementation %s; provider %s" % (se.getName(),
        impl, provider_impl))
    try:
        se.run()
    except Exception, e:
        log.exception(e)
Example #14
0
 def test_gip_conf(self):
     """
     Make sure that the $GIP_LOCATION/etc/gip.conf file is read.
     """
     old_gip_location = os.environ['GIP_LOCATION']
     tmpdir = tempfile.mkdtemp()
     try:
         os.environ['GIP_LOCATION'] = tmpdir
         etc_dir = os.path.join(tmpdir, 'etc')
         try:
             os.mkdir(etc_dir)
             cp_orig = ConfigParser.ConfigParser()
             cp_orig.add_section("gip_test")
             cp_orig.set("gip_test", "gip_conf", "True")
             gip_conf = os.path.join(etc_dir, 'gip.conf')
             fp = open(gip_conf, 'w')
             try:
                 cp_orig.write(fp)
                 fp.close()
                 cp = ConfigParser.ConfigParser()
                 cp.read([gip_conf])
                 result = cp_getBoolean(cp, "gip_test", "gip_conf", False)
                 self.failUnless(result, msg="Failed to load $GIP_LOCATION"\
                     "/etc/gip.conf")
             finally:
                 os.unlink(gip_conf)
         finally:
             os.rmdir(etc_dir)
     finally:
         os.rmdir(tmpdir)
         os.environ['GIP_LOCATION'] = old_gip_location
Example #15
0
def handle_SE(cp, section):
    """
    Run a provider for one SE.
    """
    # if you only have a classic SE, there will still be a [se] section
    # with the default_se option.  This will result in a GlueService
    # stanza being written with garbage information in it. So, check
    # for default_se = "UNKNOWN" or "UNAVAILABLE" and return if it does
    #
    # default_se is set in the [Storage] section in config.ini and is
    # required by configure-osg.py
    default_se = cp_get(cp, "se", "default_se", "UNKNOWN")
    if default_se == "UNAVAILABLE" or default_se == "UNKNOWN": return

    impl = cp_get(cp, section, "implementation", "UNKNOWN")
    provider_impl = cp_get(cp, section, "provider_implementation", "UNKNOWN")
    if provider_impl == "UNKNOWN" and not cp_getBoolean(
            cp, section, "dynamic_dcache", False):
        provider_impl = 'static'
    se_class, cp = determine_provider(provider_impl, impl, cp)
    se = se_class(cp, section=section)
    log.info("Outputting SE %s; implementation %s; provider %s" %
             (se.getName(), impl, provider_impl))
    try:
        se.run()
    except Exception, e:
        log.exception(e)
Example #16
0
def interpolateConfig(cp):

    grid = cp_get(cp, "site", "group", "")
    if cp_getBoolean(cp, "gip_tests", "oim_aware", False):
        sitelist_cmd = "wget -O - http://oim.grid.iu.edu/pub/resource/show.php?format=plain-text 2>/dev/null | grep \",%s,\" | grep \",CE\" | cut -f1 -d," % grid
        sitelist = runCommand(sitelist_cmd).read().split()
        sitelist = ",".join(sitelist)
        cp.set("gip_tests", "site_names", sitelist)
    else:
        if cp_get(cp, "gip_tests", "site_names", "") == "":
            cp.set("gip_tests", "site_names", cp_get(cp, "site", "name", ""))

    if cp_get(cp, "gip_tests", "site_dns", "") == "":
        host_parts = cp_get(cp, "ce", "name", "").split('.')
        site_dns = "%s.%s" % (host_parts[:-2], host_parts[:-1])
        cp.set("gip_tests", "site_dns", site_dns)

    if cp_get(cp, "gip_tests", "required_site", "") == "":
        cp.set("gip_tests", "required_sites",
               cp_get(cp, "gip_tests", "site_names", ""))

    cp.set("gip_tests", "bdii_port", "2170")
    cp.set("gip_tests", "egee_port", "2170")
    cp.set(
        "gip_tests", "interop_url",
        "http://oim.grid.iu.edu/publisher/get_osg_interop_bdii_ldap_list.php?grid=%s&format=html"
        % grid)

    if strContains(grid, "ITB"):
        cp.set("bdii", "endpoint", "ldap://is-itb.grid.iu.edu:2170")
        cp.set("gip_tests", "bdii_addr", "is-itb.grid.iu.edu")
        cp.set("gip_tests", "egee_bdii", "pps-bdii.cern.ch")
        cp.set(
            "gip_tests", "egee_bdii_conf_url",
            "http://egee-pre-production-service.web.cern.ch/egee-pre-production-service/bdii/pps-all-sites.conf"
        )
        web_server = "http://is-itb.grid.iu.edu"
    else:
        cp.set("gip_tests", "bdii_addr", "is.grid.iu.edu")
        cp.set("gip_tests", "egee_bdii", "lcg-bdii.cern.ch")
        cp.set("gip_tests", "egee_bdii_conf_url",
               "http://lcg-bdii-conf.cern.ch/bdii-conf/bdii.conf")
        web_server = "http://is.grid.iu.edu"

    cp.set("gip_tests", "update_url", web_server + "/cgi-bin/status.cgi")
    cp.set("gip_tests", "schema_check_url",
           web_server + "/data/cemon_processed_osg/%s.processed?which=%s")
    cp.set("gip_tests", "validator_url",
           web_server + "/data/cemon_processed_osg/%s.processed?which=%s")

    if cp_get(cp, "gip_tests", "compare_excludes", "") == "":
        compare_excludes = "GlueCEStateFreeJobSlots,GlueCEStateRunningJobs,GlueCEStateTotalJobs,GlueSiteLocation,GlueSAStateAvailableSpace,GlueSAStateUsedSpace"
        cp.set("gip_tests", "compare_excludes", compare_excludes)

    if cp_get(cp, "gip_tests", "enable_glite", "") == "":
        cp.set("gip_tests", "enable_glite", "False")

    if cp_get(cp, "gip_tests", "results_dir", "") == "":
        results_dir = os.path.expandvars("$GIP_LOCATION/../apache/htdocs/")
        cp.set("gip_tests", "results_dir", results_dir)
Example #17
0
def parseNodes(cp):
    """
    Parse the condor nodes.

    @param cp: ConfigParser object for the GIP
    @returns: A tuple consisting of the total, claimed, and unclaimed nodes.
    """
    global _nodes_cache #pylint: disable-msg=W0603
    if _nodes_cache:
        return _nodes_cache
    subtract = cp_getBoolean(cp, "condor", "subtract_owner", True)
    log.debug("Parsing condor nodes.")
    constraint = cp_get(cp, "condor", "status_constraint", "TRUE")
    fp = condorCommand(condor_status, cp, {'constraint': constraint})
    handler = ClassAdParser('Name', ['State'])
    parseCondorXml(fp, handler)
    total = 0
    claimed = 0
    unclaimed = 0
    for info in handler.getClassAds().values():
        total += 1
        if 'State' not in info:
            continue
        if info['State'] == 'Claimed':
            claimed += 1
        elif info['State'] == 'Unclaimed':
            unclaimed += 1
        elif subtract and info['State'] == 'Owner':
            total -= 1
    log.info("There are %i total; %i claimed and %i unclaimed." % \
             (total, claimed, unclaimed))
    _nodes_cache = total, claimed, unclaimed
    return total, claimed, unclaimed
Example #18
0
def runTest(cp, cls, out=None, per_site=True):
    """
    Given a test class, generate and run a test suite

    @param cp: Site configuration
    @type cp: ConfigParser
    @param cls: Test class to use to generate a test suite.  It is assumed
        that the constructor for this class has signature cls(cp, site_name).
        If per_site=False, then the signature is assumed to be cls().
    @type cls: class
    @keyword per_site: Set to true if there is one instance of the test class
        per site.
    @param out: A stream where the output from the test suite is logged
    @type out: stream
    """
    usexml = cp_getBoolean(cp, "gip_tests", "use_xml", default=False)
    if per_site:
        testSuite = generateTests(cp, cls, sys.argv[1:])
    else:
        testSuite = unittest.TestLoader().loadTestsFromTestCase(cls)

    if usexml:
        testRunner = GipUnittest.GipXmlTestRunner()
    else:
        if out is None:
            #testRunner = unittest.TextTestRunner(verbosity=2)
            testRunner = GipUnittest.GipTextTestRunner(verbosity=2)
        else:
            #testRunner = unittest.TextTestRunner(stream=out, verbosity=2)
            testRunner = GipUnittest.GipTextTestRunner(stream=out, verbosity=2)
    result = testRunner.run(testSuite)
    sys.exit(not result.wasSuccessful())
Example #19
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only:
            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            osg_grid = cp_get(cp, "osg_dirs", "grid_dir", None)

            if not osg_grid:
                raise RuntimeError('grid_dir ($OSG_GRID) not defined!')

            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {
                    'locationId': 'OSG_GRID',
                    'subClusterId': subClusterId,
                    'clusterId': cluster_id,
                    'locationName': 'OSG_GRID',
                    'version': 1.0,
                    'path': osg_grid,
                }

                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)

    except Exception, e:
        # Log error, then report it via stderr.
        log.error(e)
        sys.stdout = sys.stderr
        raise
Example #20
0
def runTest(cp, cls, out=None, per_site=True):
    """
    Given a test class, generate and run a test suite

    @param cp: Site configuration
    @type cp: ConfigParser
    @param cls: Test class to use to generate a test suite.  It is assumed
        that the constructor for this class has signature cls(cp, site_name).
        If per_site=False, then the signature is assumed to be cls().
    @type cls: class
    @keyword per_site: Set to true if there is one instance of the test class
        per site.
    @param out: A stream where the output from the test suite is logged
    @type out: stream
    """
    usexml = cp_getBoolean(cp, "gip_tests", "use_xml", default=False)
    if per_site:
        testSuite = generateTests(cp, cls, sys.argv[1:])
    else:
        testSuite = unittest.TestLoader().loadTestsFromTestCase(cls)

    if usexml:
        testRunner = GipUnittest.GipXmlTestRunner()
    else:
        if out is None:
            #testRunner = unittest.TextTestRunner(verbosity=2)
            testRunner = GipUnittest.GipTextTestRunner(verbosity=2)
        else:
            #testRunner = unittest.TextTestRunner(stream=out, verbosity=2)
            testRunner = GipUnittest.GipTextTestRunner(stream=out, verbosity=2)
    result = testRunner.run(testSuite)
    sys.exit(not result.wasSuccessful())
Example #21
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only:
    
            # Get the timestamp in the two formats we wanted
            epoch = str(time.time())
            now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
    
            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {'locationId':   'TIMESTAMP',
                        'subClusterId': subClusterId,
                        'clusterId':    cluster_id,
                        'locationName': 'TIMESTAMP',
                        'version':      epoch,
                        'path':         now,
                        }
    
                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)

    except Exception, e:
        # Log error, then report it via stderr.
        log.exception(e)
        sys.stdout = sys.stderr
        raise
Example #22
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only:

            # Get the timestamp in the two formats we wanted
            epoch = str(time.time())
            now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())

            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {
                    'locationId': 'TIMESTAMP',
                    'subClusterId': subClusterId,
                    'clusterId': cluster_id,
                    'locationName': 'TIMESTAMP',
                    'version': epoch,
                    'path': now,
                }

                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)

    except Exception, e:
        # Log error, then report it via stderr.
        log.exception(e)
        sys.stdout = sys.stderr
        raise
Example #23
0
def parseNodes(cp):
    """
    Parse the condor nodes.

    @param cp: ConfigParser object for the GIP
    @returns: A tuple consisting of the total, claimed, and unclaimed nodes.
    """
    global _nodes_cache #pylint: disable-msg=W0603
    if _nodes_cache:
        return _nodes_cache
    subtract = cp_getBoolean(cp, "condor", "subtract_owner", True)
    log.debug("Parsing condor nodes.")
    constraint = cp_get(cp, "condor", "status_constraint", "TRUE")
    fp = condorCommand(condor_status, cp, {'constraint': constraint})
    handler = ClassAdParser('Name', ['State'])
    parseCondorXml(fp, handler)
    total = 0
    claimed = 0
    unclaimed = 0
    for info in handler.getClassAds().values():
        total += 1
        if 'State' not in info:
            continue
        if info['State'] == 'Claimed':
            claimed += 1
        elif info['State'] == 'Unclaimed':
            unclaimed += 1
        elif subtract and info['State'] == 'Owner':
            total -= 1
    log.info("There are %i total; %i claimed and %i unclaimed." % \
             (total, claimed, unclaimed))
    _nodes_cache = total, claimed, unclaimed
    return total, claimed, unclaimed
Example #24
0
def getCEList(cp, extraCEs=[]):
    """
    Return a list of all the CE names at this site.

    If WS-GRAM is installed, this might additionally return some WS-GRAM
    entries (this feature is not yet implemented).

    @param cp: Site configuration
    @returns: List of strings containing all the local CE names.
    """
    jobman = cp.get(ce, "job_manager").strip().lower()
    hostnames = [cp.get(ce, 'name')] 
    hostnames += extraCEs

    prefix = 'jobmanager'
    port = 2119
    if cp_getBoolean(cp, 'cream', 'enabled', False):
        prefix = 'cream'
        port = 8443
    if cp_getBoolean(cp, 'htcondorce', 'enabled', False):
        prefix = 'htcondorce'
        port = getHTCondorCEPort()
    ce_names = ['%s:%d/%s-%s-%%s' % (hostname, port, prefix, jobman) for hostname in hostnames]

    ce_list = []
    if jobman == 'pbs':
        queue_entries = getPBSQueueList(cp)
    elif jobman == 'lsf':
        from gip.providers.lsf import bootstrapLSF
        bootstrapLSF(cp)
        queue_entries = getLSFQueueList(cp)
    elif jobman == 'condor':
        queue_entries = getCondorQueueList(cp)
    elif jobman == 'slurm':
        queue_entries = getSlurmQueueList(cp)
    elif jobman == 'sge':
        from gip.providers.sge import bootstrapSGE
        from gip_common import addToPath
        bootstrapSGE(cp)
        addToPath(cp_get(cp, "sge", "sge_path", "."))
        queue_entries = getSGEQueueList(cp)
    else:
        raise ValueError("Unknown job manager %s." % jobman)
    for queue in queue_entries:
        for ce_name in ce_names:
            ce_list.append(ce_name % queue)
    return ce_list
Example #25
0
def main():
    cp = config()
    if not cp_getBoolean(cp, "cluster", "advertise_cluster", True):
        return
    try:
        print_clusters(cp)
    except Exception, e:
        log.exception(e)
Example #26
0
def main():
    cp = config()
    if not cp_getBoolean(cp, 'cluster', 'advertise_cluster', True):
        return
    try:
        print_clusters(cp)
    except Exception, e:
        log.exception(e)
Example #27
0
def main(cp = None, return_entries=False):
    """
    Main method for the osg-info-wrapper script.  This script safely runs the
    plugin and provider modules, caching where necessary, and combines it with
    the static data.  It then outputs the final GLUE information for this site.
    """
    
    log.debug("Starting up the osg-info-wrapper.")
    if cp == None:
        cp = config()

    temp_dir = os.path.expandvars(cp_get(cp, "gip", "temp_dir", \
        gipDir("$GIP_LOCATION/var/tmp", '/var/cache/gip'))) 
    plugin_dir = os.path.expandvars(cp_get(cp, "gip", "plugin_dir", \
        gipDir("$GIP_LOCATION/plugins", '/usr/libexec/gip/plugins')))
    provider_dir = os.path.expandvars(cp_get(cp, "gip", "provider_dir", \
        gipDir("$GIP_LOCATION/providers", '/usr/libexec/gip/providers')))
    static_dir = os.path.expandvars(cp_get(cp, "gip", "static_dir", \
        gipDir("$GIP_LOCATION/var/ldif", '/etc/gip/ldif.d')))

    # Make sure that our directories exist.
    create_if_not_exist(temp_dir, plugin_dir, provider_dir, static_dir)

    # Load up our add, alter, and delete attributes
    add_attributes = os.path.expandvars(cp_get(cp, "gip", \
        "add_attributes", gipDir("$GIP_LOCATION/etc/add-attributes.conf",
                                 '/etc/gip/add-attributes.conf')))
    alter_attributes = os.path.expandvars(cp_get(cp, "gip", \
        "alter_attributes", gipDir("$GIP_LOCATION/etc/alter-attributes.conf",
                                   '/etc/gip/alter-attributes.conf')))
    remove_attributes = os.path.expandvars(cp_get(cp, "gip", \
        "remove_attributes", gipDir("$GIP_LOCATION/etc/remove-attributes.conf",
                                    '/etc/gip/remove-attributes.conf')))

    # Flush the cache if appropriate
    do_flush_cache = cp_getBoolean(cp, "gip", "flush_cache", False)
    if do_flush_cache:
        log.info("Flushing cache upon request.")
        flush_cache(temp_dir)

    # Load up our parameters
    freshness = cp_getInt(cp, "gip", "freshness", 300)
    cache_ttl = cp_getInt(cp, "gip", "cache_ttl", 600)
    response  = cp_getInt(cp, "gip", "response",  240)
    timeout = cp_getInt(cp, "gip",   "timeout",   240)

    try:
        os.setpgrp()
    except OSError, oe:
        # If launched from a batch system (condor), we might not have perms
        if oe.errno != 1:
            raise
Example #28
0
def main(cp=None, return_entries=False):
    """
    Main method for the osg-info-wrapper script.  This script safely runs the
    plugin and provider modules, caching where necessary, and combines it with
    the static data.  It then outputs the final GLUE information for this site.
    """

    log.debug("Starting up the osg-info-wrapper.")
    if cp == None:
        cp = config()

    temp_dir = os.path.expandvars(cp_get(cp, "gip", "temp_dir", \
        gipDir("$GIP_LOCATION/var/tmp", '/var/cache/gip')))
    plugin_dir = os.path.expandvars(cp_get(cp, "gip", "plugin_dir", \
        gipDir("$GIP_LOCATION/plugins", '/usr/libexec/gip/plugins')))
    provider_dir = os.path.expandvars(cp_get(cp, "gip", "provider_dir", \
        gipDir("$GIP_LOCATION/providers", '/usr/libexec/gip/providers')))
    static_dir = os.path.expandvars(cp_get(cp, "gip", "static_dir", \
        gipDir("$GIP_LOCATION/var/ldif", '/etc/gip/ldif.d')))

    # Make sure that our directories exist.
    create_if_not_exist(temp_dir, plugin_dir, provider_dir, static_dir)

    # Load up our add, alter, and delete attributes
    add_attributes = os.path.expandvars(cp_get(cp, "gip", \
        "add_attributes", vdtDir("$VDT_LOCATION/gip/etc/add-attributes.conf",
                                 '/etc/gip/add-attributes.conf')))
    alter_attributes = os.path.expandvars(cp_get(cp, "gip", \
        "alter_attributes", vdtDir("$VDT_LOCATION/gip/etc/alter-attributes.conf",
                                   '/etc/gip/alter-attributes.conf')))
    remove_attributes = os.path.expandvars(cp_get(cp, "gip", \
        "remove_attributes", vdtDir("$VDT_LOCATION/gip/etc/remove-attributes.conf",
                                    '/etc/gip/remove-attributes.conf')))

    # Flush the cache if appropriate
    do_flush_cache = cp_getBoolean(cp, "gip", "flush_cache", False)
    if do_flush_cache:
        log.info("Flushing cache upon request.")
        flush_cache(temp_dir)

    # Load up our parameters
    freshness = cp_getInt(cp, "gip", "freshness", 300)
    cache_ttl = cp_getInt(cp, "gip", "cache_ttl", 600)
    response = cp_getInt(cp, "gip", "response", 240)
    timeout = cp_getInt(cp, "gip", "timeout", 240)

    try:
        os.setpgrp()
    except OSError, oe:
        # If launched from a batch system (condor), we might not have perms
        if oe.errno != 1:
            raise
Example #29
0
def interpolateConfig(cp):
    
    grid = cp_get(cp, "site", "group", "")
    if cp_getBoolean(cp, "gip_tests", "oim_aware", False):
        sitelist_cmd = "wget -O - http://oim.grid.iu.edu/pub/resource/show.php?format=plain-text 2>/dev/null | grep \",%s,\" | grep \",CE\" | cut -f1 -d," % grid
        sitelist = runCommand(sitelist_cmd).read().split()
        sitelist = ",".join(sitelist)
        cp.set("gip_tests", "site_names", sitelist)
    else:
        if cp_get(cp, "gip_tests", "site_names", "") == "":
            cp.set("gip_tests", "site_names", cp_get(cp, "site", "name", ""))

    if cp_get(cp, "gip_tests", "site_dns", "") == "":
        host_parts = cp_get(cp, "ce", "name", "").split('.')
        site_dns = "%s.%s" % (host_parts[:-2], host_parts[:-1])
        cp.set("gip_tests", "site_dns", site_dns)

    if cp_get(cp, "gip_tests", "required_site", "") == "":
        cp.set("gip_tests", "required_sites", cp_get(cp, "gip_tests", "site_names", ""))

    cp.set("gip_tests", "bdii_port", "2170")
    cp.set("gip_tests", "egee_port", "2170")
    cp.set("gip_tests", "interop_url", "http://oim.grid.iu.edu/publisher/get_osg_interop_bdii_ldap_list.php?grid=%s&format=html" % grid)
        
    if strContains(grid, "ITB"):
        cp.set("bdii", "endpoint", "ldap://is-itb.grid.iu.edu:2170")
        cp.set("gip_tests", "bdii_addr", "is-itb.grid.iu.edu")
        cp.set("gip_tests", "egee_bdii", "pps-bdii.cern.ch")
        cp.set("gip_tests", "egee_bdii_conf_url", "http://egee-pre-production-service.web.cern.ch/egee-pre-production-service/bdii/pps-all-sites.conf")
        web_server = "http://is-itb.grid.iu.edu"
    else:
        cp.set("gip_tests", "bdii_addr", "is.grid.iu.edu")
        cp.set("gip_tests", "egee_bdii", "lcg-bdii.cern.ch")
        cp.set("gip_tests", "egee_bdii_conf_url", "http://lcg-bdii-conf.cern.ch/bdii-conf/bdii.conf")
        web_server = "http://is.grid.iu.edu"

    cp.set("gip_tests", "update_url", web_server + "/cgi-bin/status.cgi")
    cp.set("gip_tests", "schema_check_url", web_server + "/data/cemon_processed_osg/%s.processed?which=%s")
    cp.set("gip_tests", "validator_url", web_server + "/data/cemon_processed_osg/%s.processed?which=%s")

    if cp_get(cp, "gip_tests", "compare_excludes", "") == "":
        compare_excludes="GlueCEStateFreeJobSlots,GlueCEStateRunningJobs,GlueCEStateTotalJobs,GlueSiteLocation,GlueSAStateAvailableSpace,GlueSAStateUsedSpace"
        cp.set("gip_tests", "compare_excludes", compare_excludes)

    if cp_get(cp, "gip_tests", "enable_glite", "") == "":
        cp.set("gip_tests", "enable_glite", "False")

    if cp_get(cp, "gip_tests", "results_dir", "") == "":
        results_dir = os.path.expandvars("$GIP_LOCATION/../apache/htdocs/")
        cp.set("gip_tests", "results_dir", results_dir)
Example #30
0
    def getTests(self):
        test_dir = os.path.expandvars("$GIP_LOCATION/reporting")
        test_list = ls(test_dir)
        for test in test_list:
            if test in self.crit:
                self.critical_tests.append("%s/%s" % (test_dir, test))
            else:
                self.reports.append("%s/%s" % (test_dir, test))

        if cp_getBoolean(self.cp, "gip_tests", "enable_glite"):
            test_dir = os.path.expandvars("$GIP_LOCATION/reporting/glite_reports")
            tmp_list = ls(test_dir)
            for i in range(0, len(tmp_list)):
                tmp_list[i] = "%s/%s" % (test_dir, tmp_list[i])
            self.glite_tests.extend(tmp_list)
Example #31
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only and 'VDT_LOCATION' in os.environ:

            # get the VDT version
            vdt_version_cmd = os.path.expandvars(
                "$VDT_LOCATION/vdt/bin/") + 'vdt-version --no-wget'
            vdt_version_out = runCommand(vdt_version_cmd).readlines()

            gip_re = re.compile('Generic Information Provider\s+(.*?)\s*-.*')
            gip_version = 'UNKNOWN'
            for line in vdt_version_out:
                m = gip_re.match(line)
                if m:
                    gip_version = m.groups()[0]
                    break

            gip_version += '; $Revision$'

            # Get the timestamp in the two formats we wanted
            now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())

            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {
                    'locationId': 'GIP_VERSION',
                    'subClusterId': subClusterId,
                    'clusterId': cluster_id,
                    'locationName': 'GIP_VERSION',
                    'version': gip_version,
                    'path': now,
                }

                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)

    except Exception, e:
        # Log error, then report it via stderr.
        log.exception(e)
        sys.stdout = sys.stderr
        raise
Example #32
0
    def getTests(self):
        test_dir = os.path.expandvars("$GIP_LOCATION/reporting")
        test_list = ls(test_dir)
        for test in test_list:
            if test in self.crit:
                self.critical_tests.append("%s/%s" % (test_dir, test))
            else:
                self.reports.append("%s/%s" % (test_dir, test))

        if cp_getBoolean(self.cp, "gip_tests", "enable_glite"):
            test_dir = os.path.expandvars(
                "$GIP_LOCATION/reporting/glite_reports")
            tmp_list = ls(test_dir)
            for i in range(0, len(tmp_list)):
                tmp_list[i] = "%s/%s" % (test_dir, tmp_list[i])
            self.glite_tests.extend(tmp_list)
Example #33
0
    def runTests(self):
        self.getTests()
        # perfom the reports
        self.runList(self.reports, "reports")
        # perfom the glite reports
        self.runList(self.glite_tests, "glite")
        # perfom the critical tests
        self.runList(self.critical_tests, "critical")

        self.writeResultsPage()
        
        oim_plugin_enabled = cp_getBoolean(self.cp, "gip_tests", "enable_myosg_plugin", False)
        if oim_plugin_enabled:
            oim_plugin = os.path.expandvars("$GIP_LOCATION/reporting/plugins/OIM_XML_Aggregator.py")
            cmd = '/bin/bash -c "%(source)s; %(plugin)s %(args)s "' % ({"source": self.source_cmd, "plugin": oim_plugin, "args": self.args})
            runCommand(cmd)
Example #34
0
def getCEList(cp, extraCEs=[]):
    """
    Return a list of all the CE names at this site.

    If WS-GRAM is installed, this might additionally return some WS-GRAM
    entries (this feature is not yet implemented).

    @param cp: Site configuration
    @returns: List of strings containing all the local CE names.
    """
    jobman = cp.get(ce, "job_manager").strip().lower()
    hostnames = [cp.get(ce, 'name')]
    hostnames += extraCEs

    prefix = 'jobmanager'
    port = 2119
    if cp_getBoolean(cp, 'cream', 'enabled', False):
        prefix = 'cream'
        port = 8443
    ce_names = [
        '%s:%d/%s-%s-%%s' % (hostname, port, prefix, jobman)
        for hostname in hostnames
    ]

    ce_list = []
    if jobman == 'pbs':
        queue_entries = getPBSQueueList(cp)
    elif jobman == 'lsf':
        from gip.providers.lsf import bootstrapLSF
        bootstrapLSF(cp)
        queue_entries = getLSFQueueList(cp)
    elif jobman == 'condor':
        queue_entries = getCondorQueueList(cp)
    elif jobman == 'slurm':
        queue_entries = getSlurmQueueList(cp)
    elif jobman == 'sge':
        from gip.providers.sge import bootstrapSGE
        from gip_common import addToPath
        bootstrapSGE(cp)
        addToPath(cp_get(cp, "sge", "sge_path", "."))
        queue_entries = getSGEQueueList(cp)
    else:
        raise ValueError("Unknown job manager %s." % jobman)
    for queue in queue_entries:
        for ce_name in ce_names:
            ce_list.append(ce_name % queue)
    return ce_list
Example #35
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only and 'VDT_LOCATION' in os.environ:
    
            # get the VDT version
            vdt_version_cmd = os.path.expandvars("$VDT_LOCATION/vdt/bin/") + 'vdt-version --no-wget'
            vdt_version_out = runCommand(vdt_version_cmd).readlines()
    
            gip_re = re.compile('Generic Information Provider\s+(.*?)\s*-.*')
            gip_version = 'UNKNOWN'
            for line in vdt_version_out:
                m = gip_re.match(line)
                if m:
                    gip_version = m.groups()[0]
                    break
    
            gip_version += '; $Revision$'
    
            # Get the timestamp in the two formats we wanted
            now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
    
            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {'locationId':   'GIP_VERSION',
                        'subClusterId': subClusterId,
                        'clusterId':    cluster_id,
                        'locationName': 'GIP_VERSION',
                        'version':      gip_version,
                        'path':         now,
                       }
        
                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)
            
    except Exception, e:
        # Log error, then report it via stderr.
        log.exception(e)
        sys.stdout = sys.stderr
        raise
Example #36
0
 def getSESpace(self, gb=False, total=False):
     if cp_getBoolean(self._cp, self._section, 'use_df', False) or \
             self.status == False:
         # Let a configuration option override the use_df option.
         space = cp_get(self._cp, self._section, 'space', '')
         if space:
             try:
                 used, free, tot = eval(space, {}, {})
                 used, free, tot = int(used), int(free), int(tot)
             except:
                 used, free, tot = 0, 0, 0
         else:
             paths = sets.Set()
             # Lookup SA paths only if there's a single SA.
             # Otherwise, use the default path (otherwise we get a inf loop)
             if self.sas:
                 for sa in self.getSAs():
                     path = sa['path']
                     paths.add(path)
             else:
                 paths = [self.getPathForSA(space=None, \
                     section=self._section)]
             used, free, tot = 0, 0, 0
             for path in paths:
                 try:
                     stat_info = os.statvfs(path)
                     blocks = stat_info[statvfs.F_BLOCKS]
                     bsize = stat_info[statvfs.F_BSIZE]
                     avail = stat_info[statvfs.F_BFREE]
                 except Exception, e:
                     log.exception(e)
                     continue
                 used += (blocks - avail) * bsize / 1024.
                 free += avail * bsize / 1024.
                 tot += blocks * bsize / 1024.
         if total:
             if gb:
                 return int(used/1000.**2), int(free/1000.**2), \
                     int(tot/1000.**2)
             else:
                 return int(used), int(free), int(tot)
         else:
             if gb:
                 return int(used / 1000.**2), int(free / 1000.**2)
             else:
                 return int(used), int(free)
Example #37
0
 def getSESpace(self, gb=False, total=False):
     if cp_getBoolean(self._cp, self._section, 'use_df', False) or \
             self.status == False:
         # Let a configuration option override the use_df option.
         space = cp_get(self._cp, self._section, 'space', '')
         if space:
             try:
                 used, free, tot = eval(space, {}, {})
                 used, free, tot = int(used), int(free), int(tot)
             except:
                 used, free, tot = 0, 0, 0
         else:
             paths = sets.Set()
             # Lookup SA paths only if there's a single SA.
             # Otherwise, use the default path (otherwise we get a inf loop)
             if self.sas:
                 for sa in self.getSAs():
                     path = sa['path']
                     paths.add(path)
             else:
                 paths = [self.getPathForSA(space=None, \
                     section=self._section)]
             used, free, tot = 0, 0, 0
             for path in paths:
                 try:
                     stat_info = os.statvfs(path)
                     blocks = stat_info[statvfs.F_BLOCKS]
                     bsize = stat_info[statvfs.F_BSIZE]
                     avail = stat_info[statvfs.F_BFREE]
                 except Exception, e:
                     log.exception(e)
                     continue
                 used += (blocks-avail) * bsize / 1024.
                 free += avail          * bsize / 1024.
                 tot +=  blocks         * bsize / 1024.
         if total:
             if gb:
                 return int(used/1000.**2), int(free/1000.**2), \
                     int(tot/1000.**2)
             else:
                 return int(used), int(free), int(tot)
         else:
             if gb:
                 return int(used/1000.**2), int(free/1000.**2)
             else:
                 return int(used), int(free)
Example #38
0
def buildContactString(cp, batch, queue, ce_unique_id, log):
    contact_string = cp_get(cp, batch, 'job_contact', ce_unique_id)

    if contact_string.endswith("jobmanager-%s" % batch):
        contact_string += "-%s" % queue

    if cp_getBoolean(cp, 'cream', 'enabled', False) and not \
           contact_string.endswith('cream-%s' % batch):
        log.warning('CREAM CE enabled, but contact string in config.ini '
                    'does not end with "cream-%s"' % batch)
		
    if contact_string.endswith('cream-%s' % batch):
        contact_string += "-%s" % queue
        if not contact_string.startswith('https://'):
            contact_string = 'https://' + contact_string

    return contact_string
Example #39
0
def getSESections(cp):
    """
    For each SE section (non-classic), determine the corresponding section
    name.

    Returns a dict; the SE unique ID is the key, the section is the value.
    """
    se_map = {}
    for sect in cp.sections():
        if not sect.lower().startswith('se') or sect.lower() == 'se':
            continue
        if not cp_getBoolean(cp, sect, 'advertise_se', True):
            continue
        try:
            se_map[cp.get(sect, 'unique_name')] = sect
        except:
            pass
    return se_map
Example #40
0
def getClassicSEList(cp):
    """
    Return a list of all the ClassicSE's at this site

    @param cp: Site configuration
    @returns: List of all the ClassicSE's unique_ids
    """
    classic_list = []
    if cp_getBoolean(cp, "classic_se", "advertise_se", True):
        classicSE = cp_get(cp, "classic_se", "unique_name", None)
        if classicSE:
            classic_list = [classicSE]
        else:
            siteUniqueID = cp_get(cp, "site", "unique_name", "UNKNOWN")
            classicSE = siteUniqueID + "_classicSE"
            classic_list = [classicSE]

    return classic_list
Example #41
0
def getSESections(cp):
    """
    For each SE section (non-classic), determine the corresponding section
    name.

    Returns a dict; the SE unique ID is the key, the section is the value.
    """
    se_map = {}
    for sect in cp.sections():
        if not sect.lower().startswith('se') or sect.lower() == 'se':
            continue
        if not cp_getBoolean(cp, sect, 'advertise_se', True):
            continue
        try:
            se_map[cp.get(sect, 'unique_name')] = sect
        except:
            pass
    return se_map
Example #42
0
def getSESpace(cp, admin=None, gb=False, total=False, section=se):
    """
    Return the amount of space available at the SE.

    If se.dynamic_dcache=True, use dCache-based methods.
    Otherwise, use classic SE methods (do a df on the SE mounts).

    @param cp: Site configuration object
    @keyword admin: If a dCache provider, the dCache admin objects
    @keyword gb: Set to true to retun values in GB.
    @keyword total: Also return totals
    @return: used, free, total if total is True; otherwise, used, free.
      In GB if GB=True; otherwise, in KB.
    """
    if cp_getBoolean(cp, section, "dynamic_dcache", False):
        return getdCacheSESpace(cp, admin, gb, total)
    else:
        return getClassicSESpace(cp, gb=gb, total=total)
Example #43
0
def getClassicSEList(cp):
    """
    Return a list of all the ClassicSE's at this site

    @param cp: Site configuration
    @returns: List of all the ClassicSE's unique_ids
    """
    classic_list = []
    if cp_getBoolean(cp, "classic_se", "advertise_se", True):
        classicSE = cp_get(cp, "classic_se", "unique_name", None)
        if classicSE:
            classic_list = [classicSE]
        else:
            siteUniqueID = cp_get(cp, "site", "unique_name", "UNKNOWN")
            classicSE = siteUniqueID + "_classicSE"
            classic_list = [classicSE]

    return classic_list
Example #44
0
def getHTPCInfo(cp, batch, queue, log):
    # return tuple: (non-Glue HTPC information, htpc_max_slots)
    #  where htpc_max_slots is the admin-provided "maximum number of slots per job"

    htpcInfo = ('__GIP_DELETEME', 1)  # defaults

    if not cp_getBoolean(cp, batch, 'htpc_enabled', False):
        log.info("HTPC is disabled for batch %s" % batch)
        return htpcInfo

    log.info("HTPC is enabled for batch %s" % batch)
    whitelist = cp_getList(cp, batch, 'htpc_queues', [])
    blacklist = cp_getList(cp, batch, 'htpc_blacklist_queues', [])

    log.debug("HTPC whitelist: %s; HTPC blacklist %s: " % (whitelist, blacklist))

    if '*' not in whitelist and queue not in whitelist:
        log.info("HTPC Queue %s not in whitelist" % queue)
        return htpcInfo

    if queue in blacklist:
        log.info("HTPC Queue %s in blacklist" % queue)
        return htpcInfo
        
    defaultRSL = cp_get(cp, batch, 'htpc_rsl', '')
    log.debug("HTPC DefaultRSL: %s" % defaultRSL)
    queueRSL = cp_get(cp, batch, 'htpc_rsl_%s' % queue, '')

    if not queueRSL:
        queueRSL = defaultRSL
        
    if not queueRSL:
        log.info("HTPC RSL not found for queue %s" % queue)
        return htpcInfo


    htpcMaxSlots = cp_getInt(cp, batch, 'htpc_max_slots', 1)
    if htpcMaxSlots < 2:
        log.info("HTPC max slots equal to 1 or not set!")
        return htpcInfo

    # acbr stuff?

    return ('HTPCrsl: %s' % queueRSL, htpcMaxSlots)
Example #45
0
def print_classicSE(cp):
    """
    Emit the relevant GLUE entities for a ClassicSE.
    """
    if not cp_getBoolean(cp, "classic_se", "advertise_se", True):
        log.info("Not advertising a classic SE.")
        return
    else:
        log.info("Advertising a classic SE.")
    #if cp_getBoolean(cp, "se", "shares_fs_with_ce", False):
    #    log.info("Not advertising a classic SE because the SE shares a FS.")
    #    return

    status = cp_get(cp, "classic_se", "status", "Production")
    version = cp_get(cp, "classic_se", "version", "UNKNOWN")
    try:
        used, available, total = getClassicSESpace(cp, total=True, gb=True)
    except Exception, e:
        log.error("Unable to get SE space: %s" % str(e))
        used, available, total = 0, 0, 0
Example #46
0
def print_classicSE(cp):
    """
    Emit the relevant GLUE entities for a ClassicSE.
    """
    if not cp_getBoolean(cp, "classic_se", "advertise_se", True):
        log.info("Not advertising a classic SE.")
        return
    else:
        log.info("Advertising a classic SE.")
    #if cp_getBoolean(cp, "se", "shares_fs_with_ce", False):
    #    log.info("Not advertising a classic SE because the SE shares a FS.")
    #    return

    status = cp_get(cp, "classic_se", "status", "Production")
    version = cp_get(cp, "classic_se", "version", "UNKNOWN")
    try:
        used, available, total = getClassicSESpace(cp, total=True, gb=True)
    except Exception, e:
        log.error("Unable to get SE space: %s" % str(e))
        used, available, total = 0, 0, 0
Example #47
0
def main():
    cp = config()
    se_only = cp_getBoolean(cp, "gip", "se_only", False)
    if not se_only:
        job_manager = cp_get(cp, "ce", "job_manager", None)
        if job_manager:
            log.info("Using job manager %s" % job_manager)
        else:
           log.error("Job manager not specified!")
           sys.exit(2)
        if job_manager == 'pbs':
            pbs_main()
        elif job_manager == 'condor':
            condor_main()
        elif job_manager == 'sge':
            sge_main()
        elif job_manager == 'lsf':
            lsf_main()
        else:
            log.error("Unknown job manager: %s." % job_manager)
            sys.exit(1)
Example #48
0
    def runTests(self):
        self.getTests()
        # perfom the reports
        self.runList(self.reports, "reports")
        # perfom the glite reports
        self.runList(self.glite_tests, "glite")
        # perfom the critical tests
        self.runList(self.critical_tests, "critical")

        self.writeResultsPage()

        oim_plugin_enabled = cp_getBoolean(self.cp, "gip_tests",
                                           "enable_myosg_plugin", False)
        if oim_plugin_enabled:
            oim_plugin = os.path.expandvars(
                "$GIP_LOCATION/reporting/plugins/OIM_XML_Aggregator.py")
            cmd = '/bin/bash -c "%(source)s; %(plugin)s %(args)s "' % (
                {
                    "source": self.source_cmd,
                    "plugin": oim_plugin,
                    "args": self.args
                })
            runCommand(cmd)
Example #49
0
def main():
    cp = config()
    se_only = cp_getBoolean(cp, "gip", "se_only", False)
    if not se_only:
        job_manager = cp_get(cp, "ce", "job_manager", None)
        if job_manager:
            log.info("Using job manager %s" % job_manager)
        else:
           log.error("Job manager not specified!")
           sys.exit(2)
        if job_manager == 'pbs':
            pbs_main()
        elif job_manager == 'condor':
            condor_main()
        elif job_manager == 'sge':
            sge_main()
        elif job_manager == 'lsf':
            lsf_main()
        elif job_manager == 'slurm':
            slurm_main()
        else:
            log.error("Unknown job manager: %s." % job_manager)
            sys.exit(1)
Example #50
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only and 'VDT_LOCATION' in os.environ: 
    
            # get the VDT version
            vdt_version_cmd = os.path.expandvars("$VDT_LOCATION/vdt/bin/") + 'vdt-version --brief'
            vdt_version = runCommand(vdt_version_cmd).readlines()[0].strip()
            if (vdt_version == ""): vdt_version = "OLD_VDT"
            
            # Get the timestamp in the two formats we wanted
            now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
    
            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {'locationId':   'VDT_VERSION',
                        'subClusterId': subClusterId,
                        'clusterId':    cluster_id,
                        'locationName': 'VDT_VERSION',
                        'version':      vdt_version,
                        'path':         now,
                       }
        
                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)
            
    except Exception, e:
        # Log error, then report it via stderr.
        log.exception(e)
        sys.stdout = sys.stderr
        raise
Example #51
0
def print_CE(cp):
    slurmVersion = getLrmsInfo(cp)
    queueInfo = getQueueInfo(cp)
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    CE = getTemplate("GlueCE", "GlueCEUniqueID")
    try:
        excludeQueues = [i.strip() for i in cp_get(cp, "slurm", \
            "queue_exclude", "").split(',')]
    except:
        excludeQueues = []
    vo_queues = getVoQueues(cp)
    for queue, info in queueInfo.items():
        if queue in excludeQueues:
            continue
        info["lrmsVersion"] = slurmVersion
        info["job_manager"] = "slurm"

        # if no jobs are waiting in the queue, set the number of free slots
        # to (job_slots - running), or the total number of free slots on the cluster,
        # whichever is less.

        info["queue"] = queue
        info["ceName"] = ce_name

        unique_id = buildCEUniqueID(cp, ce_name, 'slurm', queue)
        ceImpl, ceImplVersion = getCEImpl(cp)
        port = getPort(cp)

        info['ceUniqueID'] = unique_id
        if "job_slots" not in info:
            log.error("no job_slots found for %s!" % queue)
        if "priority" not in info:
            info["priority"] = 0
        if "max_running" not in info:
            log.error("no max_running found for %s!" % queue)
        if "max_wall" not in info:
            info["max_wall"] = 1440

        info["free_slots"] = 0
        if info["wait"] == 0:
            freeSlots = info["job_slots"] - info["running"]
            if freeSlots > 0:
                info["free_slots"] = freeSlots

        ert, wrt = responseTimes(cp,
                                 info.get("running", 0),
                                 info.get("wait", 0),
                                 max_job_time=info["max_wall"])

        info['ert'] = ert
        info['wrt'] = wrt
        info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name)
        info['hostName'] = cp_get(cp, ce, 'host_name', ce_name)
        info['ceImpl'] = ceImpl
        info['ceImplVersion'] = ceImplVersion

        contact_string = buildContactString(cp, 'slurm', queue, unique_id, log)

        info['contact_string'] = contact_string
        info['app_dir'] = cp_get(cp, 'osg_dirs', 'app', "/UNKNOWN_APP")
        info['data_dir'] = cp_get(cp, 'osg_dirs', 'data', "/UNKNOWN_DATA")
        info['default_se'] = getDefaultSE(cp)

        if 'max_waiting' not in info:
            info['max_waiting'] = 999999
        if 'max_queuable' in info:
            info['max_total'] = info['max_queuable']
            info['free_slots'] = min(info['free_slots'], info['max_queuable'])
        else:
            info['max_total'] = info['max_waiting'] + info['max_running']
            info['free_slots'] = min(info['free_slots'], info['max_total'])

        # Enforce invariants:
        # max_total <= max_running
        # free_slots <= max_running
        info['max_total'] = min(info['max_total'], info['max_running'])
        info['free_slots'] = min(info['free_slots'], info['max_running'])

        info['assigned'] = info['job_slots']
        # Enforce invariants:
        # assigned <= max_running
        info['assigned'] = min(info['assigned'], info['max_running'])

        info['lrmsType'] = 'slurm'
        info['preemption'] = cp_get(cp, 'slurm', 'preemption', '0')
        acbr = ''
        has_vo = False
        for vo, queue2 in vo_queues:
            if queue == queue2:
                acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo
                has_vo = True
        if not has_vo:
            continue
        info['acbr'] = acbr[:-1]
        info['bdii'] = cp.get('bdii', 'endpoint')
        gramVersion = getGramVersion(cp)

        info['gramVersion'] = gramVersion
        info['port'] = port
        info['waiting'] = info['wait']
        info['referenceSI00'] = gip_cluster.getReferenceSI00(cp)
        info['clusterUniqueID'] = getClusterID(cp)

        extraCapabilities = ''
        if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

        htpcRSL, maxSlots = getHTPCInfo(cp, 'slurm', queue, log)
        info['max_slots'] = maxSlots

        if maxSlots > 1:
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'

        info['extraCapabilities'] = extraCapabilities
        info['htpc'] = htpcRSL

        printTemplate(CE, info)
    return queueInfo
Example #52
0
#!/usr/bin/env python

import os
import sys

if 'GIP_LOCATION' in os.environ:
    sys.path.insert(0, os.path.expandvars("$GIP_LOCATION/lib/python"))
from gip_common import config, cp_getBoolean
from gip.providers.cese_bind import main

if __name__ == '__main__':
    cp = config()
    se_only = cp_getBoolean(cp, "gip", "se_only", False)
    if not se_only:
        main()

Example #53
0
        myheld = sum([i.get('held', 0) for i in jinfo.values()], 0)

        max_wall = cp_getInt(cp, "condor", "max_wall", 1440)
        ert, wrt = responseTimes(cp,
                                 myrunning,
                                 myidle + myheld,
                                 max_job_time=max_wall * 60)

        referenceSI00 = gip_cluster.getReferenceSI00(cp)

        contact_string = buildContactString(cp, 'condor', group, ce_unique_id,
                                            log)
        htpcRSL, maxSlots = getHTPCInfo(cp, 'condor', group, log)

        extraCapabilities = ''
        if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

        if maxSlots > 1:
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'

        gramVersion = getGramVersion(cp)
        ceImpl, ceImplVersion = getCEImpl(cp)
        port = getPort(cp)

        # Build all the GLUE CE entity information.
        info = { \
            "ceUniqueID"     : ce_unique_id,
            'contact_string' : contact_string,
            "ceImpl"         : ceImpl,
            "ceImplVersion"  : ceImplVersion,
Example #54
0
File: sge.py Project: tiradani/gip
def print_CE(cp):
    SGEVersion = getLrmsInfo(cp)
    queueInfo, _ = getQueueInfo(cp)
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    ce_template = getTemplate("GlueCE", "GlueCEUniqueID")
    queueList = getQueueList(cp)

    vo_queues = getVoQueues(cp)

    default_max_waiting = 999999
    for queue in queueInfo.values():
        if 'name' not in queue or queue['name'] not in queueList:
            continue
        if queue['name'] == 'waiting':
            continue

        unique_id = buildCEUniqueID(cp, ce_name, 'sge', queue['name'])

        acbr = ''
        for vo, queue2 in vo_queues:
            if queue['name'] == queue2:
                acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo

        referenceSI00 = gip_cluster.getReferenceSI00(cp)
        contact_string = buildContactString(cp, 'sge', queue['name'],
                                            unique_id, log)

        extraCapabilities = ''
        if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

        htpcRSL, maxSlots = getHTPCInfo(cp, 'sge', queue, log)
        if maxSlots > 1:
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'

        gramVersion = getGramVersion(cp)
        port = getPort(cp)
        ceImpl, ceImplVersion = getCEImpl(cp)

        max_wall = queue["max_wall"]
        if cp.has_option("sge", "max_wall"):
            max_wall = cp_getInt(cp, "sge", "max_wall", 1440)

        info = { \
            "ceUniqueID" : unique_id,
            "ceName" : ce_name,
            "ceImpl" : ceImpl,
            "ceImplVersion" : ceImplVersion,
            "clusterUniqueID" : getClusterID(cp),
            "queue" : queue['name'],
            "priority" : queue['priority'],
            "lrmsType" : 'sge',
            "lrmsVersion" : SGEVersion,
            "job_manager" : "sge",
            "job_slots" : queue["slots_total"],
            "free_slots" : queue["slots_free"],
            "running" : queue["slots_used"],
            "status" : queue['status'],
            "total" : queue['slots_used'] + queue['waiting'],
            "ert" : 3600,
            "wrt" : 3600,
            "hostingCluster" : cp_get(cp, ce, 'hosting_cluster', ce_name),
            "hostName" : cp_get(cp, ce, 'host_name', ce_name),
            "contact_string" : contact_string,
            "app_dir" : cp_get(cp, 'osg_dirs', 'app', "/OSG_APP_UNKNOWN"),
            "data_dir" : cp_get(cp, 'osg_dirs', 'data', "/OSG_DATA_UNKNOWN"),
            "default_se" : getDefaultSE(cp),
            "max_running" : queue["slots_total"],
            "max_wall" : max_wall,
            "max_waiting" : default_max_waiting,
            "max_slots" : maxSlots,
            "max_total" : default_max_waiting + queue["slots_total"],
            "assigned" : queue["slots_used"],
            "preemption" : cp_get(cp, 'sge', 'preemption', '0'),
            "acbr" : acbr[:-1],
            "bdii": cp.get('bdii', 'endpoint'),
            "gramVersion" : gramVersion,
            "port" : port,
            "waiting" : queue['waiting'],
            "referenceSI00": referenceSI00,
            'extraCapabilities' : extraCapabilities,
            "htpc" : htpcRSL
        }
        printTemplate(ce_template, info)
    return queueInfo
Example #55
0
                group = name
                if len(all_group_info[name].get('vo', [])) == 1:
                    vo = all_group_info[name]['vo']
                else:
                    vo = 'unknown'
            else:
                unknown_users.add(name)
            continue

        vo_jobs = group_jobs.setdefault(group, {})

        # Add the information to the current dictionary.
        my_info = vo_jobs.get(vo, {"running":0, "idle":0, "held":0, \
            'max_running':0})
        addIntInfo(my_info, info, "running", "RunningJobs")
        if cp_getBoolean(cp, "condor", "count_flocked", True):
            addIntInfo(my_info, info, "running", "FlockedJobs")
        addIntInfo(my_info, info, "idle", "IdleJobs")
        addIntInfo(my_info, info, "held", "HeldJobs")
        addIntInfo(my_info, info, "max_running", "MaxJobsRunning")
        vo_jobs[vo] = my_info

    log.warning("The following users are non-grid users: %s" % \
        ", ".join(unknown_users))

    log.info("Job information: %s." % group_jobs)

    return group_jobs

_nodes_cache = []
def parseNodes(cp):
Example #56
0
def getGroupInfo(vo_map, cp): #pylint: disable-msg=C0103,W0613
    """
    Get the group info from condor

    The return value is a dictionary; the key is the vo name, the values are
    another dictionary of the form {'quota': integer, 'prio': integer}

    @param vo_map: VoMapper object
    @param cp: A ConfigParse object with the GIP config information
    @returns: A dictionary whose keys are VO groups and values are the quota
        and priority of the group.
    """

    configDaemon = ''
    if cp_getBoolean(cp, "condor", "query_only_local_condor", False):
        log.info("Only querying local Condor -- ignoring use_collector, if set!")
    else:
        if cp_getBoolean(cp, "condor", "use_collector", False):
            configDaemon = "-collector"
        else:
            configDaemon = "-negotiator"
                                    
    fp = condorCommand(condor_group, cp, {'daemon' : configDaemon})
    output = fp.read().split(',')
    if fp.close():
        log.info("No condor groups found.")
        return {}

    grouplist = []
    for group in output:
        grouplist.append(group.strip())
        
    excludedGroups = cp_getList(cp, "condor", "exclude_groups", [])
        
    log.debug("exclude_groups = %s" % excludedGroups)
    for excludedGroup in excludedGroups:
        excludedGroup = excludedGroup.strip()
        try:
            grouplist.remove(excludedGroup)
            log.debug("Removed excluded group %s" % excludedGroup)
        except ValueError:
            if excludedGroup != 'default':
                log.debug("Attempted to remove non-existent group %s" % excludedGroup)

    if not grouplist:
        log.info("No condor groups exist (after applying exclude_groups)")
        return{}
    
    retval = {}
    if (not (grouplist[0].startswith('Not defined'))) and \
            (len(grouplist[0]) > 0):
        for group in grouplist:
            quota = condorCommand(condor_quota, cp, \
                {'group': group, 'daemon': configDaemon}).read().strip()
            prio = condorCommand(condor_prio, cp, \
                {'group': group, 'daemon': configDaemon}).read().strip()
            vos = guessVO(cp, group)
            if not vos:
                continue
            curInfo = {'quota': 0, 'prio': 0, 'vos': vos}
            try:
                curInfo['quota'] += int(quota)
            except:
                pass
            try:
                curInfo['prio'] += int(prio)
            except:
                pass
            retval[group] = curInfo
    log.debug("The condor groups are %s." % ', '.join(retval.keys()))
    return retval
Example #57
0
def main():
    cp = config()
    se_only = cp_getBoolean(cp, "gip", "se_only", False)
    site_main()
    if not se_only:
        cluster_main()