예제 #1
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only:

            # Get the timestamp in the two formats we wanted
            epoch = str(time.time())
            now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())

            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {
                    'locationId': 'TIMESTAMP',
                    'subClusterId': subClusterId,
                    'clusterId': cluster_id,
                    'locationName': 'TIMESTAMP',
                    'version': epoch,
                    'path': now,
                }

                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)

    except Exception, e:
        # Log error, then report it via stderr.
        log.exception(e)
        sys.stdout = sys.stderr
        raise
예제 #2
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only:
    
            # Get the timestamp in the two formats we wanted
            epoch = str(time.time())
            now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
    
            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {'locationId':   'TIMESTAMP',
                        'subClusterId': subClusterId,
                        'clusterId':    cluster_id,
                        'locationName': 'TIMESTAMP',
                        'version':      epoch,
                        'path':         now,
                        }
    
                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)

    except Exception, e:
        # Log error, then report it via stderr.
        log.exception(e)
        sys.stdout = sys.stderr
        raise
예제 #3
0
파일: cluster.py 프로젝트: tiradani/gip
def print_clusters(cp):
    cluster_name = cp_get(cp, 'cluster', 'name', None)
    if not cluster_name:
        cluster_name = cp_get(cp, 'ce', 'hosting_cluster', None)
    if not cluster_name:
        cluster_name = cp_get(cp, 'ce', 'unique_name', None)
    if not cluster_name:
        getClusterName(cp)
        #raise Exception("Could not determine cluster name.")
    #clusterUniqueID = cp_get(cp, 'ce', 'unique_name', cluster_name)
    clusterUniqueID = getClusterID(cp)
    siteUniqueID = cp_get(cp, "site", "unique_name", 'UNKNOWN_SITE')
    extraCEs = cp_get(cp, 'cluster', 'other_ces', [])
    if extraCEs:
        extraCEs = [x.strip() for x in extraCEs.split(',')]

    ces = getCEList(cp, extraCEs)

    glueClusters = ''
    for ce in ces:
        glueClusters += 'GlueForeignKey: GlueCEUniqueID=%s\n' % ce
    bdii = cp_get(cp, 'gip', 'bdii', 'ldap://is.grid.iu.edu:2170')
    info = { \
        'cluster': cluster_name,
        'clusterUniqueID': clusterUniqueID,
        'tmp': cp_get(cp, "osg_dirs", "tmp", cp_get(cp, "osg_dirs", "data", \
             "/tmp")),
        'wn_tmp': cp_get(cp, "osg_dirs", "wn_tmp", "/tmp"),
        'siteUniqueID': siteUniqueID,
        'glueClusters': glueClusters,
        'bdii': bdii,
    }
    template = getTemplate("GlueCluster", "GlueClusterUniqueID")
    printTemplate(template, info)
예제 #4
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only:
            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            osg_grid = cp_get(cp, "osg_dirs", "grid_dir", None)

            if not osg_grid:
                raise RuntimeError('grid_dir ($OSG_GRID) not defined!')

            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {
                    'locationId': 'OSG_GRID',
                    'subClusterId': subClusterId,
                    'clusterId': cluster_id,
                    'locationName': 'OSG_GRID',
                    'version': 1.0,
                    'path': osg_grid,
                }

                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)

    except Exception, e:
        # Log error, then report it via stderr.
        log.error(e)
        sys.stdout = sys.stderr
        raise
예제 #5
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only:
            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            osg_grid = cp_get(cp, "osg_dirs", "grid_dir", None)
    
            if not osg_grid:
                raise RuntimeError('grid_dir ($OSG_GRID) not defined!')
                
            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {'locationId':   'OSG_GRID',
                        'subClusterId': subClusterId,
                        'clusterId':    cluster_id,
                        'locationName': 'OSG_GRID',
                        'version':      1.0,
                        'path':         osg_grid,
                       }
        
                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)
            
    except Exception, e:
        # Log error, then report it via stderr.
        log.error(e)
        sys.stdout = sys.stderr
        raise
예제 #6
0
파일: cluster.py 프로젝트: holzman/gip
def print_clusters(cp):
    cluster_name = cp_get(cp, "cluster", "name", None)
    if not cluster_name:
        cluster_name = cp_get(cp, "ce", "hosting_cluster", None)
    if not cluster_name:
        cluster_name = cp_get(cp, "ce", "unique_name", None)
    if not cluster_name:
        getClusterName(cp)
        # raise Exception("Could not determine cluster name.")
    # clusterUniqueID = cp_get(cp, 'ce', 'unique_name', cluster_name)
    clusterUniqueID = getClusterID(cp)
    siteUniqueID = cp_get(cp, "site", "unique_name", "UNKNOWN_SITE")
    extraCEs = cp_get(cp, "cluster", "other_ces", [])
    if extraCEs:
        extraCEs = [x.strip() for x in extraCEs.split(",")]

    ces = getCEList(cp, extraCEs)

    glueClusters = ""
    for ce in ces:
        glueClusters += "GlueForeignKey: GlueCEUniqueID=%s\n" % ce
    bdii = cp_get(cp, "gip", "bdii", "ldap://is.grid.iu.edu:2170")
    info = {
        "cluster": cluster_name,
        "clusterUniqueID": clusterUniqueID,
        "tmp": cp_get(cp, "osg_dirs", "tmp", cp_get(cp, "osg_dirs", "data", "/tmp")),
        "wn_tmp": cp_get(cp, "osg_dirs", "wn_tmp", "/tmp"),
        "siteUniqueID": siteUniqueID,
        "glueClusters": glueClusters,
        "bdii": bdii,
    }
    template = getTemplate("GlueCluster", "GlueClusterUniqueID")
    printTemplate(template, info)
예제 #7
0
파일: software.py 프로젝트: tiradani/gip
def print_Locations(cp):
    template = getTemplate("GlueCluster", "GlueLocationLocalID")
    cluster_id = getClusterID(cp)
    for subClusterId in getSubClusterIDs(cp):
        for entry in getApplications(cp):
            entry['subClusterId'] = subClusterId
            entry['clusterId'] = cluster_id
            printTemplate(template, entry)
예제 #8
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only and 'VDT_LOCATION' in os.environ:

            # get the VDT version
            vdt_version_cmd = os.path.expandvars(
                "$VDT_LOCATION/vdt/bin/") + 'vdt-version --no-wget'
            vdt_version_out = runCommand(vdt_version_cmd).readlines()

            gip_re = re.compile('Generic Information Provider\s+(.*?)\s*-.*')
            gip_version = 'UNKNOWN'
            for line in vdt_version_out:
                m = gip_re.match(line)
                if m:
                    gip_version = m.groups()[0]
                    break

            gip_version += '; $Revision$'

            # Get the timestamp in the two formats we wanted
            now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())

            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {
                    'locationId': 'GIP_VERSION',
                    'subClusterId': subClusterId,
                    'clusterId': cluster_id,
                    'locationName': 'GIP_VERSION',
                    'version': gip_version,
                    'path': now,
                }

                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)

    except Exception, e:
        # Log error, then report it via stderr.
        log.exception(e)
        sys.stdout = sys.stderr
        raise
예제 #9
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only and 'VDT_LOCATION' in os.environ:
    
            # get the VDT version
            vdt_version_cmd = os.path.expandvars("$VDT_LOCATION/vdt/bin/") + 'vdt-version --no-wget'
            vdt_version_out = runCommand(vdt_version_cmd).readlines()
    
            gip_re = re.compile('Generic Information Provider\s+(.*?)\s*-.*')
            gip_version = 'UNKNOWN'
            for line in vdt_version_out:
                m = gip_re.match(line)
                if m:
                    gip_version = m.groups()[0]
                    break
    
            gip_version += '; $Revision$'
    
            # Get the timestamp in the two formats we wanted
            now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
    
            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {'locationId':   'GIP_VERSION',
                        'subClusterId': subClusterId,
                        'clusterId':    cluster_id,
                        'locationName': 'GIP_VERSION',
                        'version':      gip_version,
                        'path':         now,
                       }
        
                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)
            
    except Exception, e:
        # Log error, then report it via stderr.
        log.exception(e)
        sys.stdout = sys.stderr
        raise
예제 #10
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only and 'VDT_LOCATION' in os.environ: 
    
            # get the VDT version
            vdt_version_cmd = os.path.expandvars("$VDT_LOCATION/vdt/bin/") + 'vdt-version --brief'
            vdt_version = runCommand(vdt_version_cmd).readlines()[0].strip()
            if (vdt_version == ""): vdt_version = "OLD_VDT"
            
            # Get the timestamp in the two formats we wanted
            now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
    
            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {'locationId':   'VDT_VERSION',
                        'subClusterId': subClusterId,
                        'clusterId':    cluster_id,
                        'locationName': 'VDT_VERSION',
                        'version':      vdt_version,
                        'path':         now,
                       }
        
                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)
            
    except Exception, e:
        # Log error, then report it via stderr.
        log.exception(e)
        sys.stdout = sys.stderr
        raise
예제 #11
0
def main():
    try:
        # Load up the site configuration
        cp = config()
        se_only = cp_getBoolean(cp, "gip", "se_only", False)
        if not se_only and 'VDT_LOCATION' in os.environ: 
    
            # get the VDT version
            vdt_version_cmd = os.path.expandvars("$VDT_LOCATION/vdt/bin/") + 'vdt-version --brief'
            vdt_version = runCommand(vdt_version_cmd).readlines()[0].strip()
            if (vdt_version == ""): vdt_version = "OLD_VDT"
            
            # Get the timestamp in the two formats we wanted
            now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
    
            # Load up the template for GlueLocationLocalID
            # To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
            template = getTemplate("GlueCluster", "GlueLocationLocalID")
            cluster_id = getClusterID(cp)
            for subClusterId in getSubClusterIDs(cp):
                # Dictionary of data to fill in for GlueLocationLocalID
                info = {'locationId':   'VDT_VERSION',
                        'subClusterId': subClusterId,
                        'clusterId':    cluster_id,
                        'locationName': 'VDT_VERSION',
                        'version':      vdt_version,
                        'path':         now,
                       }
        
                # Spit out our template, fill it with the appropriate info.
                printTemplate(template, info)
            
    except Exception, e:
        # Log error, then report it via stderr.
        log.exception(e)
        sys.stdout = sys.stderr
        raise
예제 #12
0
def print_CE(cp):
    slurmVersion = getLrmsInfo(cp)
    queueInfo = getQueueInfo(cp)
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    CE = getTemplate("GlueCE", "GlueCEUniqueID")
    try:
        excludeQueues = [i.strip() for i in cp_get(cp, "slurm", \
            "queue_exclude", "").split(',')]
    except:
        excludeQueues = []
    vo_queues = getVoQueues(cp)
    for queue, info in queueInfo.items():
        if queue in excludeQueues:
            continue
        info["lrmsVersion"] = slurmVersion
        info["job_manager"] = "slurm"

        # if no jobs are waiting in the queue, set the number of free slots
        # to (job_slots - running), or the total number of free slots on the cluster,
        # whichever is less.

        info["queue"] = queue
        info["ceName"] = ce_name

        unique_id = buildCEUniqueID(cp, ce_name, 'slurm', queue)
        ceImpl, ceImplVersion = getCEImpl(cp)
        port = getPort(cp)

        info['ceUniqueID'] = unique_id
        if "job_slots" not in info:
            log.error("no job_slots found for %s!" % queue)
        if "priority" not in info:
            info["priority"] = 0
        if "max_running" not in info:
            log.error("no max_running found for %s!" % queue)
        if "max_wall" not in info:
            info["max_wall"] = 1440

        info["free_slots"] = 0
        if info["wait"] == 0:
            freeSlots = info["job_slots"] - info["running"]
            if freeSlots > 0:
                info["free_slots"] = freeSlots

        ert, wrt = responseTimes(cp,
                                 info.get("running", 0),
                                 info.get("wait", 0),
                                 max_job_time=info["max_wall"])

        info['ert'] = ert
        info['wrt'] = wrt
        info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name)
        info['hostName'] = cp_get(cp, ce, 'host_name', ce_name)
        info['ceImpl'] = ceImpl
        info['ceImplVersion'] = ceImplVersion

        contact_string = buildContactString(cp, 'slurm', queue, unique_id, log)

        info['contact_string'] = contact_string
        info['app_dir'] = cp_get(cp, 'osg_dirs', 'app', "/UNKNOWN_APP")
        info['data_dir'] = cp_get(cp, 'osg_dirs', 'data', "/UNKNOWN_DATA")
        info['default_se'] = getDefaultSE(cp)

        if 'max_waiting' not in info:
            info['max_waiting'] = 999999
        if 'max_queuable' in info:
            info['max_total'] = info['max_queuable']
            info['free_slots'] = min(info['free_slots'], info['max_queuable'])
        else:
            info['max_total'] = info['max_waiting'] + info['max_running']
            info['free_slots'] = min(info['free_slots'], info['max_total'])

        # Enforce invariants:
        # max_total <= max_running
        # free_slots <= max_running
        info['max_total'] = min(info['max_total'], info['max_running'])
        info['free_slots'] = min(info['free_slots'], info['max_running'])

        info['assigned'] = info['job_slots']
        # Enforce invariants:
        # assigned <= max_running
        info['assigned'] = min(info['assigned'], info['max_running'])

        info['lrmsType'] = 'slurm'
        info['preemption'] = cp_get(cp, 'slurm', 'preemption', '0')
        acbr = ''
        has_vo = False
        for vo, queue2 in vo_queues:
            if queue == queue2:
                acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo
                has_vo = True
        if not has_vo:
            continue
        info['acbr'] = acbr[:-1]
        info['bdii'] = cp.get('bdii', 'endpoint')
        gramVersion = getGramVersion(cp)

        info['gramVersion'] = gramVersion
        info['port'] = port
        info['waiting'] = info['wait']
        info['referenceSI00'] = gip_cluster.getReferenceSI00(cp)
        info['clusterUniqueID'] = getClusterID(cp)

        extraCapabilities = ''
        if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

        htpcRSL, maxSlots = getHTPCInfo(cp, 'slurm', queue, log)
        info['max_slots'] = maxSlots

        if maxSlots > 1:
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'

        info['extraCapabilities'] = extraCapabilities
        info['htpc'] = htpcRSL

        printTemplate(CE, info)
    return queueInfo
예제 #13
0
파일: sge.py 프로젝트: holzman/gip
def print_CE(cp):
    SGEVersion = getLrmsInfo(cp)
    queueInfo, _ = getQueueInfo(cp)
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    ce_template = getTemplate("GlueCE", "GlueCEUniqueID")
    queueList = getQueueList(cp)

    vo_queues = getVoQueues(cp)

    default_max_waiting = 999999
    for queue in queueInfo.values():
        if 'name' not in queue or queue['name'] not in queueList:
            continue
        if queue['name'] == 'waiting':
            continue

        unique_id = buildCEUniqueID(cp, ce_name, 'sge', queue['name'])

        acbr = ''
        for vo, queue2 in vo_queues:
            if queue['name'] == queue2:
                acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo

        referenceSI00 = gip_cluster.getReferenceSI00(cp)
        contact_string = buildContactString(cp, 'sge', queue['name'], unique_id, log)

        extraCapabilities = ''
	if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
	    extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

	htpcRSL, maxSlots = getHTPCInfo(cp, 'sge', queue, log)
        if maxSlots > 1:
	    extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'
            
        gramVersion = getGramVersion(cp)
        port = getPort(cp)
        ceImpl, ceImplVersion = getCEImpl(cp)

        max_wall = queue["max_wall"]
        if cp.has_option("sge", "max_wall"):
            max_wall = cp_getInt(cp, "sge", "max_wall", 1440)

        info = { \
            "ceUniqueID" : unique_id,
            "ceName" : ce_name,
            "ceImpl" : ceImpl,
            "ceImplVersion" : ceImplVersion,
            "clusterUniqueID" : getClusterID(cp),
            "queue" : queue['name'],
            "priority" : queue['priority'],
            "lrmsType" : 'sge',
            "lrmsVersion" : SGEVersion,
            "job_manager" : "sge",
            "job_slots" : queue["slots_total"],
            "free_slots" : queue["slots_free"],
            "running" : queue["slots_used"],
            "status" : queue['status'],
            "total" : queue['slots_used'] + queue['waiting'],
            "ert" : 3600,
            "wrt" : 3600,
            "hostingCluster" : cp_get(cp, ce, 'hosting_cluster', ce_name),
            "hostName" : cp_get(cp, ce, 'host_name', ce_name),
            "contact_string" : contact_string,
            "app_dir" : cp_get(cp, 'osg_dirs', 'app', "/OSG_APP_UNKNOWN"),
            "data_dir" : cp_get(cp, 'osg_dirs', 'data', "/OSG_DATA_UNKNOWN"),
            "default_se" : getDefaultSE(cp),
            "max_running" : queue["slots_total"],
            "max_wall" : max_wall,
            "max_waiting" : default_max_waiting,
            "max_slots" : maxSlots,
            "max_total" : default_max_waiting + queue["slots_total"],
            "assigned" : queue["slots_used"],
            "preemption" : cp_get(cp, 'sge', 'preemption', '0'),
            "acbr" : acbr[:-1],
            "bdii": cp.get('bdii', 'endpoint'),
            "gramVersion" : gramVersion,
            "port" : port,
            "waiting" : queue['waiting'],
            "referenceSI00": referenceSI00,
            'extraCapabilities' : extraCapabilities,
            "htpc" : htpcRSL
        }
        printTemplate(ce_template, info)
    return queueInfo
예제 #14
0
파일: pbs.py 프로젝트: bbockelm/gip
def print_CE(cp):
    pbsVersion = getLrmsInfo(cp)
    queueInfo = getQueueInfo(cp)
    totalCpu, freeCpu, queueCpus = parseNodes(cp, pbsVersion)
    log.debug("totalCpu, freeCpu, queueCPus: %s %s %s" % (totalCpu, freeCpu, queueCpus))
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    CE = getTemplate("GlueCE", "GlueCEUniqueID")
    try:
        excludeQueues = [i.strip() for i in cp_get(cp, "pbs", \
            "queue_exclude", "").split(',')]
    except:
        excludeQueues = []
    vo_queues = getVoQueues(cp)
    for queue, info in queueInfo.items():
        if queue in excludeQueues:
            continue
        info["lrmsVersion"] = pbsVersion
        info["job_manager"] = "pbs"

        # if no jobs are waiting in the queue, set the number of free slots
        # to (job_slots - running), or the total number of free slots on the cluster,
        # whichever is less.

        info["queue"] = queue
        info["ceName"] = ce_name

        unique_id = buildCEUniqueID(cp, ce_name, 'pbs', queue)
        ceImpl, ceImplVersion = getCEImpl(cp)
	port = getPort(cp)

        info['ceUniqueID'] = unique_id
        if "job_slots" not in info:
            info["job_slots"] = totalCpu
        if "priority" not in info:
            info["priority"] = 0
        if "max_running" not in info:
            info["max_running"] = info["job_slots"]
        if "max_wall" not in info:
            info["max_wall"] = 1440

        
        info["free_slots"] = 0
        if info["wait"] == 0:
            freeSlots = info["job_slots"] - info["running"]
            if freeSlots > 0:
                info["free_slots"] =  min(freeSlots, freeCpu)

        log.debug("queue info: %s %s" % (queue, info))


        ert, wrt = responseTimes(cp, info.get("running", 0),
            info.get("wait", 0), max_job_time=info["max_wall"])

        info["job_slots"] = min(totalCpu, info["job_slots"])
        info['ert'] = ert
        info['wrt'] = wrt
        info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name)
        info['hostName'] = cp_get(cp, ce, 'host_name', ce_name)
        info['ceImpl'] = ceImpl
        info['ceImplVersion'] = ceImplVersion

	contact_string = buildContactString(cp, 'pbs', queue, unique_id, log)

        info['contact_string'] = contact_string
        info['app_dir'] = cp_get(cp, 'osg_dirs', 'app', "/UNKNOWN_APP")
        info['data_dir'] = cp_get(cp, 'osg_dirs', 'data', "/UNKNOWN_DATA")
        info['default_se'] = getDefaultSE(cp)
        if 'max_waiting' not in info:
            info['max_waiting'] = 999999
        if 'max_queuable' in info:
            info['max_total'] = info['max_queuable']
            info['free_slots'] = min(info['free_slots'], info['max_queuable'])
        else:
            info['max_total'] = info['max_waiting'] + info['max_running']
            info['free_slots'] = min(info['free_slots'], info['max_total'])

        # Enforce invariants:
        # max_total <= max_running
        # free_slots <= max_running
        info['max_total'] = min(info['max_total'], info['max_running'])
        info['free_slots'] = min(info['free_slots'], info['max_running'])

        info['assigned'] = info['job_slots']
        # Enforce invariants:
        # assigned <= max_running
        info['assigned'] = min(info['assigned'], info['max_running'])

        info['lrmsType'] = 'pbs'
        info['preemption'] = cp_get(cp, 'pbs', 'preemption', '0')
        acbr = ''
        has_vo = False
        for vo, queue2 in vo_queues:
            if queue == queue2:
                acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo
                has_vo = True
        if not has_vo:
            continue
        info['acbr'] = acbr[:-1]
        info['bdii'] = cp.get('bdii', 'endpoint')
        gramVersion = getGramVersion(cp)

        info['gramVersion'] = gramVersion
        info['port'] = port
        info['waiting'] = info['wait']
        info['referenceSI00'] = gip_cluster.getReferenceSI00(cp)
        info['clusterUniqueID'] = getClusterID(cp)

        extraCapabilities = ''
        if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

        htpcRSL, maxSlots = getHTPCInfo(cp, 'pbs', queue, log)
        info['max_slots'] = maxSlots
        
        if maxSlots > 1:
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'

        info['extraCapabilities'] = extraCapabilities
        info['htpc'] = htpcRSL

        print CE % info
    return queueInfo, totalCpu, freeCpu, queueCpus
예제 #15
0
파일: slurm.py 프로젝트: holzman/gip
def print_CE(cp):
    slurmVersion = getLrmsInfo(cp)
    queueInfo = getQueueInfo(cp)
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    CE = getTemplate("GlueCE", "GlueCEUniqueID")
    try:
        excludeQueues = [i.strip() for i in cp_get(cp, "slurm", "queue_exclude", "").split(",")]
    except:
        excludeQueues = []
    vo_queues = getVoQueues(cp)
    for queue, info in queueInfo.items():
        if queue in excludeQueues:
            continue
        info["lrmsVersion"] = slurmVersion
        info["job_manager"] = "slurm"

        # if no jobs are waiting in the queue, set the number of free slots
        # to (job_slots - running), or the total number of free slots on the cluster,
        # whichever is less.

        info["queue"] = queue
        info["ceName"] = ce_name

        unique_id = buildCEUniqueID(cp, ce_name, "slurm", queue)
        ceImpl, ceImplVersion = getCEImpl(cp)
        port = getPort(cp)

        info["ceUniqueID"] = unique_id
        if "job_slots" not in info:
            log.error("no job_slots found for %s!" % queue)
        if "priority" not in info:
            info["priority"] = 0
        if "max_running" not in info:
            log.error("no max_running found for %s!" % queue)
        if "max_wall" not in info:
            info["max_wall"] = 1440

        info["free_slots"] = 0
        if info["wait"] == 0:
            freeSlots = info["job_slots"] - info["running"]
            if freeSlots > 0:
                info["free_slots"] = freeSlots

        ert, wrt = responseTimes(cp, info.get("running", 0), info.get("wait", 0), max_job_time=info["max_wall"])

        info["ert"] = ert
        info["wrt"] = wrt
        info["hostingCluster"] = cp_get(cp, ce, "hosting_cluster", ce_name)
        info["hostName"] = cp_get(cp, ce, "host_name", ce_name)
        info["ceImpl"] = ceImpl
        info["ceImplVersion"] = ceImplVersion

        contact_string = buildContactString(cp, "slurm", queue, unique_id, log)

        info["contact_string"] = contact_string
        info["app_dir"] = cp_get(cp, "osg_dirs", "app", "/UNKNOWN_APP")
        info["data_dir"] = cp_get(cp, "osg_dirs", "data", "/UNKNOWN_DATA")
        info["default_se"] = getDefaultSE(cp)

        if "max_waiting" not in info:
            info["max_waiting"] = 999999
        if "max_queuable" in info:
            info["max_total"] = info["max_queuable"]
            info["free_slots"] = min(info["free_slots"], info["max_queuable"])
        else:
            info["max_total"] = info["max_waiting"] + info["max_running"]
            info["free_slots"] = min(info["free_slots"], info["max_total"])

        # Enforce invariants:
        # max_total <= max_running
        # free_slots <= max_running
        info["max_total"] = min(info["max_total"], info["max_running"])
        info["free_slots"] = min(info["free_slots"], info["max_running"])

        info["assigned"] = info["job_slots"]
        # Enforce invariants:
        # assigned <= max_running
        info["assigned"] = min(info["assigned"], info["max_running"])

        info["lrmsType"] = "slurm"
        info["preemption"] = cp_get(cp, "slurm", "preemption", "0")
        acbr = ""
        has_vo = False
        for vo, queue2 in vo_queues:
            if queue == queue2:
                acbr += "GlueCEAccessControlBaseRule: VO:%s\n" % vo
                has_vo = True
        if not has_vo:
            continue
        info["acbr"] = acbr[:-1]
        info["bdii"] = cp.get("bdii", "endpoint")
        gramVersion = getGramVersion(cp)

        info["gramVersion"] = gramVersion
        info["port"] = port
        info["waiting"] = info["wait"]
        info["referenceSI00"] = gip_cluster.getReferenceSI00(cp)
        info["clusterUniqueID"] = getClusterID(cp)

        extraCapabilities = ""
        if cp_getBoolean(cp, "site", "glexec_enabled", False):
            extraCapabilities = extraCapabilities + "\n" + "GlueCECapability: glexec"

        htpcRSL, maxSlots = getHTPCInfo(cp, "slurm", queue, log)
        info["max_slots"] = maxSlots

        if maxSlots > 1:
            extraCapabilities = extraCapabilities + "\n" + "GlueCECapability: htpc"

        info["extraCapabilities"] = extraCapabilities
        info["htpc"] = htpcRSL

        printTemplate(CE, info)
    return queueInfo
예제 #16
0
파일: lsf.py 프로젝트: holzman/gip
def print_CE(cp):
    """
    Print out the GlueCE objects for LSF; one GlueCE per grid queue.
    """
    try:
        lsfVersion = getLrmsInfo(cp)
    except:
        lsfVersion = 'Unknown'

    log.debug('Using LSF version %s' % lsfVersion)    
    queueInfo = getQueueInfo(cp)
    try:
        totalCpu, freeCpu, queueCpus = parseNodes(queueInfo, cp)
    except:
        #raise
        totalCpu, freeCpu, queueCpus = 0, 0, {}
    log.debug('Total, Free CPU: (%s, %s)' % (totalCpu, freeCpu))
    ce_name = cp.get(ce, "name")
    CE = getTemplate("GlueCE", "GlueCEUniqueID")
    try:
        excludeQueues = [i.strip() for i in cp.get("lsf", \
            "queue_exclude").split(',')]
    except:
        excludeQueues = []
    vo_queues = getVoQueues(queueInfo, cp)
    for queue, info in queueInfo.items():
        if queue in excludeQueues:
            continue
        log.debug('Processing queue %s' % queue)
        if 'running' not in info:
            info['running'] = 0
        if 'status' not in info:
            # There really should be an unknown status...
            info['status'] = 'Closed'
        if 'total' not in info:
            info['total'] = 0
        info["lrmsVersion"] = lsfVersion
        info["job_manager"] = "lsf"
        if int(info.get("wait", 0)) > 0:
            info["free_slots"] = 0
        else:
            if queue in queueCpus and 'max' in queueCpus[queue] and 'njobs' in queueCpus[queue]:
                info["free_slots"] = queueCpus[queue]['max'] - queueCpus[queue]['njobs']
            else:
                info["free_slots"] = freeCpu
        info["queue"] = queue
        info["ceName"] = ce_name

        unique_id = buildCEUniqueID(cp, ce_name, 'lsf', queue)        
        info['ceUniqueID'] = unique_id
        if "job_slots" not in info:
            if queue in queueCpus and 'max' in queueCpus[queue]:
                log.debug('queue %s, info is %s' % (queue, queueCpus[queue]))
                info['job_slots'] = queueCpus[queue]['max']
            else:
                info["job_slots"] = totalCpu
        if "priority" not in info:
            info["priority"] = 0
        if "max_running" not in info:
            info["max_running"] = info["job_slots"]
        elif not info['max_running'] or info['max_running'] == '-':
            info['max_running'] = 999999

        if cp.has_option("lsf", "max_wall"):
            info["max_wall"] = cp_getInt(cp, "lsf", "max_wall", 1440)
        else:
            if "max_wall" not in info:
                info["max_wall"] = 1440

        info["max_wall"] = int(info["max_wall"]) # glue proscribes ints 
        info["job_slots"] = min(totalCpu, info["job_slots"])

        ert, wrt = responseTimes(cp, info["running"], info["wait"],
            max_job_time=info["max_wall"])

	contact_string = buildContactString(cp, 'lsf', queue, unique_id, log)

        ceImpl, ceImplVersion = getCEImpl(cp)

        info['ert'] = ert
        info['wrt'] = wrt
        info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name)
        info['hostName'] = cp_get(cp, ce, 'host_name', ce_name)
        info['ceImpl'] = ceImpl
        info['ceImplVersion'] = ceImplVersion
        info['contact_string'] = contact_string
        info['app_dir'] = cp.get('osg_dirs', 'app')
        info['data_dir'] = cp.get('osg_dirs', 'data')
        info['default_se'] = getDefaultSE(cp)
        info['max_waiting'] = 999999
        #info['max_total'] = info['max_running']
        info['max_total'] = info['max_waiting'] + info['max_running']
        info['assigned'] = info['job_slots']
        info['lrmsType'] = 'lsf'
        info['preemption'] = str(cp_getInt(cp, 'lsf', 'preemption', '0'))
        acbr = ''
        for vo, queue2 in vo_queues:
            if queue == queue2:
                acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo.lower()
        if not acbr:
            continue
        #print info
        info['acbr'] = acbr[:-1]
        info['bdii'] = cp.get('bdii', 'endpoint')
        gramVersion = getGramVersion(cp)
        port = getPort(cp)
        info['gramVersion'] = gramVersion
        info['port'] = port
        info['waiting'] = info.get('wait', 0)
        info['referenceSI00'] = gip_cluster.getReferenceSI00(cp)
        info['clusterUniqueID'] = getClusterID(cp)

        extraCapabilities = ''
        if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

	htpcRSL, maxSlots = getHTPCInfo(cp, 'lsf', queue, log)
        info['max_slots'] = maxSlots
        info['htpc'] = htpcRSL
        
        if maxSlots > 1:
	    extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'
            
        info['extraCapabilities'] = extraCapabilities

        printTemplate(CE, info)
    return queueInfo, totalCpu, freeCpu, queueCpus
예제 #17
0
            "priority"       : ginfo.get('prio', 0),
            "assigned"       : assigned,
            "max_slots"      : maxSlots,
            "preemption"     : str(int(cp_getBoolean(cp, "condor", \
                "preemption", False))),
            "max_running"    : max_running,
            "max_waiting"    : 99999,
            "max_total"      : 99999,
            "max_wall"       : cp_getInt(cp, "condor", "max_wall", 1440),
            "status"         : status,
            'app_dir'        : cp_get(cp, 'osg_dirs', 'app', '/Unknown'),
            "data_dir"       : cp_get(cp, "osg_dirs", "data", "/Unknown"),
            "default_se"     : getDefaultSE(cp),
            "acbr"           : ginfo['acbr'],
            "referenceSI00"  : referenceSI00,
            "clusterUniqueID": getClusterID(cp),
            "bdii"           : cp_get(cp, "bdii", "endpoint", "Unknown"),
        'extraCapabilities' : extraCapabilities,
        "htpc"           : htpcRSL
        }
        printTemplate(ce_template, info)
    return total_nodes, claimed, unclaimed


def print_VOViewLocal(cp):
    """
    Print the GLUE VOView entity; shows the VO's view of the condor batch
    system.

    Config options used:
        * ce.name.  The human-readable name of the ce.
예제 #18
0
파일: condor.py 프로젝트: holzman/gip
            "priority"       : ginfo.get('prio', 0),
            "assigned"       : assigned,
            "max_slots"      : maxSlots,
            "preemption"     : str(int(cp_getBoolean(cp, "condor", \
                "preemption", False))),
            "max_running"    : max_running,
            "max_waiting"    : 99999,
            "max_total"      : 99999,
            "max_wall"       : cp_getInt(cp, "condor", "max_wall", 1440),
            "status"         : status,
            'app_dir'        : cp_get(cp, 'osg_dirs', 'app', '/Unknown'),
            "data_dir"       : cp_get(cp, "osg_dirs", "data", "/Unknown"),
            "default_se"     : getDefaultSE(cp),
            "acbr"           : ginfo['acbr'],
            "referenceSI00"  : referenceSI00,
            "clusterUniqueID": getClusterID(cp),
            "bdii"           : cp_get(cp, "bdii", "endpoint", "Unknown"),
	    'extraCapabilities' : extraCapabilities,
	    "htpc"           : htpcRSL
        }
        printTemplate(ce_template, info)
    return total_nodes, claimed, unclaimed

def print_VOViewLocal(cp):
    """
    Print the GLUE VOView entity; shows the VO's view of the condor batch
    system.

    Config options used:
        * ce.name.  The human-readable name of the ce.
        * condor.status.  The status of condor; defaults to "Production"
예제 #19
0
파일: lsf.py 프로젝트: tiradani/gip
def print_CE(cp):
    """
    Print out the GlueCE objects for LSF; one GlueCE per grid queue.
    """
    try:
        lsfVersion = getLrmsInfo(cp)
    except:
        lsfVersion = 'Unknown'

    log.debug('Using LSF version %s' % lsfVersion)
    queueInfo = getQueueInfo(cp)
    try:
        totalCpu, freeCpu, queueCpus = parseNodes(queueInfo, cp)
    except:
        #raise
        totalCpu, freeCpu, queueCpus = 0, 0, {}
    log.debug('Total, Free CPU: (%s, %s)' % (totalCpu, freeCpu))
    ce_name = cp.get(ce, "name")
    CE = getTemplate("GlueCE", "GlueCEUniqueID")
    try:
        excludeQueues = [i.strip() for i in cp.get("lsf", \
            "queue_exclude").split(',')]
    except:
        excludeQueues = []
    vo_queues = getVoQueues(queueInfo, cp)
    for queue, info in queueInfo.items():
        if queue in excludeQueues:
            continue
        log.debug('Processing queue %s' % queue)
        if 'running' not in info:
            info['running'] = 0
        if 'status' not in info:
            # There really should be an unknown status...
            info['status'] = 'Closed'
        if 'total' not in info:
            info['total'] = 0
        info["lrmsVersion"] = lsfVersion
        info["job_manager"] = "lsf"
        if int(info.get("wait", 0)) > 0:
            info["free_slots"] = 0
        else:
            if queue in queueCpus and 'max' in queueCpus[
                    queue] and 'njobs' in queueCpus[queue]:
                info["free_slots"] = queueCpus[queue]['max'] - queueCpus[
                    queue]['njobs']
            else:
                info["free_slots"] = freeCpu
        info["queue"] = queue
        info["ceName"] = ce_name

        unique_id = buildCEUniqueID(cp, ce_name, 'lsf', queue)
        info['ceUniqueID'] = unique_id
        if "job_slots" not in info:
            if queue in queueCpus and 'max' in queueCpus[queue]:
                log.debug('queue %s, info is %s' % (queue, queueCpus[queue]))
                info['job_slots'] = queueCpus[queue]['max']
            else:
                info["job_slots"] = totalCpu
        if "priority" not in info:
            info["priority"] = 0
        if "max_running" not in info:
            info["max_running"] = info["job_slots"]
        elif not info['max_running'] or info['max_running'] == '-':
            info['max_running'] = 999999

        if cp.has_option("lsf", "max_wall"):
            info["max_wall"] = cp_getInt(cp, "lsf", "max_wall", 1440)
        else:
            if "max_wall" not in info:
                info["max_wall"] = 1440

        info["max_wall"] = int(info["max_wall"])  # glue proscribes ints
        info["job_slots"] = min(totalCpu, info["job_slots"])

        ert, wrt = responseTimes(cp,
                                 info["running"],
                                 info["wait"],
                                 max_job_time=info["max_wall"])

        contact_string = buildContactString(cp, 'lsf', queue, unique_id, log)

        ceImpl, ceImplVersion = getCEImpl(cp)

        info['ert'] = ert
        info['wrt'] = wrt
        info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name)
        info['hostName'] = cp_get(cp, ce, 'host_name', ce_name)
        info['ceImpl'] = ceImpl
        info['ceImplVersion'] = ceImplVersion
        info['contact_string'] = contact_string
        info['app_dir'] = cp.get('osg_dirs', 'app')
        info['data_dir'] = cp.get('osg_dirs', 'data')
        info['default_se'] = getDefaultSE(cp)
        info['max_waiting'] = 999999
        #info['max_total'] = info['max_running']
        info['max_total'] = info['max_waiting'] + info['max_running']
        info['assigned'] = info['job_slots']
        info['lrmsType'] = 'lsf'
        info['preemption'] = str(cp_getInt(cp, 'lsf', 'preemption', '0'))
        acbr = ''
        for vo, queue2 in vo_queues:
            if queue == queue2:
                acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo.lower()
        if not acbr:
            continue
        #print info
        info['acbr'] = acbr[:-1]
        info['bdii'] = cp.get('bdii', 'endpoint')
        gramVersion = getGramVersion(cp)
        port = getPort(cp)
        info['gramVersion'] = gramVersion
        info['port'] = port
        info['waiting'] = info.get('wait', 0)
        info['referenceSI00'] = gip_cluster.getReferenceSI00(cp)
        info['clusterUniqueID'] = getClusterID(cp)

        extraCapabilities = ''
        if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

        htpcRSL, maxSlots = getHTPCInfo(cp, 'lsf', queue, log)
        info['max_slots'] = maxSlots
        info['htpc'] = htpcRSL

        if maxSlots > 1:
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'

        info['extraCapabilities'] = extraCapabilities

        printTemplate(CE, info)
    return queueInfo, totalCpu, freeCpu, queueCpus
예제 #20
0
파일: sge.py 프로젝트: tiradani/gip
def print_CE(cp):
    SGEVersion = getLrmsInfo(cp)
    queueInfo, _ = getQueueInfo(cp)
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    ce_template = getTemplate("GlueCE", "GlueCEUniqueID")
    queueList = getQueueList(cp)

    vo_queues = getVoQueues(cp)

    default_max_waiting = 999999
    for queue in queueInfo.values():
        if 'name' not in queue or queue['name'] not in queueList:
            continue
        if queue['name'] == 'waiting':
            continue

        unique_id = buildCEUniqueID(cp, ce_name, 'sge', queue['name'])

        acbr = ''
        for vo, queue2 in vo_queues:
            if queue['name'] == queue2:
                acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo

        referenceSI00 = gip_cluster.getReferenceSI00(cp)
        contact_string = buildContactString(cp, 'sge', queue['name'],
                                            unique_id, log)

        extraCapabilities = ''
        if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

        htpcRSL, maxSlots = getHTPCInfo(cp, 'sge', queue, log)
        if maxSlots > 1:
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'

        gramVersion = getGramVersion(cp)
        port = getPort(cp)
        ceImpl, ceImplVersion = getCEImpl(cp)

        max_wall = queue["max_wall"]
        if cp.has_option("sge", "max_wall"):
            max_wall = cp_getInt(cp, "sge", "max_wall", 1440)

        info = { \
            "ceUniqueID" : unique_id,
            "ceName" : ce_name,
            "ceImpl" : ceImpl,
            "ceImplVersion" : ceImplVersion,
            "clusterUniqueID" : getClusterID(cp),
            "queue" : queue['name'],
            "priority" : queue['priority'],
            "lrmsType" : 'sge',
            "lrmsVersion" : SGEVersion,
            "job_manager" : "sge",
            "job_slots" : queue["slots_total"],
            "free_slots" : queue["slots_free"],
            "running" : queue["slots_used"],
            "status" : queue['status'],
            "total" : queue['slots_used'] + queue['waiting'],
            "ert" : 3600,
            "wrt" : 3600,
            "hostingCluster" : cp_get(cp, ce, 'hosting_cluster', ce_name),
            "hostName" : cp_get(cp, ce, 'host_name', ce_name),
            "contact_string" : contact_string,
            "app_dir" : cp_get(cp, 'osg_dirs', 'app', "/OSG_APP_UNKNOWN"),
            "data_dir" : cp_get(cp, 'osg_dirs', 'data', "/OSG_DATA_UNKNOWN"),
            "default_se" : getDefaultSE(cp),
            "max_running" : queue["slots_total"],
            "max_wall" : max_wall,
            "max_waiting" : default_max_waiting,
            "max_slots" : maxSlots,
            "max_total" : default_max_waiting + queue["slots_total"],
            "assigned" : queue["slots_used"],
            "preemption" : cp_get(cp, 'sge', 'preemption', '0'),
            "acbr" : acbr[:-1],
            "bdii": cp.get('bdii', 'endpoint'),
            "gramVersion" : gramVersion,
            "port" : port,
            "waiting" : queue['waiting'],
            "referenceSI00": referenceSI00,
            'extraCapabilities' : extraCapabilities,
            "htpc" : htpcRSL
        }
        printTemplate(ce_template, info)
    return queueInfo