Example #1
0
File: pbs.py Project: tiradani/gip
def print_VOViewLocal(queue_info, cp):
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    vo_map = VoMapper(cp)
    queue_jobs = getJobsInfo(vo_map, cp)
    VOView = getTemplate("GlueCE", "GlueVOViewLocalID")
    vo_queues = getVoQueues(cp)
    for vo, queue in vo_queues:
        vo_info = queue_jobs.get(queue, {})
        info2 = vo_info.get(vo, {})

	port = getPort(cp)
        ce_unique_id = buildCEUniqueID(cp, ce_name, 'pbs', queue)
        
        my_queue_info = queue_info.setdefault(queue, {})

        max_job_time = my_queue_info.get("max_wall", 0)
        if cp.has_option("pbs", "max_wall"):
            max_job_time = cp_getInt(cp, "pbs", "max_wall", 1440)

        ert, wrt = responseTimes(cp, info2.get("running", 0),
            info2.get("wait", 0),
            max_job_time)


        free_slots = my_queue_info.get('free_slots', 0)
        waiting = info2.get('wait', 0)
        if waiting > cp_getInt(cp, 'pbs', 'idle_slack', '10'):
            free_slots = 0

        info = {
            'ceUniqueID'  : ce_unique_id,
            'job_slots'   : my_queue_info.get('job_slots', 0),
            'free_slots'  : free_slots,
            'ce_name'     : ce_name,
            'queue'       : queue,
            'vo'          : vo,
            'voLocalID'   : vo,
            'job_manager' : 'pbs',
            'running'     : info2.get('running', 0),
            'max_running' : info2.get('max_running', 0),
            'priority'    : queue_info.get(queue, {}).get('priority', 0),
            'waiting'     : waiting,
            'data'        : cp_get(cp, "osg_dirs", "data", "UNKNOWN_DATA"),
            'app'         : cp_get(cp, "osg_dirs", "app", "UNKNOWN_APP"),
            'default_se'  : getDefaultSE(cp),
            'ert'         : 3600,
            'wrt'         : 3600,
            'acbr'        : 'VO:%s' % vo
        }
        info['total'] = info['waiting'] + info['running']
        printTemplate(VOView, info)
Example #2
0
File: pbs.py Project: holzman/gip
def print_VOViewLocal(queue_info, cp):
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    vo_map = VoMapper(cp)
    queue_jobs = getJobsInfo(vo_map, cp)
    VOView = getTemplate("GlueCE", "GlueVOViewLocalID")
    vo_queues = getVoQueues(cp)
    for vo, queue in vo_queues:
        vo_info = queue_jobs.get(queue, {})
        info2 = vo_info.get(vo, {})

        port = getPort(cp)
        ce_unique_id = buildCEUniqueID(cp, ce_name, "pbs", queue)

        my_queue_info = queue_info.setdefault(queue, {})

        max_job_time = my_queue_info.get("max_wall", 0)
        if cp.has_option("pbs", "max_wall"):
            max_job_time = cp_getInt(cp, "pbs", "max_wall", 1440)

        ert, wrt = responseTimes(cp, info2.get("running", 0), info2.get("wait", 0), max_job_time)

        free_slots = my_queue_info.get("free_slots", 0)
        waiting = info2.get("wait", 0)
        if waiting > cp_getInt(cp, "pbs", "idle_slack", "10"):
            free_slots = 0

        info = {
            "ceUniqueID": ce_unique_id,
            "job_slots": my_queue_info.get("job_slots", 0),
            "free_slots": free_slots,
            "ce_name": ce_name,
            "queue": queue,
            "vo": vo,
            "voLocalID": vo,
            "job_manager": "pbs",
            "running": info2.get("running", 0),
            "max_running": info2.get("max_running", 0),
            "priority": queue_info.get(queue, {}).get("priority", 0),
            "waiting": waiting,
            "data": cp_get(cp, "osg_dirs", "data", "UNKNOWN_DATA"),
            "app": cp_get(cp, "osg_dirs", "app", "UNKNOWN_APP"),
            "default_se": getDefaultSE(cp),
            "ert": 3600,
            "wrt": 3600,
            "acbr": "VO:%s" % vo,
        }
        info["total"] = info["waiting"] + info["running"]
        printTemplate(VOView, info)
Example #3
0
File: pbs.py Project: bbockelm/gip
def print_VOViewLocal(queue_info, cp):
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    vo_map = VoMapper(cp)
    queue_jobs = getJobsInfo(vo_map, cp)
    VOView = getTemplate("GlueCE", "GlueVOViewLocalID")
    vo_queues = getVoQueues(cp)
    for vo, queue in vo_queues:
        vo_info = queue_jobs.get(queue, {})
        info2 = vo_info.get(vo, {})

	port = getPort(cp)
        ce_unique_id = buildCEUniqueID(cp, ce_name, 'pbs', queue)
        
        my_queue_info = queue_info.setdefault(queue, {})
        ert, wrt = responseTimes(cp, info2.get("running", 0),
            info2.get("wait", 0),
            max_job_time=my_queue_info.get("max_wall", 0))

        free_slots = my_queue_info.get('free_slots', 0)
        waiting = info2.get('wait', 0)
        if waiting > 0:
            free_slots = 0

        info = {
            'ceUniqueID'  : ce_unique_id,
            'job_slots'   : my_queue_info.get('job_slots', 0),
            'free_slots'  : free_slots,
            'ce_name'     : ce_name,
            'queue'       : queue,
            'vo'          : vo,
            'voLocalID'   : vo,
            'job_manager' : 'pbs',
            'running'     : info2.get('running', 0),
            'max_running' : info2.get('max_running', 0),
            'priority'    : queue_info.get(queue, {}).get('priority', 0),
            'waiting'     : waiting,
            'data'        : cp_get(cp, "osg_dirs", "data", "UNKNOWN_DATA"),
            'app'         : cp_get(cp, "osg_dirs", "app", "UNKNOWN_APP"),
            'default_se'  : getDefaultSE(cp),
            'ert'         : 3600,
            'wrt'         : 3600,
            'acbr'        : 'VO:%s' % vo
        }
        info['total'] = info['waiting'] + info['running']
        printTemplate(VOView, info)
Example #4
0
def print_CE(cp):
    slurmVersion = getLrmsInfo(cp)
    queueInfo = getQueueInfo(cp)
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    CE = getTemplate("GlueCE", "GlueCEUniqueID")
    try:
        excludeQueues = [i.strip() for i in cp_get(cp, "slurm", \
            "queue_exclude", "").split(',')]
    except:
        excludeQueues = []
    vo_queues = getVoQueues(cp)
    for queue, info in queueInfo.items():
        if queue in excludeQueues:
            continue
        info["lrmsVersion"] = slurmVersion
        info["job_manager"] = "slurm"

        # if no jobs are waiting in the queue, set the number of free slots
        # to (job_slots - running), or the total number of free slots on the cluster,
        # whichever is less.

        info["queue"] = queue
        info["ceName"] = ce_name

        unique_id = buildCEUniqueID(cp, ce_name, 'slurm', queue)
        ceImpl, ceImplVersion = getCEImpl(cp)
        port = getPort(cp)

        info['ceUniqueID'] = unique_id
        if "job_slots" not in info:
            log.error("no job_slots found for %s!" % queue)
        if "priority" not in info:
            info["priority"] = 0
        if "max_running" not in info:
            log.error("no max_running found for %s!" % queue)
        if "max_wall" not in info:
            info["max_wall"] = 1440

        info["free_slots"] = 0
        if info["wait"] == 0:
            freeSlots = info["job_slots"] - info["running"]
            if freeSlots > 0:
                info["free_slots"] = freeSlots

        ert, wrt = responseTimes(cp,
                                 info.get("running", 0),
                                 info.get("wait", 0),
                                 max_job_time=info["max_wall"])

        info['ert'] = ert
        info['wrt'] = wrt
        info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name)
        info['hostName'] = cp_get(cp, ce, 'host_name', ce_name)
        info['ceImpl'] = ceImpl
        info['ceImplVersion'] = ceImplVersion

        contact_string = buildContactString(cp, 'slurm', queue, unique_id, log)

        info['contact_string'] = contact_string
        info['app_dir'] = cp_get(cp, 'osg_dirs', 'app', "/UNKNOWN_APP")
        info['data_dir'] = cp_get(cp, 'osg_dirs', 'data', "/UNKNOWN_DATA")
        info['default_se'] = getDefaultSE(cp)

        if 'max_waiting' not in info:
            info['max_waiting'] = 999999
        if 'max_queuable' in info:
            info['max_total'] = info['max_queuable']
            info['free_slots'] = min(info['free_slots'], info['max_queuable'])
        else:
            info['max_total'] = info['max_waiting'] + info['max_running']
            info['free_slots'] = min(info['free_slots'], info['max_total'])

        # Enforce invariants:
        # max_total <= max_running
        # free_slots <= max_running
        info['max_total'] = min(info['max_total'], info['max_running'])
        info['free_slots'] = min(info['free_slots'], info['max_running'])

        info['assigned'] = info['job_slots']
        # Enforce invariants:
        # assigned <= max_running
        info['assigned'] = min(info['assigned'], info['max_running'])

        info['lrmsType'] = 'slurm'
        info['preemption'] = cp_get(cp, 'slurm', 'preemption', '0')
        acbr = ''
        has_vo = False
        for vo, queue2 in vo_queues:
            if queue == queue2:
                acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo
                has_vo = True
        if not has_vo:
            continue
        info['acbr'] = acbr[:-1]
        info['bdii'] = cp.get('bdii', 'endpoint')
        gramVersion = getGramVersion(cp)

        info['gramVersion'] = gramVersion
        info['port'] = port
        info['waiting'] = info['wait']
        info['referenceSI00'] = gip_cluster.getReferenceSI00(cp)
        info['clusterUniqueID'] = getClusterID(cp)

        extraCapabilities = ''
        if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

        htpcRSL, maxSlots = getHTPCInfo(cp, 'slurm', queue, log)
        info['max_slots'] = maxSlots

        if maxSlots > 1:
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'

        info['extraCapabilities'] = extraCapabilities
        info['htpc'] = htpcRSL

        printTemplate(CE, info)
    return queueInfo
Example #5
0
File: sge.py Project: holzman/gip
def print_CE(cp):
    SGEVersion = getLrmsInfo(cp)
    queueInfo, _ = getQueueInfo(cp)
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    ce_template = getTemplate("GlueCE", "GlueCEUniqueID")
    queueList = getQueueList(cp)

    vo_queues = getVoQueues(cp)

    default_max_waiting = 999999
    for queue in queueInfo.values():
        if 'name' not in queue or queue['name'] not in queueList:
            continue
        if queue['name'] == 'waiting':
            continue

        unique_id = buildCEUniqueID(cp, ce_name, 'sge', queue['name'])

        acbr = ''
        for vo, queue2 in vo_queues:
            if queue['name'] == queue2:
                acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo

        referenceSI00 = gip_cluster.getReferenceSI00(cp)
        contact_string = buildContactString(cp, 'sge', queue['name'], unique_id, log)

        extraCapabilities = ''
	if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
	    extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

	htpcRSL, maxSlots = getHTPCInfo(cp, 'sge', queue, log)
        if maxSlots > 1:
	    extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'
            
        gramVersion = getGramVersion(cp)
        port = getPort(cp)
        ceImpl, ceImplVersion = getCEImpl(cp)

        max_wall = queue["max_wall"]
        if cp.has_option("sge", "max_wall"):
            max_wall = cp_getInt(cp, "sge", "max_wall", 1440)

        info = { \
            "ceUniqueID" : unique_id,
            "ceName" : ce_name,
            "ceImpl" : ceImpl,
            "ceImplVersion" : ceImplVersion,
            "clusterUniqueID" : getClusterID(cp),
            "queue" : queue['name'],
            "priority" : queue['priority'],
            "lrmsType" : 'sge',
            "lrmsVersion" : SGEVersion,
            "job_manager" : "sge",
            "job_slots" : queue["slots_total"],
            "free_slots" : queue["slots_free"],
            "running" : queue["slots_used"],
            "status" : queue['status'],
            "total" : queue['slots_used'] + queue['waiting'],
            "ert" : 3600,
            "wrt" : 3600,
            "hostingCluster" : cp_get(cp, ce, 'hosting_cluster', ce_name),
            "hostName" : cp_get(cp, ce, 'host_name', ce_name),
            "contact_string" : contact_string,
            "app_dir" : cp_get(cp, 'osg_dirs', 'app', "/OSG_APP_UNKNOWN"),
            "data_dir" : cp_get(cp, 'osg_dirs', 'data', "/OSG_DATA_UNKNOWN"),
            "default_se" : getDefaultSE(cp),
            "max_running" : queue["slots_total"],
            "max_wall" : max_wall,
            "max_waiting" : default_max_waiting,
            "max_slots" : maxSlots,
            "max_total" : default_max_waiting + queue["slots_total"],
            "assigned" : queue["slots_used"],
            "preemption" : cp_get(cp, 'sge', 'preemption', '0'),
            "acbr" : acbr[:-1],
            "bdii": cp.get('bdii', 'endpoint'),
            "gramVersion" : gramVersion,
            "port" : port,
            "waiting" : queue['waiting'],
            "referenceSI00": referenceSI00,
            'extraCapabilities' : extraCapabilities,
            "htpc" : htpcRSL
        }
        printTemplate(ce_template, info)
    return queueInfo
Example #6
0
File: slurm.py Project: holzman/gip
def print_CE(cp):
    slurmVersion = getLrmsInfo(cp)
    queueInfo = getQueueInfo(cp)
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    CE = getTemplate("GlueCE", "GlueCEUniqueID")
    try:
        excludeQueues = [i.strip() for i in cp_get(cp, "slurm", "queue_exclude", "").split(",")]
    except:
        excludeQueues = []
    vo_queues = getVoQueues(cp)
    for queue, info in queueInfo.items():
        if queue in excludeQueues:
            continue
        info["lrmsVersion"] = slurmVersion
        info["job_manager"] = "slurm"

        # if no jobs are waiting in the queue, set the number of free slots
        # to (job_slots - running), or the total number of free slots on the cluster,
        # whichever is less.

        info["queue"] = queue
        info["ceName"] = ce_name

        unique_id = buildCEUniqueID(cp, ce_name, "slurm", queue)
        ceImpl, ceImplVersion = getCEImpl(cp)
        port = getPort(cp)

        info["ceUniqueID"] = unique_id
        if "job_slots" not in info:
            log.error("no job_slots found for %s!" % queue)
        if "priority" not in info:
            info["priority"] = 0
        if "max_running" not in info:
            log.error("no max_running found for %s!" % queue)
        if "max_wall" not in info:
            info["max_wall"] = 1440

        info["free_slots"] = 0
        if info["wait"] == 0:
            freeSlots = info["job_slots"] - info["running"]
            if freeSlots > 0:
                info["free_slots"] = freeSlots

        ert, wrt = responseTimes(cp, info.get("running", 0), info.get("wait", 0), max_job_time=info["max_wall"])

        info["ert"] = ert
        info["wrt"] = wrt
        info["hostingCluster"] = cp_get(cp, ce, "hosting_cluster", ce_name)
        info["hostName"] = cp_get(cp, ce, "host_name", ce_name)
        info["ceImpl"] = ceImpl
        info["ceImplVersion"] = ceImplVersion

        contact_string = buildContactString(cp, "slurm", queue, unique_id, log)

        info["contact_string"] = contact_string
        info["app_dir"] = cp_get(cp, "osg_dirs", "app", "/UNKNOWN_APP")
        info["data_dir"] = cp_get(cp, "osg_dirs", "data", "/UNKNOWN_DATA")
        info["default_se"] = getDefaultSE(cp)

        if "max_waiting" not in info:
            info["max_waiting"] = 999999
        if "max_queuable" in info:
            info["max_total"] = info["max_queuable"]
            info["free_slots"] = min(info["free_slots"], info["max_queuable"])
        else:
            info["max_total"] = info["max_waiting"] + info["max_running"]
            info["free_slots"] = min(info["free_slots"], info["max_total"])

        # Enforce invariants:
        # max_total <= max_running
        # free_slots <= max_running
        info["max_total"] = min(info["max_total"], info["max_running"])
        info["free_slots"] = min(info["free_slots"], info["max_running"])

        info["assigned"] = info["job_slots"]
        # Enforce invariants:
        # assigned <= max_running
        info["assigned"] = min(info["assigned"], info["max_running"])

        info["lrmsType"] = "slurm"
        info["preemption"] = cp_get(cp, "slurm", "preemption", "0")
        acbr = ""
        has_vo = False
        for vo, queue2 in vo_queues:
            if queue == queue2:
                acbr += "GlueCEAccessControlBaseRule: VO:%s\n" % vo
                has_vo = True
        if not has_vo:
            continue
        info["acbr"] = acbr[:-1]
        info["bdii"] = cp.get("bdii", "endpoint")
        gramVersion = getGramVersion(cp)

        info["gramVersion"] = gramVersion
        info["port"] = port
        info["waiting"] = info["wait"]
        info["referenceSI00"] = gip_cluster.getReferenceSI00(cp)
        info["clusterUniqueID"] = getClusterID(cp)

        extraCapabilities = ""
        if cp_getBoolean(cp, "site", "glexec_enabled", False):
            extraCapabilities = extraCapabilities + "\n" + "GlueCECapability: glexec"

        htpcRSL, maxSlots = getHTPCInfo(cp, "slurm", queue, log)
        info["max_slots"] = maxSlots

        if maxSlots > 1:
            extraCapabilities = extraCapabilities + "\n" + "GlueCECapability: htpc"

        info["extraCapabilities"] = extraCapabilities
        info["htpc"] = htpcRSL

        printTemplate(CE, info)
    return queueInfo
Example #7
0
File: lsf.py Project: holzman/gip
def print_CE(cp):
    """
    Print out the GlueCE objects for LSF; one GlueCE per grid queue.
    """
    try:
        lsfVersion = getLrmsInfo(cp)
    except:
        lsfVersion = 'Unknown'

    log.debug('Using LSF version %s' % lsfVersion)    
    queueInfo = getQueueInfo(cp)
    try:
        totalCpu, freeCpu, queueCpus = parseNodes(queueInfo, cp)
    except:
        #raise
        totalCpu, freeCpu, queueCpus = 0, 0, {}
    log.debug('Total, Free CPU: (%s, %s)' % (totalCpu, freeCpu))
    ce_name = cp.get(ce, "name")
    CE = getTemplate("GlueCE", "GlueCEUniqueID")
    try:
        excludeQueues = [i.strip() for i in cp.get("lsf", \
            "queue_exclude").split(',')]
    except:
        excludeQueues = []
    vo_queues = getVoQueues(queueInfo, cp)
    for queue, info in queueInfo.items():
        if queue in excludeQueues:
            continue
        log.debug('Processing queue %s' % queue)
        if 'running' not in info:
            info['running'] = 0
        if 'status' not in info:
            # There really should be an unknown status...
            info['status'] = 'Closed'
        if 'total' not in info:
            info['total'] = 0
        info["lrmsVersion"] = lsfVersion
        info["job_manager"] = "lsf"
        if int(info.get("wait", 0)) > 0:
            info["free_slots"] = 0
        else:
            if queue in queueCpus and 'max' in queueCpus[queue] and 'njobs' in queueCpus[queue]:
                info["free_slots"] = queueCpus[queue]['max'] - queueCpus[queue]['njobs']
            else:
                info["free_slots"] = freeCpu
        info["queue"] = queue
        info["ceName"] = ce_name

        unique_id = buildCEUniqueID(cp, ce_name, 'lsf', queue)        
        info['ceUniqueID'] = unique_id
        if "job_slots" not in info:
            if queue in queueCpus and 'max' in queueCpus[queue]:
                log.debug('queue %s, info is %s' % (queue, queueCpus[queue]))
                info['job_slots'] = queueCpus[queue]['max']
            else:
                info["job_slots"] = totalCpu
        if "priority" not in info:
            info["priority"] = 0
        if "max_running" not in info:
            info["max_running"] = info["job_slots"]
        elif not info['max_running'] or info['max_running'] == '-':
            info['max_running'] = 999999

        if cp.has_option("lsf", "max_wall"):
            info["max_wall"] = cp_getInt(cp, "lsf", "max_wall", 1440)
        else:
            if "max_wall" not in info:
                info["max_wall"] = 1440

        info["max_wall"] = int(info["max_wall"]) # glue proscribes ints 
        info["job_slots"] = min(totalCpu, info["job_slots"])

        ert, wrt = responseTimes(cp, info["running"], info["wait"],
            max_job_time=info["max_wall"])

	contact_string = buildContactString(cp, 'lsf', queue, unique_id, log)

        ceImpl, ceImplVersion = getCEImpl(cp)

        info['ert'] = ert
        info['wrt'] = wrt
        info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name)
        info['hostName'] = cp_get(cp, ce, 'host_name', ce_name)
        info['ceImpl'] = ceImpl
        info['ceImplVersion'] = ceImplVersion
        info['contact_string'] = contact_string
        info['app_dir'] = cp.get('osg_dirs', 'app')
        info['data_dir'] = cp.get('osg_dirs', 'data')
        info['default_se'] = getDefaultSE(cp)
        info['max_waiting'] = 999999
        #info['max_total'] = info['max_running']
        info['max_total'] = info['max_waiting'] + info['max_running']
        info['assigned'] = info['job_slots']
        info['lrmsType'] = 'lsf'
        info['preemption'] = str(cp_getInt(cp, 'lsf', 'preemption', '0'))
        acbr = ''
        for vo, queue2 in vo_queues:
            if queue == queue2:
                acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo.lower()
        if not acbr:
            continue
        #print info
        info['acbr'] = acbr[:-1]
        info['bdii'] = cp.get('bdii', 'endpoint')
        gramVersion = getGramVersion(cp)
        port = getPort(cp)
        info['gramVersion'] = gramVersion
        info['port'] = port
        info['waiting'] = info.get('wait', 0)
        info['referenceSI00'] = gip_cluster.getReferenceSI00(cp)
        info['clusterUniqueID'] = getClusterID(cp)

        extraCapabilities = ''
        if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

	htpcRSL, maxSlots = getHTPCInfo(cp, 'lsf', queue, log)
        info['max_slots'] = maxSlots
        info['htpc'] = htpcRSL
        
        if maxSlots > 1:
	    extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'
            
        info['extraCapabilities'] = extraCapabilities

        printTemplate(CE, info)
    return queueInfo, totalCpu, freeCpu, queueCpus
Example #8
0
        referenceSI00 = gip_cluster.getReferenceSI00(cp)

        contact_string = buildContactString(cp, 'condor', group, ce_unique_id,
                                            log)
        htpcRSL, maxSlots = getHTPCInfo(cp, 'condor', group, log)

        extraCapabilities = ''
        if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

        if maxSlots > 1:
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'

        gramVersion = getGramVersion(cp)
        ceImpl, ceImplVersion = getCEImpl(cp)
        port = getPort(cp)

        # Build all the GLUE CE entity information.
        info = { \
            "ceUniqueID"     : ce_unique_id,
            'contact_string' : contact_string,
            "ceImpl"         : ceImpl,
            "ceImplVersion"  : ceImplVersion,
            "hostingCluster" : cp_get(cp, ce, 'hosting_cluster', ce_name),
            "hostName"       : cp_get(cp, ce, "host_name", ce_name),
            "gramVersion"    : gramVersion,
            "lrmsType"       : "condor",
            "port"           : port,
            "running"        : myrunning,
            "idle"           : myidle,
            "held"           : myheld,
Example #9
0
        referenceSI00 = gip_cluster.getReferenceSI00(cp)

	contact_string = buildContactString(cp, 'condor', group, ce_unique_id, log)
	htpcRSL, maxSlots = getHTPCInfo(cp, 'condor', group, log)
		
        extraCapabilities = ''
	if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
	    extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

	if maxSlots > 1:
	    extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'
		
	gramVersion = getGramVersion(cp)
	ceImpl, ceImplVersion = getCEImpl(cp)
	port = getPort(cp)

        # Build all the GLUE CE entity information.
        info = { \
            "ceUniqueID"     : ce_unique_id,
            'contact_string' : contact_string,
            "ceImpl"         : ceImpl,
            "ceImplVersion"  : ceImplVersion,
            "hostingCluster" : cp_get(cp, ce, 'hosting_cluster', ce_name),
            "hostName"       : cp_get(cp, ce, "host_name", ce_name),
            "gramVersion"    : gramVersion,
            "lrmsType"       : "condor",
            "port"           : port,
            "running"        : myrunning,
            "idle"           : myidle,
            "held"           : myheld,
Example #10
0
File: sge.py Project: tiradani/gip
def print_CE(cp):
    SGEVersion = getLrmsInfo(cp)
    queueInfo, _ = getQueueInfo(cp)
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    ce_template = getTemplate("GlueCE", "GlueCEUniqueID")
    queueList = getQueueList(cp)

    vo_queues = getVoQueues(cp)

    default_max_waiting = 999999
    for queue in queueInfo.values():
        if 'name' not in queue or queue['name'] not in queueList:
            continue
        if queue['name'] == 'waiting':
            continue

        unique_id = buildCEUniqueID(cp, ce_name, 'sge', queue['name'])

        acbr = ''
        for vo, queue2 in vo_queues:
            if queue['name'] == queue2:
                acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo

        referenceSI00 = gip_cluster.getReferenceSI00(cp)
        contact_string = buildContactString(cp, 'sge', queue['name'],
                                            unique_id, log)

        extraCapabilities = ''
        if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

        htpcRSL, maxSlots = getHTPCInfo(cp, 'sge', queue, log)
        if maxSlots > 1:
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'

        gramVersion = getGramVersion(cp)
        port = getPort(cp)
        ceImpl, ceImplVersion = getCEImpl(cp)

        max_wall = queue["max_wall"]
        if cp.has_option("sge", "max_wall"):
            max_wall = cp_getInt(cp, "sge", "max_wall", 1440)

        info = { \
            "ceUniqueID" : unique_id,
            "ceName" : ce_name,
            "ceImpl" : ceImpl,
            "ceImplVersion" : ceImplVersion,
            "clusterUniqueID" : getClusterID(cp),
            "queue" : queue['name'],
            "priority" : queue['priority'],
            "lrmsType" : 'sge',
            "lrmsVersion" : SGEVersion,
            "job_manager" : "sge",
            "job_slots" : queue["slots_total"],
            "free_slots" : queue["slots_free"],
            "running" : queue["slots_used"],
            "status" : queue['status'],
            "total" : queue['slots_used'] + queue['waiting'],
            "ert" : 3600,
            "wrt" : 3600,
            "hostingCluster" : cp_get(cp, ce, 'hosting_cluster', ce_name),
            "hostName" : cp_get(cp, ce, 'host_name', ce_name),
            "contact_string" : contact_string,
            "app_dir" : cp_get(cp, 'osg_dirs', 'app', "/OSG_APP_UNKNOWN"),
            "data_dir" : cp_get(cp, 'osg_dirs', 'data', "/OSG_DATA_UNKNOWN"),
            "default_se" : getDefaultSE(cp),
            "max_running" : queue["slots_total"],
            "max_wall" : max_wall,
            "max_waiting" : default_max_waiting,
            "max_slots" : maxSlots,
            "max_total" : default_max_waiting + queue["slots_total"],
            "assigned" : queue["slots_used"],
            "preemption" : cp_get(cp, 'sge', 'preemption', '0'),
            "acbr" : acbr[:-1],
            "bdii": cp.get('bdii', 'endpoint'),
            "gramVersion" : gramVersion,
            "port" : port,
            "waiting" : queue['waiting'],
            "referenceSI00": referenceSI00,
            'extraCapabilities' : extraCapabilities,
            "htpc" : htpcRSL
        }
        printTemplate(ce_template, info)
    return queueInfo
Example #11
0
File: lsf.py Project: tiradani/gip
def print_CE(cp):
    """
    Print out the GlueCE objects for LSF; one GlueCE per grid queue.
    """
    try:
        lsfVersion = getLrmsInfo(cp)
    except:
        lsfVersion = 'Unknown'

    log.debug('Using LSF version %s' % lsfVersion)
    queueInfo = getQueueInfo(cp)
    try:
        totalCpu, freeCpu, queueCpus = parseNodes(queueInfo, cp)
    except:
        #raise
        totalCpu, freeCpu, queueCpus = 0, 0, {}
    log.debug('Total, Free CPU: (%s, %s)' % (totalCpu, freeCpu))
    ce_name = cp.get(ce, "name")
    CE = getTemplate("GlueCE", "GlueCEUniqueID")
    try:
        excludeQueues = [i.strip() for i in cp.get("lsf", \
            "queue_exclude").split(',')]
    except:
        excludeQueues = []
    vo_queues = getVoQueues(queueInfo, cp)
    for queue, info in queueInfo.items():
        if queue in excludeQueues:
            continue
        log.debug('Processing queue %s' % queue)
        if 'running' not in info:
            info['running'] = 0
        if 'status' not in info:
            # There really should be an unknown status...
            info['status'] = 'Closed'
        if 'total' not in info:
            info['total'] = 0
        info["lrmsVersion"] = lsfVersion
        info["job_manager"] = "lsf"
        if int(info.get("wait", 0)) > 0:
            info["free_slots"] = 0
        else:
            if queue in queueCpus and 'max' in queueCpus[
                    queue] and 'njobs' in queueCpus[queue]:
                info["free_slots"] = queueCpus[queue]['max'] - queueCpus[
                    queue]['njobs']
            else:
                info["free_slots"] = freeCpu
        info["queue"] = queue
        info["ceName"] = ce_name

        unique_id = buildCEUniqueID(cp, ce_name, 'lsf', queue)
        info['ceUniqueID'] = unique_id
        if "job_slots" not in info:
            if queue in queueCpus and 'max' in queueCpus[queue]:
                log.debug('queue %s, info is %s' % (queue, queueCpus[queue]))
                info['job_slots'] = queueCpus[queue]['max']
            else:
                info["job_slots"] = totalCpu
        if "priority" not in info:
            info["priority"] = 0
        if "max_running" not in info:
            info["max_running"] = info["job_slots"]
        elif not info['max_running'] or info['max_running'] == '-':
            info['max_running'] = 999999

        if cp.has_option("lsf", "max_wall"):
            info["max_wall"] = cp_getInt(cp, "lsf", "max_wall", 1440)
        else:
            if "max_wall" not in info:
                info["max_wall"] = 1440

        info["max_wall"] = int(info["max_wall"])  # glue proscribes ints
        info["job_slots"] = min(totalCpu, info["job_slots"])

        ert, wrt = responseTimes(cp,
                                 info["running"],
                                 info["wait"],
                                 max_job_time=info["max_wall"])

        contact_string = buildContactString(cp, 'lsf', queue, unique_id, log)

        ceImpl, ceImplVersion = getCEImpl(cp)

        info['ert'] = ert
        info['wrt'] = wrt
        info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name)
        info['hostName'] = cp_get(cp, ce, 'host_name', ce_name)
        info['ceImpl'] = ceImpl
        info['ceImplVersion'] = ceImplVersion
        info['contact_string'] = contact_string
        info['app_dir'] = cp.get('osg_dirs', 'app')
        info['data_dir'] = cp.get('osg_dirs', 'data')
        info['default_se'] = getDefaultSE(cp)
        info['max_waiting'] = 999999
        #info['max_total'] = info['max_running']
        info['max_total'] = info['max_waiting'] + info['max_running']
        info['assigned'] = info['job_slots']
        info['lrmsType'] = 'lsf'
        info['preemption'] = str(cp_getInt(cp, 'lsf', 'preemption', '0'))
        acbr = ''
        for vo, queue2 in vo_queues:
            if queue == queue2:
                acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo.lower()
        if not acbr:
            continue
        #print info
        info['acbr'] = acbr[:-1]
        info['bdii'] = cp.get('bdii', 'endpoint')
        gramVersion = getGramVersion(cp)
        port = getPort(cp)
        info['gramVersion'] = gramVersion
        info['port'] = port
        info['waiting'] = info.get('wait', 0)
        info['referenceSI00'] = gip_cluster.getReferenceSI00(cp)
        info['clusterUniqueID'] = getClusterID(cp)

        extraCapabilities = ''
        if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

        htpcRSL, maxSlots = getHTPCInfo(cp, 'lsf', queue, log)
        info['max_slots'] = maxSlots
        info['htpc'] = htpcRSL

        if maxSlots > 1:
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'

        info['extraCapabilities'] = extraCapabilities

        printTemplate(CE, info)
    return queueInfo, totalCpu, freeCpu, queueCpus
Example #12
0
File: pbs.py Project: bbockelm/gip
def print_CE(cp):
    pbsVersion = getLrmsInfo(cp)
    queueInfo = getQueueInfo(cp)
    totalCpu, freeCpu, queueCpus = parseNodes(cp, pbsVersion)
    log.debug("totalCpu, freeCpu, queueCPus: %s %s %s" % (totalCpu, freeCpu, queueCpus))
    ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
    CE = getTemplate("GlueCE", "GlueCEUniqueID")
    try:
        excludeQueues = [i.strip() for i in cp_get(cp, "pbs", \
            "queue_exclude", "").split(',')]
    except:
        excludeQueues = []
    vo_queues = getVoQueues(cp)
    for queue, info in queueInfo.items():
        if queue in excludeQueues:
            continue
        info["lrmsVersion"] = pbsVersion
        info["job_manager"] = "pbs"

        # if no jobs are waiting in the queue, set the number of free slots
        # to (job_slots - running), or the total number of free slots on the cluster,
        # whichever is less.

        info["queue"] = queue
        info["ceName"] = ce_name

        unique_id = buildCEUniqueID(cp, ce_name, 'pbs', queue)
        ceImpl, ceImplVersion = getCEImpl(cp)
	port = getPort(cp)

        info['ceUniqueID'] = unique_id
        if "job_slots" not in info:
            info["job_slots"] = totalCpu
        if "priority" not in info:
            info["priority"] = 0
        if "max_running" not in info:
            info["max_running"] = info["job_slots"]
        if "max_wall" not in info:
            info["max_wall"] = 1440

        
        info["free_slots"] = 0
        if info["wait"] == 0:
            freeSlots = info["job_slots"] - info["running"]
            if freeSlots > 0:
                info["free_slots"] =  min(freeSlots, freeCpu)

        log.debug("queue info: %s %s" % (queue, info))


        ert, wrt = responseTimes(cp, info.get("running", 0),
            info.get("wait", 0), max_job_time=info["max_wall"])

        info["job_slots"] = min(totalCpu, info["job_slots"])
        info['ert'] = ert
        info['wrt'] = wrt
        info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name)
        info['hostName'] = cp_get(cp, ce, 'host_name', ce_name)
        info['ceImpl'] = ceImpl
        info['ceImplVersion'] = ceImplVersion

	contact_string = buildContactString(cp, 'pbs', queue, unique_id, log)

        info['contact_string'] = contact_string
        info['app_dir'] = cp_get(cp, 'osg_dirs', 'app', "/UNKNOWN_APP")
        info['data_dir'] = cp_get(cp, 'osg_dirs', 'data', "/UNKNOWN_DATA")
        info['default_se'] = getDefaultSE(cp)
        if 'max_waiting' not in info:
            info['max_waiting'] = 999999
        if 'max_queuable' in info:
            info['max_total'] = info['max_queuable']
            info['free_slots'] = min(info['free_slots'], info['max_queuable'])
        else:
            info['max_total'] = info['max_waiting'] + info['max_running']
            info['free_slots'] = min(info['free_slots'], info['max_total'])

        # Enforce invariants:
        # max_total <= max_running
        # free_slots <= max_running
        info['max_total'] = min(info['max_total'], info['max_running'])
        info['free_slots'] = min(info['free_slots'], info['max_running'])

        info['assigned'] = info['job_slots']
        # Enforce invariants:
        # assigned <= max_running
        info['assigned'] = min(info['assigned'], info['max_running'])

        info['lrmsType'] = 'pbs'
        info['preemption'] = cp_get(cp, 'pbs', 'preemption', '0')
        acbr = ''
        has_vo = False
        for vo, queue2 in vo_queues:
            if queue == queue2:
                acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo
                has_vo = True
        if not has_vo:
            continue
        info['acbr'] = acbr[:-1]
        info['bdii'] = cp.get('bdii', 'endpoint')
        gramVersion = getGramVersion(cp)

        info['gramVersion'] = gramVersion
        info['port'] = port
        info['waiting'] = info['wait']
        info['referenceSI00'] = gip_cluster.getReferenceSI00(cp)
        info['clusterUniqueID'] = getClusterID(cp)

        extraCapabilities = ''
        if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'

        htpcRSL, maxSlots = getHTPCInfo(cp, 'pbs', queue, log)
        info['max_slots'] = maxSlots
        
        if maxSlots > 1:
            extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'

        info['extraCapabilities'] = extraCapabilities
        info['htpc'] = htpcRSL

        print CE % info
    return queueInfo, totalCpu, freeCpu, queueCpus