示例#1
0
文件: qsub.py 项目: ido/cobalt
def logjob(jobid, spec, log_to_console, ttyname=None):
    """
    log job info
    """
    # log jobid to stdout
    if jobid:
        if log_to_console:
            client_utils.logger.info(jobid)
        if spec.has_key('cobalt_log_file'):
            filename = spec['cobalt_log_file']
            template = string.Template(filename)
            filename = template.safe_substitute(jobid=jobid)
        else:
            filename = "%s/%s.cobaltlog" % (spec['outputdir'], jobid)
        try:
            with open(filename, "a") as cobalt_log_file:
                print >> cobalt_log_file, "Jobid: %s" % jobid
                print >> cobalt_log_file, "qsub %s" % (" ".join(sys.argv[1:]))
                print >> cobalt_log_file, ("%s submitted with cwd set to: %s" %
                    (client_utils.sec_to_str(time.time()), spec['cwd']))
                if ttyname is not None:
                    print >> cobalt_log_file, ("jobid %d submitted from terminal %s" %
                        (jobid, ttyname))
        except IOError:
            client_utils.logger.error("WARNING: failed to create cobalt log file at: %s: %s", filename, exc_info=True)
    else:
        client_utils.logger.error("failed to create the job.  Maybe a queue isn't there?")
示例#2
0
def logjob(jobid, spec, log_to_console, ttyname=None):
    """
    log job info
    """
    # log jobid to stdout
    if jobid:
        if log_to_console:
            client_utils.logger.info(jobid)
        if spec.has_key('cobalt_log_file'):
            filename = spec['cobalt_log_file']
            template = string.Template(filename)
            filename = template.safe_substitute(jobid=jobid)
        else:
            filename = "%s/%s.cobaltlog" % (spec['outputdir'], jobid)
        try:
            with open(filename, "a") as cobalt_log_file:
                print >> cobalt_log_file, "Jobid: %s" % jobid
                print >> cobalt_log_file, "qsub %s" % (" ".join(sys.argv[1:]))
                print >> cobalt_log_file, (
                    "%s submitted with cwd set to: %s" %
                    (client_utils.sec_to_str(time.time()), spec['cwd']))
                if ttyname is not None:
                    print >> cobalt_log_file, (
                        "jobid %d submitted from terminal %s" %
                        (jobid, ttyname))
        except IOError:
            client_utils.logger.error(
                "WARNING: failed to create cobalt log file at: %s: %s",
                filename,
                exc_info=True)
    else:
        client_utils.logger.error(
            "failed to create the job.  Maybe a queue isn't there?")
示例#3
0
文件: setres.py 项目: ido/cobalt
def validate_starttime(parser):
    """
    make sure the start time plus duration is valid
    """
    _localtime = client_utils.parse_datetime(client_utils.cobalt_date(time.localtime(time.time())))
    _res_st    = parser.options.start + parser.options.duration
    if _res_st < _localtime:
        client_utils.logger.error("Start time + duration %s already in the pass", client_utils.sec_to_str(_res_st))
        sys.exit(1)
示例#4
0
def validate_starttime(parser):
    """
    make sure the start time plus duration is valid
    """
    _localtime = client_utils.parse_datetime(
        client_utils.cobalt_date(time.localtime(time.time())))
    _res_st = parser.options.start + parser.options.duration
    if _res_st < _localtime:
        client_utils.logger.error(
            "Start time + duration %s already in the pass",
            client_utils.sec_to_str(_res_st))
        sys.exit(1)
示例#5
0
文件: qstat.py 项目: ido/cobalt
def get_output_for_jobs(parser,hinfo,queues):
    """
    get jobs from specified jobids
    """
    names              = parser.args if not parser.no_args() else ['*']
    user_name          = parser.options.user if parser.options.user != None else '*'
    query_dependencies = {'QueuedTime':
                              ['SubmitTime','StartTime'],'RunTime':['StartTime'],'TimeRemaining':['WallTime','StartTime']}

    try:
        query = []
        for n in names:
            if n=='*':
                query.append({'tag':'job', 'jobid':n, 'queue':'*'})
            elif [q['name'] for q in queues if q['name'] == n]:
                query.append({'tag':'job', 'queue':n, 'jobid':'*'})
            else:
                query.append({'tag':'job', 'jobid':int(n), 'queue':'*'})
    except ValueError:
        client_utils.logger.error("%s is not a valid jobid or queue name" % n)
        sys.exit(1)
    for q in query:
        for h in hinfo.long_header:
            if h == 'JobName':
                q.update({'jobname':'*'})
            elif h not in ['JobID', 'Queue']:
                q.update({h.lower():'*'})
            if h in query_dependencies:
                for x in query_dependencies[h]:
                    if x not in hinfo.header:
                        q.update({x.lower():'*'})
        q["user"] = user_name

    response = client_utils.component_call(QUEMGR, False, 'get_jobs', (query,))

    if not parser.no_args() and not response:
        sys.exit(1)

    if response:
        maxjoblen = max([len(str(item.get('jobid'))) for item in response])
        jobidfmt = "%%%ss" % maxjoblen
    # calculate derived values
    for j in response:
        # walltime
        walltime_secs = int(j['walltime']) * 60
        t = int(float(j['walltime']))
        h = int(math.floor(t/60))
        t -= (h * 60)
        j['walltime'] = "%02d:%02d:00" % (h, t)
        # jobid
        j['jobid'] = jobidfmt % j['jobid']
        # location
        if isinstance(j['location'], types.ListType) and len(j['location']) > 1:
            j['location'] = client_utils.merge_nodelist(j['location'])
        elif isinstance(j['location'], types.ListType) and len(j['location']) == 1:
            j['location'] = j['location'][0]
        # queuedtime
        if j.get('starttime') in ('-1', 'BUG', 'N/A', None):
            j['queuedtime'] = client_utils.get_elapsed_time(float(j.get('submittime')), time.time())
        else:
            j['queuedtime'] = client_utils.get_elapsed_time(float(j.get('submittime')), float(j['starttime']))
        # runtime
        if j.get('starttime') in ('-1', 'BUG', 'N/A', None):
            j['runtime'] = 'N/A'
        else:
            currtime = time.time()
            j['runtime'] = client_utils.get_elapsed_time( float(j['starttime']), time.time())
        # starttime
        if j.get('starttime') in ('-1', 'BUG', None):
            j['starttime'] = 'N/A'
        else:
            orig_starttime = float(j['starttime'])
            j['starttime'] = client_utils.sec_to_str(float(j['starttime']))
        # timeremaining
        if j.get('starttime') in ['-1', 'BUG', 'N/A' ,None]:
            j['timeremaining'] = 'N/A'
        else:
            time_remaining = walltime_secs - (currtime - orig_starttime)
            if time_remaining < 0:
                j['timeremaining'] = '00:00:00'
            else:
                s = int(time_remaining) % 60
                m = (int(time_remaining) % 3600) / 60
                h = int(time_remaining) / 3600
                j['timeremaining'] = "%02d:%02d:%02d" % (h, m, s)
        # jobname
        outputpath = j.get('outputpath')
        jobname    = j.get('jobname')
        # envs
        if j['envs'] is None:
            j.update({'envs':''})
        else:
            j['envs'] = ' '.join([str(x) + '=' + str(y) for x, y in j['envs'].iteritems()])
        # args
        j['args'] = ' '.join(j['args'])

        # make the SubmitTime readable by humans
        j['submittime'] = client_utils.sec_to_str(float(j['submittime']))

        j['outputpath'] = outputpath
        j['errorpath'] = j.get('errorpath')
        j['user_list'] = ':'.join(j['user_list'])

        if j['geometry'] != None:
            j['geometry'] = "x".join([str(i) for i in j['geometry']])
        else:
            j['geometry'] = 'Any'
    # any header that was not present in the query response has value set to '-'
    output = [[j.get(x, '-') for x in [y.lower() for y in hinfo.header]]
              for j in response]

    return output
示例#6
0
def main():
    """
    showres main
    """
    # setup logging for client. The clients should call this before doing anything else.
    client_utils.setup_logging(logging.INFO)

    # list of callback with its arguments
    callbacks = [
        # <cb function>     <cb args>
        [ cb_debug        , () ] ]

    # Get the version information
    opt_def =  __doc__.replace('__revision__',__revision__)
    opt_def =  opt_def.replace('__version__',__version__)

    parser = ArgParse(opt_def,callbacks)

    parser.parse_it() # parse the command line

    if not parser.no_args():
        client_utils.logger.error("No arguments needed")
    
    if parser.options.verbose != None and parser.options.really_verbose != None:
        client_utils.logger.error('Only use -l or -x not both')
        sys.exit(1)

    cluster = False
    if 'cluster' in client_utils.component_call(SYSMGR, False, 'get_implementation', ()):
        cluster = True

    reservations = client_utils.component_call(SCHMGR, False, 'get_reservations', 
                                               ([{'name':'*', 'users':'*','start':'*', 'duration':'*', 'partitions':'*', 
                                                  'cycle': '*', 'queue': '*', 'res_id': '*', 'cycle_id': '*','project':'*', 
                                                  'block_passthrough':'*'}], ))

    output = []

    verbose        = False
    really_verbose = False
    header = [('Reservation', 'Queue', 'User', 'Start', 'Duration','Passthrough', 'Partitions', 'Remaining','T-Minus')]

    if parser.options.verbose:
        verbose = True
        header = [('Reservation', 'Queue', 'User', 'Start', 'Duration',
                   'End Time', 'Cycle Time', 'Passthrough', 'Partitions', 'Remaining', 'T-Minus')]
    elif parser.options.really_verbose:
        really_verbose = True
        header = [('Reservation', 'Queue', 'User', 'Start', 'Duration','End Time', 'Cycle Time','Passthrough','Partitions', 
                   'Project', 'ResID', 'CycleID', 'Remaining', 'T-Minus' )]

    for res in reservations:

        passthrough = "Allowed"
        if res['block_passthrough']:
            passthrough = "Blocked"

        start     = float(res['start'])
        duration  = float(res['duration'])
        now       = time.time()

        deltatime = now - start
        remaining = "inactive" if deltatime < 0.0 else client_utils.get_elapsed_time(deltatime, duration, True)
        remaining = "00:00:00" if '-' in remaining else remaining
        tminus    = "active" if deltatime >= 0.0 else client_utils.get_elapsed_time(deltatime, duration, True)

        # do some crazy stuff to make reservations which cycle display the 
        # "next" start time
        if res['cycle']:
            cycle = float(res['cycle'])
            periods = math.floor((now - start)/cycle)
            # reservations can't become active until they pass the start time 
            # -- so negative periods aren't allowed
            if periods < 0:
                pass
            # if we are still inside the reservation, show when it started
            elif (now - start) % cycle < duration:
                start += periods * cycle
            # if we are in the dead time after the reservation ended, show 
            # when the next one starts
            else:
                start += (periods+1) * cycle
        if res['cycle_id'] == None:
            res['cycle_id'] = '-'

        if res['cycle']:
            cycle = float(res['cycle'])
            if cycle < (60 * 60 * 24):
                cycle = "%02d:%02d" % (cycle/3600, (cycle/60)%60)
            else:
                cycle = "%0.1f days" % (cycle / (60 * 60 * 24))
        else:
            cycle = None
        dmin = (duration/60)%60
        dhour = duration/3600

        time_fmt = "%c"
        starttime = time.strftime(time_fmt, time.localtime(start))
        endtime   = time.strftime(time_fmt, time.localtime(start + duration)) 

        if parser.options.oldts == None:
            #time_fmt += " %z (%Z)"
            starttime = client_utils.sec_to_str(start)
            endtime = client_utils.sec_to_str(start + duration)

        if really_verbose:
            output.append((res['name'], res['queue'], res['users'], 
                           starttime,"%02d:%02d" % (dhour, dmin),
                           endtime, cycle, passthrough,
                           mergelist(res['partitions'], cluster), 
                           res['project'], res['res_id'], res['cycle_id'], remaining, tminus))
        elif verbose:
            output.append((res['name'], res['queue'], res['users'], 
                           starttime,"%02d:%02d" % (dhour, dmin),
                           endtime, cycle, passthrough,
                           mergelist(res['partitions'], cluster), 
                           remaining, tminus))
        else:
            output.append((res['name'], res['queue'], res['users'], 
                           starttime,"%02d:%02d" % (dhour, dmin), passthrough,
                           mergelist(res['partitions'], cluster), 
                           remaining, tminus))

    output.sort( (lambda x,y: cmp( time.mktime(time.strptime(x[3].split('+')[0].split('-')[0].strip(), time_fmt)), 
                                   time.mktime(time.strptime(y[3].split('+')[0].split('-')[0].strip(), time_fmt))) ) )
    client_utils.print_tabular(header + output)
示例#7
0
def get_output_for_jobs(parser, hinfo, queues):
    """
    get jobs from specified jobids
    """
    names = parser.args if not parser.no_args() else ['*']
    user_name = parser.options.user if parser.options.user != None else '*'
    query_dependencies = {
        'QueuedTime': ['SubmitTime', 'StartTime'],
        'RunTime': ['StartTime'],
        'TimeRemaining': ['WallTime', 'StartTime']
    }

    try:
        query = []
        for n in names:
            if n == '*':
                query.append({'tag': 'job', 'jobid': n, 'queue': '*'})
            elif [q['name'] for q in queues if q['name'] == n]:
                query.append({'tag': 'job', 'queue': n, 'jobid': '*'})
            else:
                query.append({'tag': 'job', 'jobid': int(n), 'queue': '*'})
    except ValueError:
        client_utils.logger.error("%s is not a valid jobid or queue name" % n)
        sys.exit(1)
    for q in query:
        for h in hinfo.long_header:
            if h == 'JobName':
                q.update({'jobname': '*'})
            elif h not in ['JobID', 'Queue']:
                q.update({h.lower(): '*'})
            if h in query_dependencies:
                for x in query_dependencies[h]:
                    if x not in hinfo.header:
                        q.update({x.lower(): '*'})
        q["user"] = user_name

    response = client_utils.component_call(QUEMGR, False, 'get_jobs',
                                           (query, ))

    if not parser.no_args() and not response:
        sys.exit(1)

    if response:
        maxjoblen = max([len(str(item.get('jobid'))) for item in response])
        jobidfmt = "%%%ss" % maxjoblen
    # calculate derived values
    for j in response:
        # walltime
        walltime_secs = int(j['walltime']) * 60
        t = int(float(j['walltime']))
        h = int(math.floor(t / 60))
        t -= (h * 60)
        j['walltime'] = "%02d:%02d:00" % (h, t)
        # jobid
        j['jobid'] = jobidfmt % j['jobid']
        # location
        if isinstance(j['location'],
                      types.ListType) and len(j['location']) > 1:
            j['location'] = client_utils.merge_nodelist(j['location'])
        elif isinstance(j['location'], types.ListType) and len(
                j['location']) == 1:
            j['location'] = j['location'][0]
        # queuedtime
        if j.get('starttime') in ('-1', 'BUG', 'N/A', None):
            j['queuedtime'] = client_utils.get_elapsed_time(
                float(j.get('submittime')), time.time())
        else:
            j['queuedtime'] = client_utils.get_elapsed_time(
                float(j.get('submittime')), float(j['starttime']))
        # runtime
        if j.get('starttime') in ('-1', 'BUG', 'N/A', None):
            j['runtime'] = 'N/A'
        else:
            currtime = time.time()
            j['runtime'] = client_utils.get_elapsed_time(
                float(j['starttime']), time.time())
        # starttime
        if j.get('starttime') in ('-1', 'BUG', None):
            j['starttime'] = 'N/A'
        else:
            orig_starttime = float(j['starttime'])
            j['starttime'] = client_utils.sec_to_str(float(j['starttime']))
        # timeremaining
        if j.get('starttime') in ['-1', 'BUG', 'N/A', None]:
            j['timeremaining'] = 'N/A'
        else:
            time_remaining = walltime_secs - (currtime - orig_starttime)
            if time_remaining < 0:
                j['timeremaining'] = '00:00:00'
            else:
                s = int(time_remaining) % 60
                m = (int(time_remaining) % 3600) / 60
                h = int(time_remaining) / 3600
                j['timeremaining'] = "%02d:%02d:%02d" % (h, m, s)
        # jobname
        outputpath = j.get('outputpath')
        jobname = j.get('jobname')
        # envs
        if j['envs'] is None:
            j.update({'envs': ''})
        else:
            j['envs'] = ' '.join(
                [str(x) + '=' + str(y) for x, y in j['envs'].iteritems()])
        # args
        j['args'] = ' '.join(j['args'])

        # make the SubmitTime readable by humans
        j['submittime'] = client_utils.sec_to_str(float(j['submittime']))

        j['outputpath'] = outputpath
        j['errorpath'] = j.get('errorpath')
        j['user_list'] = ':'.join(j['user_list'])

        if j['geometry'] != None:
            j['geometry'] = "x".join([str(i) for i in j['geometry']])
        else:
            j['geometry'] = 'Any'
    # any header that was not present in the query response has value set to '-'
    output = [[j.get(x, '-') for x in [y.lower() for y in hinfo.header]]
              for j in response]

    return output
示例#8
0
def main():
    """
    showres main
    """
    # setup logging for client. The clients should call this before doing anything else.
    client_utils.setup_logging(logging.INFO)

    # list of callback with its arguments
    callbacks = [
        # <cb function>     <cb args>
        [cb_debug, ()]
    ]

    # Get the version information
    opt_def = __doc__.replace('__revision__', __revision__)
    opt_def = opt_def.replace('__version__', __version__)

    parser = ArgParse(opt_def, callbacks)

    parser.parse_it()  # parse the command line

    if not parser.no_args():
        client_utils.logger.error("No arguments needed")

    if parser.options.verbose != None and parser.options.really_verbose != None:
        client_utils.logger.error('Only use -l or -x not both')
        sys.exit(1)

    cluster = False
    if 'cluster' in client_utils.component_call(SYSMGR, False,
                                                'get_implementation', ()):
        cluster = True

    reservations = client_utils.component_call(SCHMGR, False,
                                               'get_reservations',
                                               ([{
                                                   'name': '*',
                                                   'users': '*',
                                                   'start': '*',
                                                   'duration': '*',
                                                   'partitions': '*',
                                                   'cycle': '*',
                                                   'queue': '*',
                                                   'res_id': '*',
                                                   'cycle_id': '*',
                                                   'project': '*',
                                                   'block_passthrough': '*'
                                               }], ))

    output = []

    verbose = False
    really_verbose = False
    header = [('Reservation', 'Queue', 'User', 'Start', 'Duration',
               'Passthrough', 'Partitions', 'Remaining', 'T-Minus')]

    if parser.options.verbose:
        verbose = True
        header = [
            ('Reservation', 'Queue', 'User', 'Start', 'Duration', 'End Time',
             'Cycle Time', 'Passthrough', 'Partitions', 'Remaining', 'T-Minus')
        ]
    elif parser.options.really_verbose:
        really_verbose = True
        header = [('Reservation', 'Queue', 'User', 'Start', 'Duration',
                   'End Time', 'Cycle Time', 'Passthrough', 'Partitions',
                   'Project', 'ResID', 'CycleID', 'Remaining', 'T-Minus')]

    for res in reservations:

        passthrough = "Allowed"
        if res['block_passthrough']:
            passthrough = "Blocked"

        start = float(res['start'])
        duration = float(res['duration'])
        now = time.time()

        deltatime = now - start
        remaining = "inactive" if deltatime < 0.0 else client_utils.get_elapsed_time(
            deltatime, duration, True)
        remaining = "00:00:00" if '-' in remaining else remaining
        tminus = "active" if deltatime >= 0.0 else client_utils.get_elapsed_time(
            deltatime, duration, True)

        # do some crazy stuff to make reservations which cycle display the
        # "next" start time
        if res['cycle']:
            cycle = float(res['cycle'])
            periods = math.floor((now - start) / cycle)
            # reservations can't become active until they pass the start time
            # -- so negative periods aren't allowed
            if periods < 0:
                pass
            # if we are still inside the reservation, show when it started
            elif (now - start) % cycle < duration:
                start += periods * cycle
            # if we are in the dead time after the reservation ended, show
            # when the next one starts
            else:
                start += (periods + 1) * cycle
        if res['cycle_id'] == None:
            res['cycle_id'] = '-'

        if res['cycle']:
            cycle = float(res['cycle'])
            if cycle < (60 * 60 * 24):
                cycle = "%02d:%02d" % (cycle / 3600, (cycle / 60) % 60)
            else:
                cycle = "%0.1f days" % (cycle / (60 * 60 * 24))
        else:
            cycle = None
        dmin = (duration / 60) % 60
        dhour = duration / 3600

        time_fmt = "%c"
        starttime = time.strftime(time_fmt, time.localtime(start))
        endtime = time.strftime(time_fmt, time.localtime(start + duration))

        if parser.options.oldts == None:
            #time_fmt += " %z (%Z)"
            starttime = client_utils.sec_to_str(start)
            endtime = client_utils.sec_to_str(start + duration)

        if really_verbose:
            output.append(
                (res['name'], res['queue'], res['users'], starttime,
                 "%02d:%02d" % (dhour, dmin), endtime, cycle, passthrough,
                 mergelist(res['partitions'], cluster), res['project'],
                 res['res_id'], res['cycle_id'], remaining, tminus))
        elif verbose:
            output.append(
                (res['name'], res['queue'], res['users'], starttime,
                 "%02d:%02d" % (dhour, dmin), endtime, cycle, passthrough,
                 mergelist(res['partitions'], cluster), remaining, tminus))
        else:
            output.append((res['name'], res['queue'], res['users'], starttime,
                           "%02d:%02d" % (dhour, dmin), passthrough,
                           mergelist(res['partitions'],
                                     cluster), remaining, tminus))

    output.sort((lambda x, y: cmp(
        time.mktime(
            time.strptime(x[3].split('+')[0].split('-')[0].strip(), time_fmt)),
        time.mktime(
            time.strptime(y[3].split('+')[0].split('-')[0].strip(), time_fmt)))
                 ))
    client_utils.print_tabular(header + output)