Esempio n. 1
0
def update_job_status(j):
    start = None
    end = None
    if j.status == "D":
        try:
            start, end = read_time_file(j.time_file)
        except Exception as e:
            log.warning("Execution time could not be loaded into DB: %s", j.jobid[:6])
            log.warning(e)
    db.update_task(j.jobid, status=j.status, tm_start=start, tm_end=end)
    return start, end
Esempio n. 2
0
def update_job_status(j):
    start = None
    end = None
    if j.status == "D":
        try:
            start, end = read_time_file(j.time_file)
        except Exception as e:
            log.warning("Execution time could not be loaded into DB: %s",
                        j.jobid[:6])
            log.warning(e)
    db.update_task(j.jobid, status=j.status, tm_start=start, tm_end=end)
    return start, end
Esempio n. 3
0
def update_task_states_recursively(task):
    task_start = 0
    task_end = 0
    for j in task.jobs:
        if isjob(j):
            start, end = update_job_status(j)
        else:
            start, end = update_task_states_recursively(j)
        task_start = min(task_start, start) if task_start > 0 else start
        task_end = max(task_end, end)

    db.update_task(task.taskid, status=task.status, tm_start=task_start, tm_end=task_end)
    return task_start, task_end
Esempio n. 4
0
def update_task_states_recursively(task):
    task_start = 0
    task_end = 0
    for j in task.jobs:
        if isjob(j):
            start, end = update_job_status(j)
        else:
            start, end = update_task_states_recursively(j)
        task_start = min(task_start, start) if task_start > 0 else start
        task_end = max(task_end, end)

    db.update_task(task.taskid,
                   status=task.status,
                   tm_start=task_start,
                   tm_end=task_end)
    return task_start, task_end
Esempio n. 5
0
def launch_jobs(jobs, conf):
    # Group jobs with identical config
    sge_path = GLOBALS["sge_dir"]

    conf2jobs = defaultdict(list)
    for j, cmd in jobs:
        job_config = conf["sge"].copy()
        job_config["-pe smp"] = j.cores
        for k, v in six.iteritems(j.sge):
            job_config[k] = v
        conf_key = tuple(sorted(job_config.items()))
        conf2jobs[conf_key].append((j, cmd))

    for job_config, commands in six.iteritems(conf2jobs):
        job_config = dict(job_config)
        job_file = "%s_%d_jobs" % (time.ctime().replace(" ", "_").replace(
            ":", "-"), len(commands))
        cmds_file = os.path.join(sge_path, job_file + ".cmds")
        qsub_file = os.path.join(sge_path, job_file + ".qsub")

        script = '''#!/bin/sh\n'''
        for k, v in six.iteritems(job_config):
            if not k.startswith("_"):
                script += '#$ %s %s\n' % (k, v)
        script += '#$ -f \n'
        script += '#$ -o %s\n' % sge_path
        script += '#$ -e %s\n' % sge_path
        script += '#$ -N %s\n' % "NPR%djobs" % len(commands)
        script += '#$ -t 1-%d\n' % len(commands)
        script += 'SEEDFILE=%s\n' % cmds_file
        script += 'sh -c "`cat $SEEDFILE | head -n $SGE_TASK_ID | tail -n 1`" \n'

        open(cmds_file, "w").write('\n'.join([cmd for j, cmd in commands]))
        open(qsub_file, "w").write(script)

        log.log(28, "Launching %d SGE jobs." % len(commands))
        log.debug(script)
        answer = run("SGE_CELL=%s qsub %s" % (job_config["_cell"], qsub_file))
        log.debug(answer)
        match = re.search(OK_PATTERN, answer)
        if match:
            jobid = match.groups()[0]
            for j, cmd in commands:
                db.update_task(j.jobid, status="Q", host="@sge", pid=jobid)
        else:
            raise SgeError(answer)
Esempio n. 6
0
File: sge.py Progetto: Ward9250/ete
def launch_jobs(jobs, conf):
    # Group jobs with identical config
    sge_path = GLOBALS["sge_dir"]

    conf2jobs = defaultdict(list)
    for j, cmd in jobs:
        job_config = conf["sge"].copy()
        job_config["-pe smp"] = j.cores
        for k,v in six.iteritems(j.sge):
            job_config[k] = v
        conf_key = tuple(sorted(job_config.items()))
        conf2jobs[conf_key].append((j,cmd))

    for job_config, commands in six.iteritems(conf2jobs):
        job_config = dict(job_config)
        job_file = "%s_%d_jobs" %(time.ctime().replace(" ", "_").replace(":","-"),
                                  len(commands))
        cmds_file = os.path.join(sge_path, job_file+".cmds")
        qsub_file = os.path.join(sge_path, job_file+".qsub")

        script =  '''#!/bin/sh\n'''
        for k,v in six.iteritems(job_config):
            if not k.startswith("_"):
                script += '#$ %s %s\n' %(k,v)
        script += '#$ -f \n'
        script += '#$ -o %s\n' % sge_path
        script += '#$ -e %s\n' % sge_path
        script += '#$ -N %s\n' % "NPR%djobs" %len(commands)
        script += '#$ -t 1-%d\n' % len(commands)
        script += 'SEEDFILE=%s\n' % cmds_file
        script += 'sh -c "`cat $SEEDFILE | head -n $SGE_TASK_ID | tail -n 1`" \n'

        open(cmds_file, "w").write('\n'.join([cmd for j,cmd in commands]))
        open(qsub_file, "w").write(script)

        log.log(28, "Launching %d SGE jobs." %len(commands))
        log.debug(script)
        answer = run("SGE_CELL=%s qsub %s" %(job_config["_cell"], qsub_file))
        log.debug(answer)
        match =  re.search(OK_PATTERN, answer)
        if match:
            jobid = match.groups()[0]
            for j, cmd in commands:
                db.update_task(j.jobid, status = "Q", host="@sge", pid=jobid)
        else:
            raise SgeError(answer)