Exemplo n.º 1
0
    def _debug_is_eq_v(vl1, vl2):
        if isinstance(vl1, str) and isinstance(vl2, str):
            return vl1 == vl2

        if isinstance(vl1, tuple) and isinstance(vl2, tuple):
            return is_eq(vl1[0], vl2[0]) and is_eq(vl1[1], vl2[1])

        return False
    def _debug_is_eq_v(vl1, vl2):
        if isinstance(vl1, str) and isinstance(vl2, str):
            return vl1 == vl2

        if isinstance(vl1, tuple) and isinstance(vl2, tuple):
            return is_eq(vl1[0], vl2[0]) and is_eq(vl1[1], vl2[1])

        return False
Exemplo n.º 3
0
def schedule(jobs, sizs, estimator, memfile_path=None):
    if bconfig.DEBUG_PROCESS_PEAK_MEMORY and memfile_path:

        def register_memory_usage():
            pid = os.getpid()
            peak = get_peak_mem()
            fp = open(memfile_path, 'a')
            print_tortoise_memory_log(
                {
                    'pid': pid,
                    'peak1': peak[0],
                    'peak2': peak[1],
                    'est': sizs[idx],
                    'bibs': bibs[idx]
                }, fp)
            fp.close()
    else:

        def register_memory_usage():
            pass

    def run_job(idx):
        try:
            sys.stdout = output_killer
            jobs[idx]()
            register_memory_usage()
            os._exit(os.EX_OK)
        except Exception as e:
            f = open('/tmp/exception-%s' % str(os.getpid()), "w")
            f.write(str(e) + '\n')
            f.close()
            os._exit(os.EX_SOFTWARE)

    max_workers = get_cores_count()
    pid_2_idx = {}
    # free = get_free_memory()
    initial = get_total_memory()
    free = initial
    output_killer = open(os.devnull, 'w')

    ret_status = [None] * len(jobs)
    bibs = sizs
    sizs = map(estimator, sizs)
    free_idxs = range(len(jobs))
    assert len(jobs) == len(sizs) == len(ret_status) == len(bibs) == len(
        free_idxs)

    done = 0.
    total = sum(sizs)
    biggest = max(sizs)

    logger.update_status(0., "0 / %d" % len(jobs))
    too_big = [idx for idx in free_idxs if sizs[idx] > free]
    for idx in too_big:
        pid = os.fork()
        if pid == 0:  # child
            run_job(idx)
        else:  # parent
            done += sizs[idx]
            del free_idxs[idx]
            cpid, status = os.wait()
            logger.update_status(
                done / total,
                "%d / %d" % (len(jobs) - len(free_idxs), len(jobs)))
            ret_status[idx] = status
            assert cpid == pid

    while free_idxs or pid_2_idx:
        while len(pid_2_idx) < max_workers:
            idx = get_biggest_job_below(free, (sizs[idx] for idx in free_idxs))
            if idx != -1:
                job_idx = free_idxs[idx]
                pid = os.fork()
                if pid == 0:  # child
                    os.nice(int((float(sizs[idx]) * 20.0 / biggest)))
                    run_job(job_idx)
                else:  # parent
                    pid_2_idx[pid] = job_idx
                    assert free > sizs[job_idx]
                    free -= sizs[job_idx]
                    del free_idxs[idx]
            else:
                break

        pid, status = os.wait()
        assert pid in pid_2_idx
        idx = pid_2_idx[pid]
        freed = sizs[idx]
        done += freed
        ret_status[idx] = status
        free += freed
        del pid_2_idx[pid]
        logger.update_status(
            done / total, "%d / %d" %
            (len(jobs) - len(free_idxs) - len(pid_2_idx), len(jobs)))

    logger.update_status_final("%d / %d" % (len(jobs), len(jobs)))
    assert is_eq(free, initial)
    assert not pid_2_idx
    assert not free_idxs
    assert len(jobs) == len(sizs) == len(ret_status) == len(bibs)
    assert all(stat is not None for stat in ret_status)

    return ret_status
Exemplo n.º 4
0
                pid = os.fork()
                if pid == 0: # child
                    os.nice(int((float(sizs[idx]) * 20.0 / biggest)))
                    run_job(job_idx)
                else: # parent
                    pid_2_idx[pid] = job_idx
                    assert free > sizs[job_idx]
                    free -= sizs[job_idx]
                    del free_idxs[idx]
            else:
                break

        pid, status = os.wait()
        assert pid in pid_2_idx
        idx = pid_2_idx[pid]
        freed = sizs[idx]
        done += freed
        ret_status[idx] = status
        free += freed
        del pid_2_idx[pid]
        update_status(done / total, "%d / %d" % (len(jobs) - len(free_idxs) - len(pid_2_idx), len(jobs)))

    update_status_final("%d / %d" % (len(jobs), len(jobs)))
    assert is_eq(free, initial)
    assert not pid_2_idx
    assert not free_idxs
    assert len(jobs) == len(sizs) == len(ret_status) == len(bibs)
    assert all(stat != None for stat in ret_status)

    return ret_status
Exemplo n.º 5
0
                    os.nice(int((float(sizs[idx]) * 20.0 / biggest)))
                    run_job(job_idx)
                else:  # parent
                    pid_2_idx[pid] = job_idx
                    assert free > sizs[job_idx]
                    free -= sizs[job_idx]
                    del free_idxs[idx]
            else:
                break

        pid, status = os.wait()
        assert pid in pid_2_idx
        idx = pid_2_idx[pid]
        freed = sizs[idx]
        done += freed
        ret_status[idx] = status
        free += freed
        del pid_2_idx[pid]
        update_status(
            done / total, "%d / %d" %
            (len(jobs) - len(free_idxs) - len(pid_2_idx), len(jobs)))

    update_status_final("%d / %d" % (len(jobs), len(jobs)))
    assert is_eq(free, initial)
    assert not pid_2_idx
    assert not free_idxs
    assert len(jobs) == len(sizs) == len(ret_status) == len(bibs)
    assert all(stat != None for stat in ret_status)

    return ret_status