def _debug_is_eq_v(vl1, vl2): if isinstance(vl1, str) and isinstance(vl2, str): return vl1 == vl2 if isinstance(vl1, tuple) and isinstance(vl2, tuple): return is_eq(vl1[0], vl2[0]) and is_eq(vl1[1], vl2[1]) return False
def schedule(jobs, sizs, estimator, memfile_path=None): if bconfig.DEBUG_PROCESS_PEAK_MEMORY and memfile_path: def register_memory_usage(): pid = os.getpid() peak = get_peak_mem() fp = open(memfile_path, 'a') print_tortoise_memory_log( { 'pid': pid, 'peak1': peak[0], 'peak2': peak[1], 'est': sizs[idx], 'bibs': bibs[idx] }, fp) fp.close() else: def register_memory_usage(): pass def run_job(idx): try: sys.stdout = output_killer jobs[idx]() register_memory_usage() os._exit(os.EX_OK) except Exception as e: f = open('/tmp/exception-%s' % str(os.getpid()), "w") f.write(str(e) + '\n') f.close() os._exit(os.EX_SOFTWARE) max_workers = get_cores_count() pid_2_idx = {} # free = get_free_memory() initial = get_total_memory() free = initial output_killer = open(os.devnull, 'w') ret_status = [None] * len(jobs) bibs = sizs sizs = map(estimator, sizs) free_idxs = range(len(jobs)) assert len(jobs) == len(sizs) == len(ret_status) == len(bibs) == len( free_idxs) done = 0. total = sum(sizs) biggest = max(sizs) logger.update_status(0., "0 / %d" % len(jobs)) too_big = [idx for idx in free_idxs if sizs[idx] > free] for idx in too_big: pid = os.fork() if pid == 0: # child run_job(idx) else: # parent done += sizs[idx] del free_idxs[idx] cpid, status = os.wait() logger.update_status( done / total, "%d / %d" % (len(jobs) - len(free_idxs), len(jobs))) ret_status[idx] = status assert cpid == pid while free_idxs or pid_2_idx: while len(pid_2_idx) < max_workers: idx = get_biggest_job_below(free, (sizs[idx] for idx in free_idxs)) if idx != -1: job_idx = free_idxs[idx] pid = os.fork() if pid == 0: # child os.nice(int((float(sizs[idx]) * 20.0 / biggest))) run_job(job_idx) else: # parent pid_2_idx[pid] = job_idx assert free > sizs[job_idx] free -= sizs[job_idx] del free_idxs[idx] else: break pid, status = os.wait() assert pid in pid_2_idx idx = pid_2_idx[pid] freed = sizs[idx] done += freed ret_status[idx] = status free += freed del pid_2_idx[pid] logger.update_status( done / total, "%d / %d" % (len(jobs) - len(free_idxs) - len(pid_2_idx), len(jobs))) logger.update_status_final("%d / %d" % (len(jobs), len(jobs))) assert is_eq(free, initial) assert not pid_2_idx assert not free_idxs assert len(jobs) == len(sizs) == len(ret_status) == len(bibs) assert all(stat is not None for stat in ret_status) return ret_status
pid = os.fork() if pid == 0: # child os.nice(int((float(sizs[idx]) * 20.0 / biggest))) run_job(job_idx) else: # parent pid_2_idx[pid] = job_idx assert free > sizs[job_idx] free -= sizs[job_idx] del free_idxs[idx] else: break pid, status = os.wait() assert pid in pid_2_idx idx = pid_2_idx[pid] freed = sizs[idx] done += freed ret_status[idx] = status free += freed del pid_2_idx[pid] update_status(done / total, "%d / %d" % (len(jobs) - len(free_idxs) - len(pid_2_idx), len(jobs))) update_status_final("%d / %d" % (len(jobs), len(jobs))) assert is_eq(free, initial) assert not pid_2_idx assert not free_idxs assert len(jobs) == len(sizs) == len(ret_status) == len(bibs) assert all(stat != None for stat in ret_status) return ret_status
os.nice(int((float(sizs[idx]) * 20.0 / biggest))) run_job(job_idx) else: # parent pid_2_idx[pid] = job_idx assert free > sizs[job_idx] free -= sizs[job_idx] del free_idxs[idx] else: break pid, status = os.wait() assert pid in pid_2_idx idx = pid_2_idx[pid] freed = sizs[idx] done += freed ret_status[idx] = status free += freed del pid_2_idx[pid] update_status( done / total, "%d / %d" % (len(jobs) - len(free_idxs) - len(pid_2_idx), len(jobs))) update_status_final("%d / %d" % (len(jobs), len(jobs))) assert is_eq(free, initial) assert not pid_2_idx assert not free_idxs assert len(jobs) == len(sizs) == len(ret_status) == len(bibs) assert all(stat != None for stat in ret_status) return ret_status