def main(): t0 = time.time() np.random.seed(SEED) logging.basicConfig(level=getattr(logging, LOGGING_LEVEL)) lods = get_lods(describe=True) lods_copy = get_lods() lods_copy.destroy_data() out_data = init_distributed_dicts(lods) out_data['lods'] = lods_copy out_data['CONFIG'] = get_config_dict() q = Queue() for i in range(N_RUNS): q.put(i) for i in range(N_WORKERS): q.put(-1) # -1 as a flag for stopping processes = [] for i in range(N_WORKERS): p = OneWorkerProcess(lods, q, out_data) p.start() os.system("taskset -p -c %d %d" % (((N_WORKERS * 0 + i) % os.cpu_count()), p.pid)) processes.append(p) for p in processes: p.join() print_err_aggregates(out_data['errs']) t_final = time.time() script_duration = t_final - t0 logging.info("Script total running time: {} minutes and {} seconds".format( script_duration // 60, int(script_duration % 60)))
def main(): t0 = time.time() np.random.seed(SEED) logging.basicConfig(level=getattr(logging, LOGGING_LEVEL)) lods = get_lods(describe=True) # 1. fetch the triplets and fetch data for extracting # encodings triplet_idxs = get_triplet_idxs(lods, with_shuffle=True, random_state=SEED) n_triplets = len(triplet_idxs) logging.info("Number of triplets: {}".format(n_triplets)) triplet_idxs = triplet_idxs[:n_triplets // 100] logging.info("Number of triplets (after trimming): {}".format( len(triplet_idxs))) lods_copy = get_lods() lods_copy.destroy_data() out_data = init_distributed_dicts(lods) out_data['lods'] = lods_copy out_data['CONFIG'] = get_config_dict() q = Queue() for i in range(N_RUNS): q.put(i) for i in range(N_WORKERS): q.put(-1) # -1 as a flag for stopping processes = [] for i in range(N_WORKERS): p = OneWorkerProcess(lods, triplet_idxs, q, out_data) p.start() os.system("taskset -p -c %d %d" % (((N_WORKERS * 1 + i) % os.cpu_count()), p.pid)) processes.append(p) for p in processes: p.join() print_err_aggregates(out_data['errs']) t_final = time.time() script_duration = t_final - t0 logging.info("Script total running time: {} minutes and {} seconds".format( script_duration // 60, int(script_duration % 60)))
def main(): t0 = time.time() np.random.seed(SEED) logging.basicConfig(level=getattr(logging, LOGGING_LEVEL)) lods = get_lods(describe=True) # 1. fetch the triplets and fetch data for extracting # encodings Ya, Yp, Yn = get_triplets(lods, random_state=SEED) triplets = {'Ya': Ya, 'Yp': Yp, 'Yn': Yn} logging.info("Triplets shapes: {}, {}, {}".format(Ya.shape, Yp.shape, Yn.shape)) lods_copy = get_lods() lods_copy.destroy_data() out_data = init_distributed_dicts(lods) out_data['lods'] = lods_copy out_data['CONFIG'] = get_config_dict() q = Queue() for i in range(N_RUNS): q.put(i) for i in range(N_WORKERS): q.put(-1) # -1 as a flag for stopping processes = [] for i in range(N_WORKERS): p = OneWorkerProcess(lods, triplets, q, out_data) p.start() processes.append(p) for p in processes: p.join() print_err_aggregates(out_data['errs']) t_final = time.time() script_duration = t_final - t0 logging.info("Script total running time: {} minutes and {} seconds".format( script_duration // 60, int(script_duration % 60)))