def main():
t0 = time.time()
np.random.seed(SEED)
logging.basicConfig(level=getattr(logging, LOGGING_LEVEL))
lods = get_lods(describe=True)
lods_copy = get_lods()
lods_copy.destroy_data()
out_data = init_distributed_dicts(lods)
out_data['lods'] = lods_copy
out_data['CONFIG'] = get_config_dict()
q = Queue()
for i in range(N_RUNS):
q.put(i)
for i in range(N_WORKERS):
q.put(-1) # -1 as a flag for stopping
processes = []
for i in range(N_WORKERS):
p = OneWorkerProcess(lods, q, out_data)
p.start()
os.system("taskset -p -c %d %d" %
(((N_WORKERS * 0 + i) % os.cpu_count()), p.pid))
processes.append(p)
for p in processes:
p.join()
print_err_aggregates(out_data['errs'])
t_final = time.time()
script_duration = t_final - t0
logging.info("Script total running time: {} minutes and {} seconds".format(
script_duration // 60, int(script_duration % 60)))
def main():
t0 = time.time()
np.random.seed(SEED)
logging.basicConfig(level=getattr(logging, LOGGING_LEVEL))
lods = get_lods(describe=True)
# 1. fetch the triplets and fetch data for extracting
# encodings
triplet_idxs = get_triplet_idxs(lods, with_shuffle=True, random_state=SEED)
n_triplets = len(triplet_idxs)
logging.info("Number of triplets: {}".format(n_triplets))
triplet_idxs = triplet_idxs[:n_triplets // 100]
logging.info("Number of triplets (after trimming): {}".format(
len(triplet_idxs)))
lods_copy = get_lods()
lods_copy.destroy_data()
out_data = init_distributed_dicts(lods)
out_data['lods'] = lods_copy
out_data['CONFIG'] = get_config_dict()
q = Queue()
for i in range(N_RUNS):
q.put(i)
for i in range(N_WORKERS):
q.put(-1) # -1 as a flag for stopping
processes = []
for i in range(N_WORKERS):
p = OneWorkerProcess(lods, triplet_idxs, q, out_data)
p.start()
os.system("taskset -p -c %d %d" %
(((N_WORKERS * 1 + i) % os.cpu_count()), p.pid))
processes.append(p)
for p in processes:
p.join()
print_err_aggregates(out_data['errs'])
t_final = time.time()
script_duration = t_final - t0
logging.info("Script total running time: {} minutes and {} seconds".format(
script_duration // 60, int(script_duration % 60)))
def main():
t0 = time.time()
np.random.seed(SEED)
logging.basicConfig(level=getattr(logging, LOGGING_LEVEL))
lods = get_lods(describe=True)
# 1. fetch the triplets and fetch data for extracting
# encodings
Ya, Yp, Yn = get_triplets(lods, random_state=SEED)
triplets = {'Ya': Ya,
'Yp': Yp,
'Yn': Yn}
logging.info("Triplets shapes: {}, {}, {}".format(Ya.shape,
Yp.shape,
Yn.shape))
lods_copy = get_lods()
lods_copy.destroy_data()
out_data = init_distributed_dicts(lods)
out_data['lods'] = lods_copy
out_data['CONFIG'] = get_config_dict()
q = Queue()
for i in range(N_RUNS):
q.put(i)
for i in range(N_WORKERS):
q.put(-1) # -1 as a flag for stopping
processes = []
for i in range(N_WORKERS):
p = OneWorkerProcess(lods, triplets, q, out_data)
p.start()
processes.append(p)
for p in processes:
p.join()
print_err_aggregates(out_data['errs'])
t_final = time.time()
script_duration = t_final - t0
logging.info("Script total running time: {} minutes and {} seconds".format(
script_duration // 60, int(script_duration % 60)))
