def __init__(self, count): if Echo.__instance is not None: return else: Echo.count = count Echo.t = time.time() Echo.echos_left = count Echo.__instance = self networking.message_clients("echo", env=app.config["ENV"])
def report_pos(client_names=None): for chrom in store: if chrom == "meta": continue data = {} data[chrom] = store[f"{chrom}/PCA_passed"].value msg = pickle.dumps(data) networking.message_clients("pca/pcapos", data=msg, env=app.config["ENV"]) time.sleep(ServerHTTP.wait_time) # #TODO fix this hack. Give time for clients to finish networking.message_clients("pca/cov", data=msg, env=app.config["ENV"])
def start_client_qc_task(filters, stage=Commands.QC): global TIME TIME = time.time() if stage == Commands.QC: filters["mask_prefix"] = "QC" else: filters["mask_prefix"] = "PCA" data = pickle.dumps(filters) networking.message_clients("qc", data=data, env=app.config["ENV"]) for client in clients: Registry.get_instance().set_client_state(client['name'], stage)
def echo(self, client_name): instances = Registry.get_instance() instances.set_client_state(client_name, Commands.ECHO) if instances.num_clients_in_state(Commands.ECHO) == len(instances): Echo.echos_left -= 1 for client in instances.list_clients(): instances.set_client_state(client["name"], None) if not Echo.echos_left: Echo.__instance = None # remove the instance (essentially) avg_t = (time.time() - Echo.t) / Echo.count networking.message_clients("End_echo", env=app.config["ENV"], data=avg_t) return avg_t else: networking.message_clients("echo", env=app.config["ENV"]) return None
def eigenDecompose(n_components): chroms = sorted([int(v) for v in store.keys() if v != 'meta']) cov_size = 0 meta = store["meta"] if "Vs" not in meta: for chrom in chroms: cov_size += meta["{}_{}".format(chrom, chrom)].shape[0] logging.info(f"""Starting covariance matrix of size {cov_size} x {cov_size} \ using chromosomes: {chroms}""") cov = np.empty((cov_size, cov_size), dtype=np.float32) i_old = 0 for chrom1 in chroms: j_old = 0 for chrom2 in chroms: if chrom2 > chrom1: continue cov_name = "{}_{}".format(chrom1, chrom2) if cov_name in meta: pcov = meta[cov_name].value logging.debug(f"{cov_name} is of size {pcov.shape}.") cov[i_old:i_old+pcov.shape[0], j_old:j_old+pcov.shape[1]] = pcov cov[j_old:j_old+pcov.shape[1], i_old:i_old+pcov.shape[0]] = pcov.T j_old += pcov.shape[1] i_old += pcov.shape[0] cov /= (cov.shape[0]) sigma, v = eig(cov, k=n_components, ncv=3*n_components) # , maxiter=20*cov.shape[0]) sigma, v = zip(*sorted(zip(sigma, v.T), reverse=True)) v = np.array(v) sigma = np.array(sigma) sigma[sigma < 0] = 0 logging.info(f"Top eigenvalues are {sigma}") meta.create_dataset('Sigmas', data=sigma) meta.create_dataset('Vs', data=v) else: logging.info("Eigenvalue decomposition has already been done.") sigma = meta["Sigmas"].value v = meta["Vs"].value sigma = np.sqrt(sigma) * np.sqrt(v.shape[0]) inv_sigma = sigma.copy() inv_sigma[inv_sigma > 0] = 1 / inv_sigma[inv_sigma > 0] msg = {"ISIG": inv_sigma, "V": v, "CHROMS": chroms} msg = pickle.dumps(msg) networking.message_clients("pca/eig", data=msg, env=app.config["ENV"]) logging.info(f"PCA took roughly {time.time()-TIME:.1f} seconds.")
def count_stats(): N = float(store.attrs["N"]) task = "INIT" clients = Registry.get_instance().list_clients() for chrom in store.keys(): counts_dset = store["{}/counts".format(chrom)].value missing_rate = counts_dset[:, 3] / float(N) store.create_dataset(f"{chrom}/missing_rates", data=missing_rate) af = (counts_dset[:, 2] * 2 + counts_dset[:, 1]).astype(float) af /= (np.sum(counts_dset[:, :3], axis=1) * 2).astype(float) # af = np.minimum(af, 1-af) store.create_dataset("{}/allele_freq".format(chrom), data=af) # var = counts_dset[:,0] * (2*af)**2 # var += counts_dset[:,1] * (1-2*af)**2 # var += counts_dset[:,2] * (2-2*af)**2 # var /= (N-counts_dset[:,3]) # 2*af*(1-af) var = 2 * af * (1 - af) store.create_dataset("{}/var".format(chrom), data=var) hwe = hweP(counts_dset[:, :3].astype(np.int32), 1, 0) # Need to Recompile HWEP with uint32 msg = { "TASK": task, "SUBTASK": "STATS", "CHROM": chrom, "HWE": hwe, "MISS": missing_rate, "AF": af, "VAR": var } msg = pickle.dumps(msg) networking.message_clients("init/stats", env=app.config["ENV"], data=msg) store.create_dataset("{}/hwe".format(chrom), data=hwe) for client in clients: Registry.get_instance().set_client_state(client['name'], "DONE_INIT") logging.info("Done with initialization") make_plots("QC_pre_filter.png") logging.info(f"Initialization took roughly {time.time()-TIME:.1f} seconds.")
def start_init_task(): global TIME TIME = time.time() for client in Registry.get_instance().list_clients(): Registry.get_instance().set_client_state(client['name'], Commands.INIT) networking.message_clients("init", env=app.config["ENV"])
def send_request(self, data, subtask): to_send = pickle.dumps(data) networking.message_clients(f"asso/{subtask}", data=to_send, env=app.config["ENV"])
def send_request(self, data, params=None): to_send = pickle.dumps(data) if params is None: networking.message_clients("pca/ld", env=app.config["ENV"], data=to_send) else: networking.message_clients("pca/ld", env=app.config["ENV"], data=to_send, args=params)