def __init__(self, count):
if Echo.__instance is not None:
return
else:
Echo.count = count
Echo.t = time.time()
Echo.echos_left = count
Echo.__instance = self
networking.message_clients("echo", env=app.config["ENV"])
def report_pos(client_names=None):
for chrom in store:
if chrom == "meta":
continue
data = {}
data[chrom] = store[f"{chrom}/PCA_passed"].value
msg = pickle.dumps(data)
networking.message_clients("pca/pcapos", data=msg, env=app.config["ENV"])
time.sleep(ServerHTTP.wait_time) # #TODO fix this hack. Give time for clients to finish
networking.message_clients("pca/cov", data=msg, env=app.config["ENV"])
def start_client_qc_task(filters, stage=Commands.QC):
global TIME
TIME = time.time()
if stage == Commands.QC:
filters["mask_prefix"] = "QC"
else:
filters["mask_prefix"] = "PCA"
data = pickle.dumps(filters)
networking.message_clients("qc", data=data, env=app.config["ENV"])
for client in clients:
Registry.get_instance().set_client_state(client['name'], stage)
def echo(self, client_name):
instances = Registry.get_instance()
instances.set_client_state(client_name, Commands.ECHO)
if instances.num_clients_in_state(Commands.ECHO) == len(instances):
Echo.echos_left -= 1
for client in instances.list_clients():
instances.set_client_state(client["name"], None)
if not Echo.echos_left:
Echo.__instance = None # remove the instance (essentially)
avg_t = (time.time() - Echo.t) / Echo.count
networking.message_clients("End_echo",
env=app.config["ENV"],
data=avg_t)
return avg_t
else:
networking.message_clients("echo", env=app.config["ENV"])
return None
def eigenDecompose(n_components):
chroms = sorted([int(v) for v in store.keys() if v != 'meta'])
cov_size = 0
meta = store["meta"]
if "Vs" not in meta:
for chrom in chroms:
cov_size += meta["{}_{}".format(chrom, chrom)].shape[0]
logging.info(f"""Starting covariance matrix of size {cov_size} x {cov_size} \
using chromosomes: {chroms}""")
cov = np.empty((cov_size, cov_size), dtype=np.float32)
i_old = 0
for chrom1 in chroms:
j_old = 0
for chrom2 in chroms:
if chrom2 > chrom1:
continue
cov_name = "{}_{}".format(chrom1, chrom2)
if cov_name in meta:
pcov = meta[cov_name].value
logging.debug(f"{cov_name} is of size {pcov.shape}.")
cov[i_old:i_old+pcov.shape[0], j_old:j_old+pcov.shape[1]] = pcov
cov[j_old:j_old+pcov.shape[1], i_old:i_old+pcov.shape[0]] = pcov.T
j_old += pcov.shape[1]
i_old += pcov.shape[0]
cov /= (cov.shape[0])
sigma, v = eig(cov, k=n_components, ncv=3*n_components) # , maxiter=20*cov.shape[0])
sigma, v = zip(*sorted(zip(sigma, v.T), reverse=True))
v = np.array(v)
sigma = np.array(sigma)
sigma[sigma < 0] = 0
logging.info(f"Top eigenvalues are {sigma}")
meta.create_dataset('Sigmas', data=sigma)
meta.create_dataset('Vs', data=v)
else:
logging.info("Eigenvalue decomposition has already been done.")
sigma = meta["Sigmas"].value
v = meta["Vs"].value
sigma = np.sqrt(sigma) * np.sqrt(v.shape[0])
inv_sigma = sigma.copy()
inv_sigma[inv_sigma > 0] = 1 / inv_sigma[inv_sigma > 0]
msg = {"ISIG": inv_sigma, "V": v, "CHROMS": chroms}
msg = pickle.dumps(msg)
networking.message_clients("pca/eig", data=msg, env=app.config["ENV"])
logging.info(f"PCA took roughly {time.time()-TIME:.1f} seconds.")
def count_stats():
N = float(store.attrs["N"])
task = "INIT"
clients = Registry.get_instance().list_clients()
for chrom in store.keys():
counts_dset = store["{}/counts".format(chrom)].value
missing_rate = counts_dset[:, 3] / float(N)
store.create_dataset(f"{chrom}/missing_rates", data=missing_rate)
af = (counts_dset[:, 2] * 2 + counts_dset[:, 1]).astype(float)
af /= (np.sum(counts_dset[:, :3], axis=1) * 2).astype(float)
# af = np.minimum(af, 1-af)
store.create_dataset("{}/allele_freq".format(chrom), data=af)
# var = counts_dset[:,0] * (2*af)**2
# var += counts_dset[:,1] * (1-2*af)**2
# var += counts_dset[:,2] * (2-2*af)**2
# var /= (N-counts_dset[:,3]) # 2*af*(1-af)
var = 2 * af * (1 - af)
store.create_dataset("{}/var".format(chrom), data=var)
hwe = hweP(counts_dset[:, :3].astype(np.int32), 1, 0)
# Need to Recompile HWEP with uint32
msg = {
"TASK": task,
"SUBTASK": "STATS",
"CHROM": chrom,
"HWE": hwe,
"MISS": missing_rate,
"AF": af,
"VAR": var
}
msg = pickle.dumps(msg)
networking.message_clients("init/stats", env=app.config["ENV"], data=msg)
store.create_dataset("{}/hwe".format(chrom), data=hwe)
for client in clients:
Registry.get_instance().set_client_state(client['name'], "DONE_INIT")
logging.info("Done with initialization")
make_plots("QC_pre_filter.png")
logging.info(f"Initialization took roughly {time.time()-TIME:.1f} seconds.")
def start_init_task():
global TIME
TIME = time.time()
for client in Registry.get_instance().list_clients():
Registry.get_instance().set_client_state(client['name'], Commands.INIT)
networking.message_clients("init", env=app.config["ENV"])
def send_request(self, data, subtask):
to_send = pickle.dumps(data)
networking.message_clients(f"asso/{subtask}", data=to_send, env=app.config["ENV"])
def send_request(self, data, params=None):
to_send = pickle.dumps(data)
if params is None:
networking.message_clients("pca/ld", env=app.config["ENV"], data=to_send)
else:
networking.message_clients("pca/ld", env=app.config["ENV"], data=to_send, args=params)
