def _export_alns(pfam_url: str, dt: DirectoryTree, buffer_size: int = 1000):
logger.info("processing Pfam alignments")
df = DumpFile(dt.mktemp(), compress=True)
cnt = 0
iterator = pfam.get_alignments(pfam_url)
for entry_acc, aln_type, aln_bytes, count in iterator:
df.dump((entry_acc, f"alignment:{aln_type}", aln_bytes,
"application/gzip", count))
cnt += 1
if cnt == buffer_size:
df.close()
yield df.path
df = DumpFile(dt.mktemp(), compress=True)
cnt = 0
df.close()
yield df.path
def _export_hmms(p_uniprot2matches: str,
pro_url: str,
dt: DirectoryTree,
buffer_size: int = 1000):
logger.info("counting hits per model")
signatures = {}
with Store(p_uniprot2matches) as u2matches:
cnt = 0
for entries in u2matches.values():
for entry_acc, locations in entries.items():
for loc in locations:
if loc["model"] is None:
continue # InterPro entries
try:
models = signatures[entry_acc]
except KeyError:
models = signatures[entry_acc] = {}
try:
models[loc["model"]] += 1
except KeyError:
models[loc["model"]] = 1
cnt += 1
if not cnt % 10e6:
logger.info(f"{cnt:>12,}")
logger.info(f"{cnt:>12,}")
for entry_acc, models in signatures.items():
# Select the model with the most hits
model_acc = sorted(models, key=lambda k: (-models[k], k))[0]
signatures[entry_acc] = model_acc
logger.info("processing models")
df = DumpFile(dt.mktemp(), compress=True)
cnt = 0
ignored = 0
iterator = ippro.get_hmms(pro_url, multi_models=True)
for entry_acc, model_acc, hmm_bytes in iterator:
try:
representative_model = signatures[entry_acc]
except KeyError:
# Signature without matches, i.e. without representative model
ignored += 1
continue
if model_acc and model_acc != representative_model:
continue
hmm_str = gzip.decompress(hmm_bytes).decode("utf-8")
df.dump((entry_acc, "hmm", hmm_bytes, "application/gzip", None))
with StringIO(hmm_str) as stream:
hmm = hmmer.HMMFile(stream)
df.dump((entry_acc, "logo",
json.dumps(hmm.logo("info_content_all",
"hmm")), "application/json", None))
cnt += 2
if cnt >= buffer_size:
df.close()
yield df.path
df = DumpFile(dt.mktemp(), compress=True)
cnt = 0
df.close()
yield df.path
logger.info(f" {ignored} models ignored")