def calculate_allele_length(output_dir, database, interval=20):
lf = logs.LoggerFactory()
lf.addConsoleHandler()
logger = lf.create()
db.load_database_config(logger=logger)
logger.info("Start calculating allele length heatmap...")
plot_length_heamap(output_dir, database, interval=interval)
def calculate_loci_coverage(input_dir, output_dir, database):
lf = logs.LoggerFactory()
lf.addConsoleHandler()
logger = lf.create()
db.load_database_config(logger=logger)
logger.info("Start calculating locus coverage...")
subject_number = count_subjects(input_dir)
logger.info("Start plotting locus coverage...")
plot_stats(output_dir, subject_number, database)
def power(database):
lf = logs.LoggerFactory()
lf.addConsoleHandler()
logger = lf.create()
db.load_database_config(logger=logger)
sql = "select locus_id, count(locus_id) as counts from pairs group by locus_id;"
counts = db.from_sql(sql, database=database)
counts["log_counts"] = np.log2(counts["counts"])
return np.sum(counts["log_counts"])
def profiling(output_dir, input_dir, database, threads, occr_level=None, selected_loci=None, logger=None,
aligcov_cut=0.5, identity=90):
load_database_config()
if not logger:
logger = logs.console_logger(__name__)
logger.info("Renaming contigs...")
query_dir = files.joinpath(output_dir, "query")
files.create_if_not_exist(query_dir)
namemap = rename(query_dir, input_dir)
with open(files.joinpath(output_dir, "namemap.json"), "w") as f:
f.write(json.dumps(namemap))
if os.path.isdir(database):
logger.info("Profiling loci...")
refseq_fna = files.joinpath(database, "panRefSeq.fa")
profile_loci(refseq_fna, query_dir, output_dir, aligcov_cut, identity, threads)
logger.info("Allocating alleles...")
profile_alleles(query_dir, database, output_dir, threads, occr_level)
else:
logger.info("Identifying loci and allocating alleles...")
# select loci by scheme
if selected_loci:
selected_loci = set(selected_loci)
else:
query = "select locus_id from scheme where occurence>={};".format(occr_level)
selected_loci = set(sql_query(query, database=database).iloc[:, 0])
temp_dir = os.path.join(query_dir, "temp")
files.create_if_not_exist(temp_dir)
collect = []
args = [(os.path.join(query_dir, filename), temp_dir) for filename in os.listdir(query_dir) if filename.endswith(".fa")]
with ProcessPoolExecutor(threads) as executor:
for filename in executor.map(identify_loci, args):
genome_id = files.fasta_filename(filename)
target_file = os.path.join(temp_dir, genome_id + ".locus.fna")
profile = profile_by_query(target_file, genome_id, selected_loci, database)
collect.append(profile)
result = pd.concat(collect, axis=1)
result.to_csv(files.joinpath(output_dir, "wgmlst.tsv"), sep="\t")
shutil.rmtree(query_dir)
def richness(database, weighted=True):
lf = logs.LoggerFactory()
lf.addConsoleHandler()
logger = lf.create()
db.load_database_config(logger=logger)
sql = "select a.locus_id, a.allele_id, b.count" \
" from pairs as a" \
" left join (select allele_id, count from alleles) as b" \
" on a.allele_id=b.allele_id;"
counts = db.from_sql(sql, database=database)
ent = counts.groupby("locus_id").agg({"count": locus_entropy})
if weighted:
sql = "select locus_id, occurrence from loci;"
loci = db.from_sql(sql, database=database)
weight = pd.merge(ent, loci, left_index=True, right_on="locus_id")
return np.average(weight["count"], weights=weight["occurrence"])
else:
return np.average(ent)
def make_database(output_dir, drop_by_occur, logger=None, threads=2):
if not logger:
lf = logs.LoggerFactory()
lf.addConsoleHandler()
lf.addFileHandler(os.path.join(output_dir, "make_database.log"))
logger = lf.create()
db.load_database_config(logger=logger)
logger.info("Calculating the pan genome...")
min_identity = 95
c = cmds.form_roary_cmd(os.path.join(output_dir, "GFF"), output_dir,
min_identity, threads)
logger.info("Run roary with following command: " + c)
subprocess.run(c, shell=True)
logger.info("Creating database")
dbname = os.path.basename(
output_dir[:-1] if output_dir.endswith("/") else output_dir)
db.createdb(dbname)
db.create_pgadb_relations(dbname)
logger.info("Extract profiles from roary result matrix...")
matrix_file = os.path.join(output_dir, "roary",
"gene_presence_absence.csv")
profiles, total_isolates = extract_profiles(matrix_file, dbname)
logger.info(
"Collecting allele profiles and making allele frequencies and reference sequence..."
)
ffn_dir = os.path.join(output_dir, "FFN")
profile_file = os.path.join(output_dir, "allele_profiles.tsv")
profiles, freq = collect_allele_info(profiles, ffn_dir)
logger.info("Checking duplicated loci by self-blastp...")
blastp_out_file, ref_length = reference_self_blastp(output_dir, freq)
logger.info(
"Filter out high identity loci and drop loci which occurrence less than {}..."
.format(drop_by_occur))
filtered_loci = filter_locus(blastp_out_file, ref_length, total_isolates,
drop_by_occur)
os.remove(blastp_out_file)
logger.info("Updating and saving profiles...")
freq = {l: freq[l] for l in filtered_loci}
profiles = profiles[profiles.index.isin(filtered_loci)]
profiles.to_csv(profile_file, sep="\t")
logger.info("Saving allele sequences...")
refseqs = {
locus: counter.most_common(1)[0][0]
for locus, counter in freq.items()
}
save_sequences(freq, refseqs, dbname)
logger.info("Making dynamic schemes...")
refseqs = dict(
map(lambda x: (x[0], operations.make_seqid(x[1])), refseqs.items()))
make_schemes(refseqs, total_isolates)
logger.info("Done!!")
return dbname
def profiling(output_dir,
input_dir,
database,
threads,
occr_level=None,
selected_loci=None,
profile_file="profile",
enable_adding_new_alleles=True,
generate_profiles=True,
generate_bn=True,
logger=None,
debug=False):
if not logger:
lf = logs.LoggerFactory()
lf.addConsoleHandler()
lf.addFileHandler(os.path.join(output_dir, "profiling.log"))
logger = lf.create()
db.load_database_config(logger=logger)
pid = uuid.uuid4().hex[0:8]
logger.info("Formating contigs...")
query_dir = os.path.join(output_dir, "query_{}".format(pid))
os.makedirs(query_dir, exist_ok=True)
contighandler = files.ContigHandler()
contighandler.new_format(input_dir, query_dir)
model = re.search('[a-zA-Z]+\w[a-zA-Z]+', database).group(0)
logger.info("Used model: {}".format(model))
logger.info("Selecting loci by specified scheme {}%...".format(occr_level))
if selected_loci:
selected_loci = set(selected_loci)
else: # select loci by scheme
query = "select locus_id from loci where occurrence>={};".format(
occr_level)
selected_loci = set(db.from_sql(query, database=database).iloc[:, 0])
logger.info("Making reference blastdb for blastp...")
temp_dir = os.path.join(query_dir, "temp_{}".format(pid))
os.makedirs(temp_dir, exist_ok=True)
ref_db = os.path.join(temp_dir, "ref_blastpdb_{}".format(pid))
ref_len = make_ref_blastpdb(ref_db, database)
logger.info("Identifying loci and allocating alleles...")
args = [(os.path.join(query_dir, filename), temp_dir, model)
for filename in os.listdir(query_dir) if filename.endswith(".fa")]
with ThreadPoolExecutor(threads) as executor:
id_allele_list = list(executor.map(identify_alleles, args))
if enable_adding_new_alleles:
logger.info("Adding new alleles to database...")
add_new_alleles(id_allele_list, ref_db, temp_dir, ref_len)
logger.info("Collecting allele profiles of each genomes...")
allele_counts = Counter()
if generate_profiles:
collect = []
for genome_id, alleles in id_allele_list:
profile = profile_by_query(alleles, genome_id, selected_loci,
database)
collect.append(profile)
allele_counts.update(alleles.keys())
result = pd.concat(collect, axis=1)
result.to_csv(os.path.join(output_dir, profile_file + ".tsv"),
sep="\t")
if generate_bn:
bio = to_bionumerics_format(result)
bio.to_csv(os.path.join(output_dir,
"bionumerics_{}.csv".format(pid)),
index=False)
else:
logger.info("Not going to output profiles.")
for genome_id, alleles in id_allele_list:
allele_counts.update(alleles.keys())
allele_counts = pd.DataFrame(allele_counts, index=[0]).T\
.reset_index().rename(columns={"index": "allele_id", 0: "count"})
update_allele_counts(allele_counts, database,
"batch_add_counts_{}".format(pid))
if not debug and os.path.exists(query_dir):
shutil.rmtree(query_dir)
logger.info("Done!")
from flask import abort, send_file
from flask_restful import Resource, reqparse
from werkzeug.datastructures import FileStorage
import psycopg2
from datetime import datetime
import hashlib
import os
from threading import Thread
from src.api import internals
from src.utils import db
INDIR = "input"
OUTDIR = "output"
DB = "profiling"
db.load_database_config()
def create_if_not_exist(path):
if not os.path.exists(path):
os.makedirs(path)
def get_seq_id(file):
file.seek(0)
m = hashlib.sha256()
m.update(file.read())
return m.hexdigest()
class UploadListAPI(Resource):
def __init__(self):