def pipeline_STEP6(cfg, logger, cpu_number):
s = "Working with STEP 6: SECOND ALIGNMENT CLEANING"
print(s)
logger.info(s)
input_folder = cfg["input_folder"]
output_folder = cfg["output_folder"]
in_suffix = cfg["input_suffix"]
out_suffix = cfg["output_suffix"]
in_format = cfg["input_format"]
create_result_folder(output_folder, logger)
os.chdir(input_folder)
starting = get_starting_files(input_folder, output_folder, in_format,
out_suffix)
starting_files = [x for x in starting if in_suffix in x]
logger.info("There are {} input fasta files".format(len(starting_files)))
if starting_files:
manager = Manager()
fastas = manager.Queue()
result_dict = manager.dict()
process_future_fasta(
STEP6.trim_clean,
starting_files,
result_dict,
fastas,
cpu_number,
logger,
cfg,
tqdm_desc="Cleaning",
)
logger.info("STEP 6 finished")
def runMakeblastdb(cfg, logger, cpu_number):
logger.info("Running blastdb")
in_folder = cfg["input_folder"]
in_suffix = cfg["input_suffix"]
blastn_db_folder = cfg["blastn_databases_folder"]
create_result_folder(blastn_db_folder, logger)
os.chdir(in_folder)
finished_files = [x.split(".nin")[0] for x in os.listdir(blastn_db_folder)]
starting_files = sorted([
x for x in os.listdir(os.getcwd())
if in_suffix in x and x.split(in_suffix)[0] not in finished_files
])
if cfg["fasta_sanity_check"]:
for fasta in starting_files:
is_fasta(fasta)
logger.info("There are {} input fasta files".format(len(starting_files)))
if starting_files:
manager = Manager()
fastas = manager.Queue()
result_dict = manager.dict()
process_future_fasta(
primaryScript.run_makeblastdb,
starting_files,
result_dict,
fastas,
cpu_number,
logger,
cfg,
tqdm_desc="Running blastdb",
)
logger.info("run_makeblastdb finished")
def pipeline_STEP5(cfg, logger, cpu_number):
s = "Working with STEP 5: INPARALOGS SEPARATION"
print(s)
logger.info(s)
input_folder = cfg["input_folder"]
output_folder = cfg["output_folder"]
in_suffix = cfg["input_suffix"]
out_suffix = cfg["output_suffix"]
in_format = cfg["input_format"]
min_taxa = cfg["min_taxa_in_alignment"]
create_result_folder(output_folder, logger)
os.chdir(input_folder)
starting = get_starting_files(input_folder, output_folder, in_format,
out_suffix)
starting_files = [
x for x in starting if in_suffix in x and find_n_taxa(x, min_taxa)
]
logger.info("There are {} input fasta files".format(len(starting_files)))
if starting_files:
manager = Manager()
fastas = manager.Queue()
result_dict = manager.dict()
process_future_fasta(
STEP5.inparalog_separation,
starting_files,
result_dict,
fastas,
cpu_number,
logger,
cfg,
tqdm_desc="Inparalogs",
)
cleaning_dnadist(logger)
logger.info("STEP 5 finished")
def pipeline_STEP3(cfg, logger, cpu_number):
s = "Working with STEP 3: ALIGNMENT CLEANING"
print(s)
logger.info(s)
input_folder = cfg["input_folder"]
output_folder = cfg["output_folder"]
in_suffix = cfg["input_suffix"]
out_suffix = cfg["output_suffix"]
create_result_folder(output_folder, logger)
os.chdir(input_folder)
starting_files = get_starting_files(input_folder, output_folder, in_suffix,
out_suffix)
logger.debug("There are {} input fasta files".format(len(starting_files)))
if starting_files:
manager = Manager()
fastas = manager.Queue()
result_dict = manager.dict()
process_future_fasta(
STEP3.find_unspliced_segments,
starting_files,
result_dict,
fastas,
cpu_number,
logger,
cfg,
tqdm_desc="Cleaning",
)
logger.info("STEP 3 finished")
def runBLAST(cfg, logger, cpu_number):
logger.info("Running BLAST")
in_folder = cfg["input_folder"]
blastn_dbs = cfg["blastn_databases_folder"]
in_suffix = cfg["input_suffix"]
out_folder = cfg["blastn_results_folder"]
create_result_folder(out_folder, logger)
if not os.path.exists(blastn_dbs) or not os.listdir(blastn_dbs):
exception = "[error]: either blastdb folder {} is empty or \
it has not been created by running 'run_makeblastdb'".format(blastn_dbs)
raise Exception(exception)
sys.exit()
if not os.path.exists(in_folder) or not os.listdir(in_folder):
exception = "[error]: either fasta input folder {} is empty or \
it does not exist".format(in_folder)
raise Exception(exception)
sys.exit()
os.chdir(in_folder)
starting_files = sorted(
[x for x in os.listdir(os.getcwd()) if in_suffix in x])
if cfg["fasta_sanity_check"]:
for fasta in starting_files:
primaryScript.is_fasta(fasta)
logger.info("There are {} input fasta files".format(len(starting_files)))
fasta_to_db = primaryScript.fasta_to_blastdb(starting_files, cfg)
logger.info("Starting BLAST...")
if fasta_to_db:
manager = Manager()
fastas = manager.Queue()
result_dict = manager.dict()
process_future_fasta(
primaryScript.run_blastn,
fasta_to_db,
result_dict,
fastas,
cpu_number,
logger,
cfg,
tqdm_desc="Running BLAST",
)
logger.info("run_BLAST finished")
def parseXMLs(cfg, logger, cpu_number, temporary_folder):
logger.info("Parsing xmls")
in_folder = cfg["blastn_results_folder"]
if not os.path.exists(in_folder) or not [
x for x in os.listdir(in_folder) if ".xml" in x
]:
exception = "[error]: either blastn_results_folder {} is empty or \
it has not been created by running 'run_BLAST'".format(in_folder)
raise Exception(exception)
sys.exit()
os.chdir(in_folder)
starting_files = sorted(
[x for x in os.listdir(os.getcwd()) if ".xml" in x])
logger.info("There are {} input xml files".format(len(starting_files)))
logger.debug("Parsing xml files")
if starting_files:
manager = Manager()
fastas = manager.Queue()
result_dict = manager.dict()
process_future_fasta(
primaryScript.parse_pickle,
starting_files,
result_dict,
fastas,
cpu_number,
logger,
temporary_folder,
min_align_length=cfg["min_align_len"],
tqdm_desc="Parsing xmls",
)
os.chdir(temporary_folder)
dat_files = [x for x in os.listdir(os.getcwd()) if ".dat" in x]
taxa_pairs = primaryScript.find_dat_pairs(dat_files)
common_hits = shelve.open("common_hits_dict")
for pair in taxa_pairs:
primaryScript.populate_shelve(common_hits, pair)
common_hits.close()
logger.info("Done parsing xmls")
def PhaseAlleles(cfg, logger, cpu_number, **kargs):
logger.info("Phasing alleles starting")
in_folder = cfg["input_folder"]
in_suffix = cfg["input_suffix"]
out_suffix = cfg["output_suffix"]
out_folder = cfg["output_folder"]
create_result_folder(out_folder, logger)
logger.debug("Checking the settings")
temporary_folder = kargs["temp"]
phase.check_settings(cfg, logger)
starting = get_starting_files(in_folder, out_folder, in_suffix, out_suffix)
starting_files = [x for x in starting if in_suffix in x] # [:1]
s = "There are {} input fasta files".format(len(starting_files))
print(s)
logger.info(s)
samtools_v = phase.find_samtools_version(cfg)
print("samtools version is: .X", samtools_v)
if starting_files:
manager = Manager()
fastas = manager.Queue()
result_dict = manager.dict()
process_future_fasta(
phase.phasing_contig,
starting_files,
result_dict,
fastas,
cpu_number,
logger,
cfg,
temp=temporary_folder,
samtools_version=samtools_v,
tqdm_desc="phasing",
)
logger.info("Phasing alleles finished")
def MakeSecondaryAssortments(cfg, logger, cpu_number, **kargs):
logger.info("Retrieving the Secondary sequence assortments")
in_folder = cfg["input_folder"]
in_suffix = cfg["input_suffix"]
max_size = cfg["limit_large_file"]
filter_taxa = cfg["filtered_taxa_list"]
filter_taxa_occurences = cfg["filtered_taxon_occurences"]
filter_taxa_bool = cfg["filtered_taxon_boolean"]
out_passed = cfg["output_folder_passed"]
create_result_folder(out_passed, logger)
out_failed = cfg["output_folder_failed"]
create_result_folder(out_failed, logger)
temporary_folder = kargs["temp"]
remove_temp = kargs["remove_temp"]
out_files = [
re.split("[L][0-9]+", x)[0] for x in os.listdir(out_passed)
] + [re.split("[L][0-9]+", x)[0] for x in os.listdir(out_failed)]
os.chdir(in_folder)
starting_files = sorted([
x for x in os.listdir(os.getcwd())
if (in_suffix in x and "_core" not in x and "_addit" not in x
and "_temp_" not in x and os.stat(x).st_size != 0
and x not in out_files)
])
if cfg["fasta_sanity_check"]:
logger.debug("Performing sanity check")
for fasta in starting_files:
primary.is_fasta(fasta)
logger.debug("Searching for large fasta files")
starting_fasta = "fasta files:" + ",".join(starting_files)
logger.debug(starting_fasta)
filtering_modes = "filtering_mode_size: {0}, filtering_mode_taxa: {1}".format(
cfg["filtering_mode_size"], cfg["filtering_mode_taxa"])
logger.debug(filtering_modes)
if cfg["filtering_mode_size"] and not cfg["filtering_mode_taxa"]:
large_fastas = [
x for x in starting_files
if not secondary.get_large_files(x, max_size + 1)
]
elif cfg["filtering_mode_taxa"] and not cfg["filtering_mode_size"]:
large_fastas = [
x for x in starting_files if not secondary.filter_with_taxa(
x, filter_taxa, filter_taxa_occurences, filter_taxa_bool)
]
elif cfg["filtering_mode_taxa"] and cfg["filtering_mode_size"]:
large_fastas = [
x for x in starting_files
if (not secondary.filter_with_taxa(
x, filter_taxa, filter_taxa_occurences, filter_taxa_bool)
and not secondary.get_large_files(x, max_size + 1))
]
else:
large_fastas = []
small_fastas = list(set(starting_files).difference(set(large_fastas)))
for x in small_fastas:
try:
shutil.copy(x, out_passed)
except:
print("[Error]: unable to move file {0} to folder {1}".format(
x, out_passed))
logger.info(
"There are {} small fasta files that have been validated".format(
len(small_fastas)))
logger.info(
"There are {} large fasta files that need to be processed".format(
len(large_fastas)))
if cfg["proceed_with_secondary_search"] and large_fastas:
manager = Manager()
fastas = manager.Queue()
result_dict = manager.dict()
process_future_fasta(
secondary.mk_secondary,
large_fastas,
result_dict,
fastas,
cpu_number,
logger,
cfg,
temp=temporary_folder,
tqdm_desc="2nd search",
remove_temp=remove_temp,
)
logger.debug("Done MakeSecondaryAssortments")
def pipeline_STEP1(cfg, logger, cpu_number):
s = "Working with STEP 1: running MAFFT"
print(s)
logger.info(s)
input_folder = cfg["input_folder"]
output_folder = cfg["output_folder"]
out_suffix = cfg["output_suffix"]
create_result_folder(output_folder, logger)
os.chdir(input_folder)
unwanted_files = [
x for x in os.listdir(os.getcwd()) if "_temp_aligned.fasta" in x
]
# Remove files from a previously aborted run .
for f in unwanted_files:
os.remove(f)
finished_files = [
x.split(out_suffix)[0] for x in os.listdir(output_folder)
]
starting_files = sorted([
x for x in os.listdir(os.getcwd()) if cfg["input_suffix"] in x
and find_n_seqs(x, 1, cfg["upper_sequence_limit"]) and "_core" not in x
and "_addit" not in x and x.split(out_suffix)[0] not in finished_files
])
for fasta in starting_files:
is_fasta(fasta)
# Size threshold for switching from small_fastas alignment to large_fastas
size_threshold = cfg["MAFFT_upper_limit_addfragments"]
small_fastas = [
x for x in starting_files if find_n_seqs(x, 1, size_threshold + 1)
]
logger.debug("there are {} small_fastas".format(len(small_fastas)))
large_fastas = [
x for x in starting_files if not find_n_seqs(x, 1, size_threshold + 1)
]
logger.debug("there are {} large_fastas".format(len(large_fastas)))
if small_fastas:
manager = Manager()
fastas = manager.Queue()
result_dict = manager.dict()
logger.info("Running Mafft on small fasta")
process_future_fasta(
STEP1.run_MAFFT_small,
small_fastas,
result_dict,
fastas,
cpu_number,
logger,
cfg,
tqdm_desc="Mafft on small files",
)
if large_fastas:
manager = Manager()
fastas = manager.Queue()
result_dict = manager.dict()
logger.info("Running Mafft on large fasta")
process_future_fasta(
STEP1.run_MAFFT_large,
large_fastas,
result_dict,
fastas,
cpu_number,
logger,
cfg,
tqdm_desc="Mafft on large files",
)
logger.info("STEP 1 finished")