def standard(**kwargs):
"""
Protocol:
Infer ECs from alignment using plmc. Use complex protocol
for heteromultimeric complexes instead.
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required
and infer_plmc()
Returns
-------
outcfg : dict
Output configuration of the pipeline, including
the following fields:
raw_ec_file
model_file
num_sites
num_sequences
effective_sequences
focus_mode (passed through)
focus_sequence (passed through)
segments (passed through)
"""
# for additional required parameters, see infer_plmc()
check_required(kwargs, [
"prefix",
"min_sequence_distance",
])
prefix = kwargs["prefix"]
# infer ECs and load them
outcfg, ecs, segments = infer_plmc(**kwargs)
model = CouplingsModel(outcfg["model_file"])
# following computations are mostly specific to monomer pipeline
is_single_segment = segments is None or len(segments) == 1
outcfg = {
**outcfg,
**_postprocess_inference(ecs,
kwargs,
model,
outcfg,
prefix,
generate_enrichment=is_single_segment,
generate_line_plot=is_single_segment)
}
# dump output config to YAML file for debugging/logging
write_config_file(prefix + ".couplings_standard.outcfg", outcfg)
return outcfg
def jackhmmer_search(**kwargs):
"""
Protocol:
Iterative jackhmmer search against a sequence database.
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required
.. todo::
explain meaning of parameters in detail.
Returns
-------
outcfg : dict
Output configuration of the protocol, including
the following fields:
* sequence_id (passed through from input)
* first_index (passed through from input)
* target_sequence_file
* sequence_file
* raw_alignment_file
* hittable_file
* focus_mode
* focus_sequence
* segments
"""
check_required(kwargs, [
"prefix", "sequence_id", "sequence_file", "sequence_download_url",
"region", "first_index", "use_bitscores", "domain_threshold",
"sequence_threshold", "database", "iterations", "cpu", "nobias",
"reuse_alignment", "checkpoints_hmm", "checkpoints_ali", "jackhmmer",
"extract_annotation"
])
prefix = kwargs["prefix"]
# make sure output directory exists
create_prefix_folders(prefix)
# store search sequence file here
target_sequence_file = prefix + ".fa"
full_sequence_file = prefix + "_full.fa"
# make sure search sequence is defined and load it
full_seq_file, (full_seq_id, full_seq) = fetch_sequence(
kwargs["sequence_id"], kwargs["sequence_file"],
kwargs["sequence_download_url"], full_sequence_file)
# cut sequence to target region and save in sequence_file
# (this is the main sequence file used downstream)
(region_start, region_end), cut_seq = cut_sequence(full_seq,
kwargs["sequence_id"],
kwargs["region"],
kwargs["first_index"],
target_sequence_file)
# run jackhmmer... allow to reuse pre-exisiting
# Stockholm alignment file here
ali_outcfg_file = prefix + ".align_jackhmmer_search.outcfg"
# determine if to rerun, only possible if previous results
# were stored in ali_outcfg_file
if kwargs["reuse_alignment"] and valid_file(ali_outcfg_file):
ali = read_config_file(ali_outcfg_file)
# check if the alignment file itself is also there
verify_resources(
"Tried to reuse alignment, but empty or "
"does not exist", ali["alignment"], ali["domtblout"])
else:
# otherwise, we have to run the alignment
# modify search thresholds to be suitable for jackhmmer
seq_threshold, domain_threshold = search_thresholds(
kwargs["use_bitscores"], kwargs["sequence_threshold"],
kwargs["domain_threshold"], len(cut_seq))
# run search process
ali = at.run_jackhmmer(
query=target_sequence_file,
database=kwargs[kwargs["database"]],
prefix=prefix,
use_bitscores=kwargs["use_bitscores"],
domain_threshold=domain_threshold,
seq_threshold=seq_threshold,
iterations=kwargs["iterations"],
nobias=kwargs["nobias"],
cpu=kwargs["cpu"],
checkpoints_hmm=kwargs["checkpoints_hmm"],
checkpoints_ali=kwargs["checkpoints_ali"],
binary=kwargs["jackhmmer"],
)
# get rid of huge stdout log file immediately
# (do not use /dev/null option of jackhmmer function
# to make no assumption about operating system)
try:
os.remove(ali.output)
except OSError:
pass
# turn namedtuple into dictionary to make
# restarting code nicer
ali = dict(ali._asdict())
# save results of search for possible restart
write_config_file(ali_outcfg_file, ali)
# prepare output dictionary with result files
outcfg = {
"sequence_id": kwargs["sequence_id"],
"target_sequence_file": target_sequence_file,
"sequence_file": full_sequence_file,
"first_index": kwargs["first_index"],
"focus_mode": True,
"raw_alignment_file": ali["alignment"],
"hittable_file": ali["domtblout"],
}
# define a single protein segment based on target sequence
outcfg["segments"] = [
Segment("aa", kwargs["sequence_id"], region_start, region_end,
range(region_start, region_end + 1)).to_list()
]
outcfg["focus_sequence"] = "{}/{}-{}".format(kwargs["sequence_id"],
region_start, region_end)
return outcfg
def existing(**kwargs):
"""
Protocol:
Use external sequence alignment and extract all relevant
information from there (e.g. sequence, region, etc.),
then apply gap & fragment filtering as usual
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required
Returns
-------
outcfg : dict
Output configuration of the pipeline, including
the following fields:
* sequence_id (passed through from input)
* alignment_file
* raw_focus_alignment_file
* statistics_file
* sequence_file
* first_index
* target_sequence_file
* annotation_file (None)
* frequencies_file
* identities_file
* focus_mode
* focus_sequence
* segments
"""
check_required(kwargs, [
"prefix", "input_alignment", "sequence_id", "first_index",
"extract_annotation"
])
prefix = kwargs["prefix"]
# make sure output directory exists
create_prefix_folders(prefix)
# this file is starting point of pipeline;
# check if input alignment actually exists
input_alignment = kwargs["input_alignment"]
verify_resources("Input alignment does not exist", input_alignment)
# first try to autodetect format of alignment
with open(input_alignment) as f:
format = detect_format(f)
if format is None:
raise InvalidParameterError(
"Format of input alignment {} could not be "
"automatically detected.".format(input_alignment))
with open(input_alignment) as f:
ali_raw = Alignment.from_file(f, format)
# save annotation in sequence headers (species etc.)
annotation_file = None
if kwargs["extract_annotation"]:
annotation_file = prefix + "_annotation.csv"
from_anno_line = (format == "stockholm")
annotation = extract_header_annotation(ali_raw,
from_annotation=from_anno_line)
annotation.to_csv(annotation_file, index=False)
# Target sequence of alignment
sequence_id = kwargs["sequence_id"]
if sequence_id is None:
raise InvalidParameterError("Parameter sequence_id must be defined")
# First, find focus sequence in alignment
focus_index = None
for i, id_ in enumerate(ali_raw.ids):
if id_.startswith(sequence_id):
focus_index = i
break
# if we didn't find it, cannot continue
if focus_index is None:
raise InvalidParameterError(
"Target sequence {} could not be found in alignment".format(
sequence_id))
# identify what columns (non-gap) to keep for focus
focus_seq = ali_raw[focus_index]
focus_cols = np.array([
c not in [ali_raw._match_gap, ali_raw._insert_gap] for c in focus_seq
])
# extract focus alignment
focus_ali = ali_raw.select(columns=focus_cols)
focus_seq_nogap = "".join(focus_ali[focus_index])
# determine region of sequence. If first_index is given,
# use that in any case, otherwise try to autodetect
full_focus_header = ali_raw.ids[focus_index]
focus_id = full_focus_header.split()[0]
# try to extract region from sequence header
id_, region_start, region_end = parse_header(focus_id)
# override with first_index if given
if kwargs["first_index"] is not None:
region_start = kwargs["first_index"]
region_end = region_start + len(focus_seq_nogap) - 1
if region_start is None or region_end is None:
raise InvalidParameterError(
"Could not extract region information " +
"from sequence header {} ".format(full_focus_header) +
"and first_index parameter is not given.")
# resubstitute full sequence ID from identifier
# and region information
header = "{}/{}-{}".format(id_, region_start, region_end)
focus_ali.ids[focus_index] = header
# write target sequence to file
target_sequence_file = prefix + ".fa"
with open(target_sequence_file, "w") as f:
write_fasta([(header, focus_seq_nogap)], f)
# apply sequence identity and fragment filters,
# and gap threshold
mod_outcfg, ali = modify_alignment(focus_ali, focus_index, id_,
region_start, **kwargs)
# generate output configuration of protocol
outcfg = {
**mod_outcfg,
"sequence_id": sequence_id,
"sequence_file": target_sequence_file,
"first_index": region_start,
"target_sequence_file": target_sequence_file,
"focus_sequence": header,
"focus_mode": True,
}
if annotation_file is not None:
outcfg["annotation_file"] = annotation_file
# dump config to YAML file for debugging/logging
write_config_file(prefix + ".align_existing.outcfg", outcfg)
# return results of protocol
return outcfg
def complex(**kwargs):
"""
Protocol:
Run monomer alignment protocol and postprocess it for
EVcomplex calculations
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required
Returns
-------
outcfg : dict
Output configuration of the alignment protocol, and
the following additional field:
genome_location_file : path to file containing
the genomic locations for CDs's corresponding to
identifiers in the alignment.
"""
check_required(kwargs, [
"prefix", "alignment_protocol", "uniprot_to_embl_table",
"ena_genome_location_table"
])
verify_resources("Uniprot to EMBL mapping table does not exist",
kwargs["uniprot_to_embl_table"])
verify_resources("ENA genome location table does not exist",
kwargs["ena_genome_location_table"])
prefix = kwargs["prefix"]
# make sure output directory exists
create_prefix_folders(prefix)
# run the regular alignment protocol
# (standard, existing, ...)
alignment_protocol = kwargs["alignment_protocol"]
if alignment_protocol not in PROTOCOLS:
raise InvalidParameterError(
"Invalid choice for alignment protocol: {}".format(
alignment_protocol))
outcfg = PROTOCOLS[kwargs["alignment_protocol"]](**kwargs)
# if the user selected the existing alignment protocol
# they can supply an input annotation file
# which overwrites the annotation file generated by the existing protocol
if alignment_protocol == "existing":
check_required(kwargs, ["override_annotation_file"])
if kwargs["override_annotation_file"] is not None:
verify_resources("Override annotation file does not exist",
kwargs["override_annotation_file"])
outcfg["annotation_file"] = prefix + "_annotation.csv"
annotation_data = pd.read_csv(kwargs["override_annotation_file"])
annotation_data.to_csv(outcfg["annotation_file"])
# extract cds identifiers for alignment uniprot IDs
cds_ids = extract_cds_ids(outcfg["alignment_file"],
kwargs["uniprot_to_embl_table"])
# extract genome location information from ENA
genome_location_filename = prefix + "_genome_location.csv"
genome_location_table = extract_embl_annotation(
cds_ids, kwargs["ena_genome_location_table"], genome_location_filename)
genome_location_table = add_full_header(genome_location_table,
outcfg["alignment_file"])
genome_location_table.to_csv(genome_location_filename)
outcfg["genome_location_file"] = genome_location_filename
# dump output config to YAML file for debugging/logging
write_config_file(prefix + ".align_complex.outcfg", outcfg)
return outcfg
def standard(**kwargs):
"""
Protocol:
Standard buildali4 workflow (run iterative jackhmmer
search against sequence database, than determine which
sequences and columns to include in the calculation based
on coverage and maximum gap thresholds).
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required
Returns
-------
outcfg : dict
Output configuration of the pipeline, including
the following fields:
* sequence_id (passed through from input)
* first_index (passed through from input)
* alignment_file
* raw_alignment_file
* raw_focus_alignment_file
* statistics_file
* target_sequence_file
* sequence_file
* annotation_file
* frequencies_file
* identities_file
* hittable_file
* focus_mode
* focus_sequence
* segments
ali : Alignment
Final sequence alignment
"""
check_required(kwargs, [
"prefix",
"extract_annotation",
])
prefix = kwargs["prefix"]
# make sure output directory exists
create_prefix_folders(prefix)
# first step of protocol is to get alignment using
# jackhmmer; initialize output configuration with
# results of this search
jackhmmer_outcfg = jackhmmer_search(**kwargs)
stockholm_file = jackhmmer_outcfg["raw_alignment_file"]
segment = Segment.from_list(jackhmmer_outcfg["segments"][0])
target_seq_id = segment.sequence_id
region_start = segment.region_start
region_end = segment.region_end
# read in stockholm format (with full annotation)
with open(stockholm_file) as a:
ali_raw = Alignment.from_file(a, "stockholm")
# and store as FASTA file first (disabled for now
# since equivalent information easily be obtained
# from Stockholm file
"""
ali_raw_fasta_file = prefix + "_raw.fasta"
with open(ali_raw_fasta_file, "w") as f:
ali_raw.write(f, "fasta")
"""
# save annotation in sequence headers (species etc.)
if kwargs["extract_annotation"]:
annotation_file = prefix + "_annotation.csv"
annotation = extract_header_annotation(ali_raw)
annotation.to_csv(annotation_file, index=False)
# center alignment around focus/search sequence
focus_cols = np.array([c != "-" for c in ali_raw[0]])
focus_ali = ali_raw.select(columns=focus_cols)
target_seq_index = 0
mod_outcfg, ali = modify_alignment(focus_ali, target_seq_index,
target_seq_id, region_start, **kwargs)
# merge results of jackhmmer_search and modify_alignment stage
outcfg = {
**jackhmmer_outcfg,
**mod_outcfg, "annotation_file": annotation_file
}
# dump output config to YAML file for debugging/logging
write_config_file(prefix + ".align_standard.outcfg", outcfg)
# return results of protocol
return outcfg
def hmmbuild_and_search(**kwargs):
"""
Protocol:
Build HMM from sequence alignment using hmmbuild and
search against a sequence database using hmmsearch.
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required
Returns
-------
outcfg : dict
Output configuration of the protocol, including
the following fields:
* target_sequence_file
* sequence_file
* raw_alignment_file
* hittable_file
* focus_mode
* focus_sequence
* segments
"""
def _format_alignment_for_hmmbuild(input_alignment_file, **kwargs):
# this file is starting point of pipeline;
# check if input alignment actually exists
verify_resources("Input alignment does not exist",
input_alignment_file)
# first try to autodetect format of alignment
with open(input_alignment_file) as f:
format = detect_format(f)
if format is None:
raise InvalidParameterError(
"Format of input alignment {} could not be "
"automatically detected.".format(input_alignment_file))
with open(input_alignment_file) as f:
ali_raw = Alignment.from_file(f, format)
# Target sequence of alignment
sequence_id = kwargs["sequence_id"]
if sequence_id is None:
raise InvalidParameterError(
"Parameter sequence_id must be defined")
# First, find focus sequence in alignment
focus_index = None
for i, id_ in enumerate(ali_raw.ids):
if id_.startswith(sequence_id):
focus_index = i
break
# if we didn't find it, cannot continue
if focus_index is None:
raise InvalidParameterError(
"Target sequence {} could not be found in alignment".format(
sequence_id))
# identify what columns (non-gap) to keep for focus
# this should be all columns in the raw_focus_alignment_file
# but checking anyway
focus_seq = ali_raw[focus_index]
focus_cols = np.array([
c not in [ali_raw._match_gap, ali_raw._insert_gap]
for c in focus_seq
])
# extract focus alignment
focus_ali = ali_raw.select(columns=focus_cols)
focus_seq_nogap = "".join(focus_ali[focus_index])
# determine region of sequence. If first_index is given,
# use that in any case, otherwise try to autodetect
full_focus_header = ali_raw.ids[focus_index]
focus_id = full_focus_header.split()[0]
# try to extract region from sequence header
id_, region_start, region_end = parse_header(focus_id)
# override with first_index if given
if kwargs["first_index"] is not None:
region_start = kwargs["first_index"]
region_end = region_start + len(focus_seq_nogap) - 1
if region_start is None or region_end is None:
raise InvalidParameterError(
"Could not extract region information " +
"from sequence header {} ".format(full_focus_header) +
"and first_index parameter is not given.")
# resubstitute full sequence ID from identifier
# and region information
header = "{}/{}-{}".format(id_, region_start, region_end)
focus_ali.ids[focus_index] = header
# write target sequence to file
target_sequence_file = prefix + ".fa"
with open(target_sequence_file, "w") as f:
write_fasta([(header, focus_seq_nogap)], f)
# swap target sequence to first position if it is not
# the first sequence in alignment;
# this is particularly important for hhfilter run
# because target sequence might otherwise be filtered out
if focus_index != 0:
indices = np.arange(0, len(focus_ali))
indices[0] = focus_index
indices[focus_index] = 0
focus_index = 0
focus_ali = focus_ali.select(sequences=indices)
# write the raw focus alignment for hmmbuild
focus_fasta_file = prefix + "_raw_focus_input.fasta"
with open(focus_fasta_file, "w") as f:
focus_ali.write(f, "fasta")
return focus_fasta_file, target_sequence_file, region_start, region_end
# define the gap threshold for inclusion in HMM's build by HMMbuild.
SYMFRAC_HMMBUILD = 0.0
# check for required options
check_required(kwargs, [
"prefix", "sequence_id", "alignment_file", "use_bitscores",
"domain_threshold", "sequence_threshold", "database", "cpu", "nobias",
"reuse_alignment", "hmmbuild", "hmmsearch"
])
prefix = kwargs["prefix"]
# make sure output directory exists
create_prefix_folders(prefix)
# prepare input alignment for hmmbuild
focus_fasta_file, target_sequence_file, region_start, region_end = \
_format_alignment_for_hmmbuild(
kwargs["alignment_file"], **kwargs
)
# run hmmbuild_and_search... allow to reuse pre-exisiting
# Stockholm alignment file here
ali_outcfg_file = prefix + ".align_hmmbuild_and_search.outcfg"
# determine if to rerun, only possible if previous results
# were stored in ali_outcfg_file
if kwargs["reuse_alignment"] and valid_file(ali_outcfg_file):
ali = read_config_file(ali_outcfg_file)
# check if the alignment file itself is also there
verify_resources(
"Tried to reuse alignment, but empty or "
"does not exist", ali["alignment"], ali["domtblout"])
else:
# otherwise, we have to run the alignment
# modify search thresholds to be suitable for hmmsearch
sequence_length = region_end - region_start + 1
seq_threshold, domain_threshold = search_thresholds(
kwargs["use_bitscores"], kwargs["sequence_threshold"],
kwargs["domain_threshold"], sequence_length)
# create the hmm
hmmbuild_result = at.run_hmmbuild(
alignment_file=focus_fasta_file,
prefix=prefix,
symfrac=SYMFRAC_HMMBUILD,
cpu=kwargs["cpu"],
binary=kwargs["hmmbuild"],
)
hmmfile = hmmbuild_result.hmmfile
# run the alignment from the hmm
ali = at.run_hmmsearch(
hmmfile=hmmfile,
database=kwargs[kwargs["database"]],
prefix=prefix,
use_bitscores=kwargs["use_bitscores"],
domain_threshold=domain_threshold,
seq_threshold=seq_threshold,
nobias=kwargs["nobias"],
cpu=kwargs["cpu"],
binary=kwargs["hmmsearch"],
)
# get rid of huge stdout log file immediately
try:
os.remove(ali.output)
except OSError:
pass
# turn namedtuple into dictionary to make
# restarting code nicer
ali = dict(ali._asdict())
# only item from hmmsearch_result to save is the hmmfile
ali["hmmfile"] = hmmfile
# save results of search for possible restart
write_config_file(ali_outcfg_file, ali)
# prepare output dictionary with result files
outcfg = {
"sequence_file": target_sequence_file,
"first_index": region_start,
"input_raw_focus_alignment": focus_fasta_file,
"target_sequence_file": target_sequence_file,
"focus_mode": True,
"raw_alignment_file": ali["alignment"],
"hittable_file": ali["domtblout"],
}
# convert the raw output alignment to fasta format
# and add the appropriate query sequecne
raw_focus_alignment_file = _make_hmmsearch_raw_fasta(outcfg, prefix)
outcfg["raw_focus_alignment_file"] = raw_focus_alignment_file
# define a single protein segment based on target sequence
outcfg["segments"] = [
Segment("aa", kwargs["sequence_id"], region_start, region_end,
range(region_start, region_end + 1)).to_list()
]
outcfg["focus_sequence"] = "{}/{}-{}".format(kwargs["sequence_id"],
region_start, region_end)
return outcfg
def infer_plmc(**kwargs):
"""
Run EC computation on alignment. This function contains
the functionality shared between monomer and complex EC
inference.
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required
Returns
-------
outcfg : dict
Output configuration of the pipeline, including
the following fields:
raw_ec_file
model_file
num_sites
num_sequences
effective_sequences
focus_mode (passed through)
focus_sequence (passed through)
segments (passed through)
"""
check_required(
kwargs,
[
"prefix", "alignment_file",
"focus_mode", "focus_sequence", "theta",
"alphabet", "segments", "ignore_gaps", "iterations",
"lambda_h", "lambda_J", "lambda_group",
"scale_clusters",
"cpu", "plmc", "reuse_ecs",
]
)
prefix = kwargs["prefix"]
# for now disable option to not save model, since
# otherwise mutate stage will crash. To remove model
# file at end, use delete option in management section.
"""
if kwargs["save_model"]:
model = prefix + ".model"
else:
model = None
"""
model = prefix + ".model"
outcfg = {
"model_file": model,
"raw_ec_file": prefix + "_ECs.txt",
"ec_file": prefix + "_CouplingScores.csv",
# the following are passed through stage...
"focus_mode": kwargs["focus_mode"],
"focus_sequence": kwargs["focus_sequence"],
"segments": kwargs["segments"],
}
# make sure input alignment exists
verify_resources(
"Input alignment does not exist",
kwargs["alignment_file"]
)
# make sure output directory exists
create_prefix_folders(prefix)
# regularization strength on couplings J_ij
lambda_J = kwargs["lambda_J"]
segments = kwargs["segments"]
if segments is not None:
segments = [
mapping.Segment.from_list(s) for s in segments
]
# first determine size of alphabet;
# default is amino acid alphabet
if kwargs["alphabet"] is None:
alphabet = ALPHABET_PROTEIN
alphabet_setting = None
else:
alphabet = kwargs["alphabet"]
# allow shortcuts for protein, DNA, RNA
if alphabet in ALPHABET_MAP:
alphabet = ALPHABET_MAP[alphabet]
# if we have protein alphabet, do not set
# as plmc parameter since default parameter,
# has some implementation advantages for focus mode
if alphabet == ALPHABET_PROTEIN:
alphabet_setting = None
else:
alphabet_setting = alphabet
# scale lambda_J to proportionally compensate
# for higher number of J_ij compared to h_i?
if kwargs["lambda_J_times_Lq"]:
num_symbols = len(alphabet)
# if we ignore gaps, there is one character less
if kwargs["ignore_gaps"]:
num_symbols -= 1
# second, determine number of uppercase positions
# that are included in the calculation
with open(kwargs["alignment_file"]) as f:
seq_id, seq = next(read_fasta(f))
# gap character is by convention first char in alphabet
gap = alphabet[0]
uppercase = [
c for c in seq if c == c.upper() or c == gap
]
L = len(uppercase)
# finally, scale lambda_J
lambda_J *= (num_symbols - 1) * (L - 1)
# run plmc... or reuse pre-exisiting results from previous run
plm_outcfg_file = prefix + ".couplings_standard_plmc.outcfg"
# determine if to rerun, only possible if previous results
# were stored in ali_outcfg_file
if kwargs["reuse_ecs"] and valid_file(plm_outcfg_file):
plmc_result = read_config_file(plm_outcfg_file)
# check if the EC/parameter files are there
required_files = [outcfg["raw_ec_file"]]
if outcfg["model_file"] is not None:
required_files += [outcfg["model_file"]]
verify_resources(
"Tried to reuse ECs, but empty or "
"does not exist",
*required_files
)
else:
# run plmc binary
plmc_result = ct.run_plmc(
kwargs["alignment_file"],
outcfg["raw_ec_file"],
outcfg["model_file"],
focus_seq=kwargs["focus_sequence"],
alphabet=alphabet_setting,
theta=kwargs["theta"],
scale=kwargs["scale_clusters"],
ignore_gaps=kwargs["ignore_gaps"],
iterations=kwargs["iterations"],
lambda_h=kwargs["lambda_h"],
lambda_J=lambda_J,
lambda_g=kwargs["lambda_group"],
cpu=kwargs["cpu"],
binary=kwargs["plmc"],
)
# save iteration table to file
iter_table_file = prefix + "_iteration_table.csv"
plmc_result.iteration_table.to_csv(
iter_table_file
)
# turn namedtuple into dictionary to make
# restarting code nicer
plmc_result = dict(plmc_result._asdict())
# then replace table with filename so
# we can store results in config file
plmc_result["iteration_table"] = iter_table_file
# save results of search for possible restart
write_config_file(plm_outcfg_file, plmc_result)
# store useful information about model in outcfg
outcfg.update({
"num_sites": plmc_result["num_valid_sites"],
"num_valid_sequences": plmc_result["num_valid_seqs"],
"effective_sequences": plmc_result["effective_samples"],
"region_start": plmc_result["region_start"],
})
# read and sort ECs
ecs = pairs.read_raw_ec_file(outcfg["raw_ec_file"])
if segments is not None:
# create index mapping
seg_mapper = mapping.SegmentIndexMapper(
kwargs["focus_mode"], outcfg["region_start"], *segments
)
# apply to EC table
ecs = mapping.segment_map_ecs(ecs, seg_mapper)
return outcfg, ecs, segments
def mean_field(**kwargs):
"""
Protocol:
Infer ECs from alignment using mean field direct coupling analysis.
For now, mean field DCA can only be run in focus mode, gaps
included.
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required.
Returns
-------
outcfg : dict
Output configuration of the pipeline, including
the following fields:
* raw_ec_file
* model_file
* num_sites
* num_sequences
* effective_sequences
* focus_mode (passed through)
* focus_sequence (passed through)
* segments (passed through)
"""
check_required(
kwargs,
[
"prefix", "alignment_file", "segments",
"focus_mode", "focus_sequence", "theta",
"pseudo_count", "alphabet",
"min_sequence_distance", # "save_model",
]
)
if not kwargs["focus_mode"]:
raise InvalidParameterError(
"For now, mean field DCA can only be run in focus mode."
)
prefix = kwargs["prefix"]
# option to save model disabled
"""
if kwargs["save_model"]:
model = prefix + ".model"
else:
model = None
"""
model = prefix + ".model"
outcfg = {
"model_file": model,
"raw_ec_file": prefix + "_ECs.txt",
"ec_file": prefix + "_CouplingScores.csv",
# TODO: the following are passed through stage...
# keep this or unnecessary?
"focus_mode": kwargs["focus_mode"],
"focus_sequence": kwargs["focus_sequence"],
"segments": kwargs["segments"],
}
# make sure input alignment exists
alignment_file = kwargs["alignment_file"]
verify_resources(
"Input alignment does not exist",
kwargs["alignment_file"]
)
# make sure output directory exists
create_prefix_folders(prefix)
segments = kwargs["segments"]
if segments is not None:
segments = [
mapping.Segment.from_list(s) for s in segments
]
# determine alphabet
# default is protein
if kwargs["alphabet"] is None:
alphabet = ALPHABET_PROTEIN
else:
alphabet = kwargs["alphabet"]
# allow shortcuts for protein, DNA, RNA
if alphabet in ALPHABET_MAP:
alphabet = ALPHABET_MAP[alphabet]
# read in a2m alignment
with open(alignment_file) as f:
input_alignment = Alignment.from_file(
f, alphabet=alphabet,
format="fasta"
)
# init mean field direct coupling analysis
mf_dca = MeanFieldDCA(input_alignment)
# run mean field approximation
model = mf_dca.fit(
theta=kwargs["theta"],
pseudo_count=kwargs["pseudo_count"]
)
# write ECs to file
model.to_raw_ec_file(
outcfg["raw_ec_file"]
)
# write model file
if outcfg["model_file"] is not None:
model.to_file(
outcfg["model_file"],
file_format="plmc_v2"
)
# store useful information about model in outcfg
outcfg.update({
"num_sites": model.L,
"num_valid_sequences": model.N_valid,
"effective_sequences": float(round(model.N_eff, 1)),
"region_start": int(model.index_list[0]),
})
# read and sort ECs
ecs = pd.read_csv(
outcfg["raw_ec_file"], sep=" ",
# for now, call the last two columns
# "fn" and "cn" to prevent compare
# stage from crashing
names=["i", "A_i", "j", "A_j", "fn", "cn"]
# names=["i", "A_i", "j", "A_j", "mi", "di"]
).sort_values(
by="cn",
ascending=False
)
is_single_segment = segments is None or len(segments) == 1
outcfg = {
**outcfg,
**_postprocess_inference(
ecs, kwargs, model, outcfg, prefix,
generate_enrichment=is_single_segment,
generate_line_plot=is_single_segment
)
}
# dump output config to YAML file for debugging/logging
write_config_file(prefix + ".couplings_meanfield.outcfg", outcfg)
return outcfg
def complex(**kwargs):
"""
Protocol:
Infer ECs for protein complexes from alignment using plmc.
Allows user to select scoring protocol.
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required
and infer_plmc()
Returns
-------
outcfg : dict
Output configuration of the pipeline, including
the following fields:
raw_ec_file
model_file
num_sites
num_sequences
effective_sequences
focus_mode (passed through)
focus_sequence (passed through)
segments (passed through)
"""
# for additional required parameters, see infer_plmc()
check_required(
kwargs,
[
"prefix", "min_sequence_distance",
"scoring_model", "use_all_ecs_for_scoring",
]
)
prefix = kwargs["prefix"]
# infer ECs and load them
outcfg, ecs, segments = infer_plmc(**kwargs)
model = CouplingsModel(outcfg["model_file"])
# following computations are mostly specific to complex pipeline
# add mixture model probability
if kwargs["scoring_model"] in SCORING_MODELS:
if kwargs["use_all_ecs_for_scoring"] is not None:
use_all_ecs = kwargs["use_all_ecs_for_scoring"]
else:
use_all_ecs = False
ecs = complex_probability(
ecs, kwargs["scoring_model"], use_all_ecs
)
else:
raise InvalidParameterError(
"Invalid scoring_model parameter: " +
"{}. Valid options are: {}".format(
kwargs["protocol"], ", ".join(SCORING_MODELS)
)
)
# also create line-drawing script (for multiple chains)
# by convention, we map first segment to chain A,
# second to B, a.s.f.
chain_mapping = dict(
zip(
[s.segment_id for s in segments],
string.ascii_uppercase,
)
)
outcfg = {
**outcfg,
**_postprocess_inference(
ecs, kwargs, model, outcfg, prefix,
generate_line_plot=True,
generate_enrichment=False,
ec_filter="segment_i != segment_j or abs(i - j) >= {}",
chain=chain_mapping
)
}
# save just the inter protein ECs
## TODO: eventually have this accomplished by _postprocess_inference
## right now avoiding a second call with a different ec_filter
ecs = pd.read_csv(outcfg["ec_file"])
outcfg["inter_ec_file"] = prefix + "_CouplingScores_inter.csv"
inter_ecs = ecs.query("segment_i != segment_j")
inter_ecs.to_csv(outcfg["inter_ec_file"], index=False)
# dump output config to YAML file for debugging/logging
write_config_file(prefix + ".couplings_complex.outcfg", outcfg)
# TODO: make the following complex-ready
# EC enrichment:
#
# 1) think about making EC enrichment complex-ready and add
# it back here - so far it only makes sense if all ECs are
# on one segment
#
# EVzoom:
#
# 1) at the moment, EVzoom will use numbering before remapping
# we should eventually get this to a point where segments + residue
# index are displayed on EVzoom
#
# 2) note that this will currently use the default mixture model
# selection for determining the EC cutoff, rather than the selection
# used for the EC table above
return outcfg
def execute(**config):
"""
Execute a pipeline configuration
Parameters
----------
**config
Input configuration for pipeline
(see pipeline config files for
example of how this should look like)
Returns
-------
global_state : dict
Global output state of pipeline
"""
check_required(config, ["pipeline", "stages", "global"])
# check if valid pipeline was selected
if config["pipeline"] not in PIPELINES:
raise InvalidParameterError("Not a valid pipeline selection. "
"Valid choices are:\n{}".format(", ".join(
PIPELINES.keys())))
stages = config["stages"]
if stages is None:
raise InvalidParameterError("No stages defined, need at least one.")
# get definition of selected pipeline
pipeline = PIPELINES[config["pipeline"]]
prefix = config["global"]["prefix"]
# make sure output directory exists
create_prefix_folders(prefix)
# this is the global state of results as
# we move through different stages of
# the pipeline
global_state = config["global"]
# keep track of how many stages are still
# to be run, so we can leave out stages at
# the end of workflow below
num_stages_to_run = len(stages)
# get job tracker
tracker = get_result_tracker(config)
# set job status to running and also initalize global state
tracker.update(status=EStatus.RUN, results=global_state)
# iterate through individual stages
for (stage, runner, key_prefix) in pipeline:
# check if anything else is left to
# run, otherwise skip
if num_stages_to_run == 0:
break
# check if config for stage is there
check_required(config, [stage])
# output files for stage into an individual folder
stage_prefix = insert_dir(prefix, stage)
create_prefix_folders(stage_prefix)
# config files for input and output of stage
stage_incfg = "{}_{}.incfg".format(stage_prefix, stage)
stage_outcfg = "{}_{}.outcfg".format(stage_prefix, stage)
# update current stage of job
tracker.update(stage=stage)
# check if stage should be executed
if stage in stages:
# global state inserted at end, overrides any
# stage-specific settings (except for custom prefix)
incfg = {
**config["tools"],
**config["databases"],
**config[stage],
**global_state, "prefix": stage_prefix
}
# save input of stage in config file
write_config_file(stage_incfg, incfg)
# run stage
outcfg = runner(**incfg)
# prefix output keys if this parameter is
# given in stage configuration, to avoid
# name clashes if same protocol run multiple times
if key_prefix is not None:
outcfg = {key_prefix + k: v for k, v in outcfg.items()}
# save output of stage in config file
write_config_file(stage_outcfg, outcfg)
# one less stage to put through after we ran this...
num_stages_to_run -= 1
else:
# skip state by injecting state from previous run
verify_resources(
"Trying to skip, but output configuration "
"for stage '{}' does not exist. Has it already "
"been run?".format(stage, stage), stage_outcfg)
# read output configuration
outcfg = read_config_file(stage_outcfg)
# verify all the output files are there
outfiles = [
filepath for f, filepath in outcfg.items()
if f.endswith("_file") and filepath is not None
]
verify_resources(
"Output files from stage '{}' "
"missing".format(stage), *outfiles)
# update global state with outputs of stage
global_state = {**global_state, **outcfg}
# update state in tracker accordingly
tracker.update(results=outcfg)
# create results archive
archive_file = create_archive(config, global_state, prefix)
# only store results archive if a result file was created
if archive_file is not None:
global_state["archive_file"] = archive_file
# prepare update for tracker, but only store in last
# go when job is set to done
tracker_archive_update = {"archive_file": archive_file}
else:
tracker_archive_update = None
# set job status to done and transfer archive if selected for syncing
tracker.update(status=EStatus.DONE, results=tracker_archive_update)
# delete selected output files if requested;
# tracker does not need to update here since it won't
# sync entries of delete list in the first place
global_state = delete_outputs(config, global_state)
# write final global state of pipeline
write_config_file(prefix + FINAL_CONFIG_SUFFIX, global_state)
return global_state
def run_jobs(configs,
global_config,
overwrite=False,
workdir=None,
abort_on_error=True,
environment=None):
"""
Submit config to pipeline
Parameters
----------
configs : dict
Configurations for individual subjobs
global_config : dict
Master configuration (if only one job,
the contents of this dictionary will be
equal to the single element of config_files)
overwrite : bool, optional (default: False)
If True, allows overwriting previous run of the same
config, otherwise will fail if results from previous
execution are present
workdir : str, optional (default: None)
Workdir in which to run job (will combine
workdir and prefix in joint path)
abort_on_error : bool, optional (default: True)
Abort entire job submission if error occurs for
one of the jobs by propagating RuntimeError
environment : str, optional (default: None)
Allow to pass value for environment parameter
of submitter, will override environment.configuration
from global_config (e.g., for setting environment
variables like passwords)
Returns
-------
job_ids : dict
Mapping from subjob prefix (keys in configs parameter)
to identifier returned by submitter for each of the jobs
that was *successfully* submitted (i.e. missing keys from
configs param indicate these jobs could not be submitted).
Raises
------
RuntimeError
If error encountered during submission and abort_on_error
is True
"""
cmd_base = environ.get("EVCOUPLINGS_RUNCFG_APP") or "evcouplings_runcfg"
summ_base = environ.get(
"EVCOUPLINGS_SUMMARIZE_APP") or "evcouplings_summarize"
# determine output directory for config files
prefix = global_config["global"]["prefix"]
# integrate working directory into output prefix
# if it is given; if prefix contains an absolute path,
# this will override the workdir according to
# implementation of path.join()
if workdir is not None:
out_prefix = path.join(workdir, prefix)
else:
out_prefix = prefix
# save configuration file, make sure we do not overwrite previous run
# if overwrite protection is activated
# (but only if it is a valid configuration file with contents)
cfg_filename = CONFIG_NAME.format(out_prefix)
if not overwrite and valid_file(cfg_filename):
raise InvalidParameterError(
"Existing configuration file {} ".format(cfg_filename) +
"indicates current prefix {} ".format(prefix) +
"would overwrite existing results. Use --yolo " +
"flag to deactivate overwrite protection (e.g. for "
"restarting a job or running a different stage).")
# make sure working directory exists
create_prefix_folders(cfg_filename)
# write global config file
write_config_file(cfg_filename, global_config)
# also write individual subjob configuration files
# (we have to write these before submitting, since
# the job summarizer needs the paths to all files)
for subjob_prefix, subjob_cfg in configs.items():
# determine working dir for each subjob, since subjob
# prefix may contain slashes leading to subfolder creation
if workdir is not None:
subjob_out_prefix = path.join(workdir, subjob_prefix)
else:
subjob_out_prefix = subjob_prefix
subcfg_filename = CONFIG_NAME.format(subjob_out_prefix)
# make sure output subfolder exists
create_prefix_folders(subcfg_filename)
# write subjob configuration file
write_config_file(subcfg_filename, subjob_cfg)
# now create list of subjob config files relative to working
# directory (above, we allow to run submitted in arbitrary directory)
config_files = [
CONFIG_NAME.format(subjob_prefix) for subjob_prefix in configs
]
# create command for summarizer (needs to know all subjob config files)
summ_cmd = "{} {} {} {}".format(summ_base, global_config["pipeline"],
global_config["global"]["prefix"],
" ".join(config_files))
# create submitter from global (pre-unrolling) configuration
submitter = utils.SubmitterFactory(global_config["environment"]["engine"],
db_path=out_prefix +
"_job_database.txt")
# collect individual submitted jobs here
commands = []
# record subjob IDs returned by submitter for each job
job_ids = {}
# prepare individual jobs for submission
for job, job_cfg in configs.items():
job_prefix = job_cfg["global"]["prefix"]
job_cfg_file = CONFIG_NAME.format(job)
# create submission command
env = job_cfg["environment"]
cmd = utils.Command(
["{} {}".format(cmd_base, job_cfg_file), summ_cmd],
name=job_prefix,
environment=environment or env["configuration"],
workdir=workdir,
resources={
utils.EResource.queue: env["queue"],
utils.EResource.time: env["time"],
utils.EResource.mem: env["memory"],
utils.EResource.nodes: env["cores"],
utils.EResource.out: job_prefix + "_stdout.log",
utils.EResource.error: job_prefix + "_stderr.log",
})
# store job for later dependency creation
commands.append(cmd)
tracker = get_result_tracker(job_cfg)
try:
# finally, submit job
current_job_id = submitter.submit(cmd)
# store run identifier returned by submitter
# TODO: consider storing current_job_id using tracker right away
job_ids[job] = current_job_id
# set job status in database to pending
tracker.update(status=EStatus.PEND)
except RuntimeError as e:
# set job as failed in database
tracker.update(status=EStatus.FAIL, message=str(e))
# fail entire job submission if requested
if abort_on_error:
raise
# submit final summarizer
# (hold for now - summarizer is run after each subjob finishes)
# wait for all runs to finish (but only if blocking)
submitter.join()
# return job identifiers
return job_ids
def run_jobs(configs, global_config, overwrite=False, workdir=None):
"""
Submit config to pipeline
Parameters
----------
configs : dict
Configurations for individual subjobs
global_config : dict
Master configuration (if only one job,
the contents of this dictionary will be
equal to the single element of config_files)
"""
python = executable
pipeline_path = path.abspath(pipeline.__file__)
summarize_path = path.abspath(summarize.__file__)
cmd_base = "{} {}".format(python, pipeline_path)
summ_base = "{} {}".format(python, summarize_path)
# determine output directory for config files
prefix = global_config["global"]["prefix"]
# integrate working directory into output prefix
# if it is given; if prefix contains an absolute path,
# this will override the workdir according to
# implementation of path.join()
if workdir is not None:
out_prefix = path.join(workdir, prefix)
else:
out_prefix = prefix
# save configuration file, make sure we do not overwrite previous run
# if overwrite protection is activated
# (but only if it is a valid configuration file with contents)
cfg_filename = CONFIG_NAME.format(out_prefix)
if not overwrite and valid_file(cfg_filename):
raise InvalidParameterError(
"Existing configuration file {} ".format(cfg_filename) +
"indicates current prefix {} ".format(prefix) +
"would overwrite existing results. Use --yolo " +
"flag to deactivate overwrite protection (e.g. for "
"restarting a job or running a different stage)."
)
# make sure working directory exists
create_prefix_folders(cfg_filename)
# write global config file
write_config_file(cfg_filename, global_config)
# also write individual subjob configuration files
# (we have to write these before submitting, since
# the job summarizer needs the paths to all files)
for subjob_prefix, subjob_cfg in configs.items():
# determine working dir for each subjob, since subjob
# prefix may contain slashes leading to subfolder creation
if workdir is not None:
subjob_out_prefix = path.join(workdir, subjob_prefix)
else:
subjob_out_prefix = subjob_prefix
subcfg_filename = CONFIG_NAME.format(subjob_out_prefix)
# make sure output subfolder exists
create_prefix_folders(subcfg_filename)
# write subjob configuration file
write_config_file(subcfg_filename, subjob_cfg)
# now create list of subjob config files relative to working
# directory (above, we allow to run submitted in arbitrary directory)
config_files = [
CONFIG_NAME.format(subjob_prefix) for subjob_prefix in configs
]
# create command for summarizer (needs to know all subjob config files)
summ_cmd = "{} {} {} {}".format(
summ_base,
global_config["pipeline"],
global_config["global"]["prefix"],
" ".join(config_files)
)
# create submitter from global (pre-unrolling) configuration
submitter = utils.SubmitterFactory(
global_config["environment"]["engine"],
db_path=out_prefix + "_job_database.txt"
)
# collect individual submitted jobs here
commands = []
# prepare individual jobs for submission
for job, job_cfg in configs.items():
job_prefix = job_cfg["global"]["prefix"]
job_cfg_file = CONFIG_NAME.format(job)
# set job status in database to pending
pipeline.update_job_status(job_cfg, status=database.EStatus.PEND)
# create submission command
env = job_cfg["environment"]
cmd = utils.Command(
[
"{} {}".format(cmd_base, job_cfg_file),
summ_cmd
],
name=job_prefix,
environment=env["configuration"],
workdir=workdir,
resources={
utils.EResource.queue: env["queue"],
utils.EResource.time: env["time"],
utils.EResource.mem: env["memory"],
utils.EResource.nodes: env["cores"],
utils.EResource.out: job_prefix + "_stdout.log",
utils.EResource.error: job_prefix + "_stderr.log",
}
)
# store job for later dependency creation
commands.append(cmd)
# finally, submit job
submitter.submit(cmd)
# submit final summarizer
# (hold for now - summarizer is run after each subjob finishes)
# wait for all runs to finish (but only if blocking)
submitter.join()
def standard(**kwargs):
"""
Protocol:
Infer ECs from alignment using plmc.
.. todo::
1. make EC enrichment calculation segment-ready
2. explain meaning of parameters in detail.
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required
Returns
-------
outcfg : dict
Output configuration of the pipeline, including
the following fields:
* raw_ec_file
* model_file
* num_sites
* num_sequences
* effective_sequences
* focus_mode (passed through)
* focus_sequence (passed through)
* segments (passed through)
"""
check_required(
kwargs,
[
"prefix", "alignment_file",
"focus_mode", "focus_sequence", "theta",
"alphabet", "segments", "ignore_gaps", "iterations",
"lambda_h", "lambda_J", "lambda_group",
"scale_clusters",
"cpu", "plmc", "reuse_ecs",
"min_sequence_distance", # "save_model",
]
)
prefix = kwargs["prefix"]
# for now disable option to not save model, since
# otherwise mutate stage will crash. To remove model
# file at end, use delete option in management section.
"""
if kwargs["save_model"]:
model = prefix + ".model"
else:
model = None
"""
model = prefix + ".model"
outcfg = {
"model_file": model,
"raw_ec_file": prefix + "_ECs.txt",
"ec_file": prefix + "_CouplingScores.csv",
# TODO: the following are passed through stage...
# keep this or unnecessary?
"focus_mode": kwargs["focus_mode"],
"focus_sequence": kwargs["focus_sequence"],
"segments": kwargs["segments"],
}
# make sure input alignment exists
verify_resources(
"Input alignment does not exist",
kwargs["alignment_file"]
)
# make sure output directory exists
create_prefix_folders(prefix)
# regularization strength on couplings J_ij
lambda_J = kwargs["lambda_J"]
segments = kwargs["segments"]
if segments is not None:
segments = [
mapping.Segment.from_list(s) for s in segments
]
# first determine size of alphabet;
# default is amino acid alphabet
if kwargs["alphabet"] is None:
alphabet = ALPHABET_PROTEIN
alphabet_setting = None
else:
alphabet = kwargs["alphabet"]
# allow shortcuts for protein, DNA, RNA
if alphabet in ALPHABET_MAP:
alphabet = ALPHABET_MAP[alphabet]
# if we have protein alphabet, do not set
# as plmc parameter since default parameter,
# has some implementation advantages for focus mode
if alphabet == ALPHABET_PROTEIN:
alphabet_setting = None
else:
alphabet_setting = alphabet
# scale lambda_J to proportionally compensate
# for higher number of J_ij compared to h_i?
if kwargs["lambda_J_times_Lq"]:
num_symbols = len(alphabet)
# if we ignore gaps, there is one character less
if kwargs["ignore_gaps"]:
num_symbols -= 1
# second, determine number of uppercase positions
# that are included in the calculation
with open(kwargs["alignment_file"]) as f:
seq_id, seq = next(read_fasta(f))
# gap character is by convention first char in alphabet
gap = alphabet[0]
uppercase = [
c for c in seq if c == c.upper() or c == gap
]
L = len(uppercase)
# finally, scale lambda_J
lambda_J *= (num_symbols - 1) * (L - 1)
# run plmc... or reuse pre-exisiting results from previous run
plm_outcfg_file = prefix + ".couplings_standard_plmc.outcfg"
# determine if to rerun, only possible if previous results
# were stored in ali_outcfg_file
if kwargs["reuse_ecs"] and valid_file(plm_outcfg_file):
plmc_result = read_config_file(plm_outcfg_file)
# check if the EC/parameter files are there
required_files = [outcfg["raw_ec_file"]]
if outcfg["model_file"] is not None:
required_files += [outcfg["model_file"]]
verify_resources(
"Tried to reuse ECs, but empty or "
"does not exist",
*required_files
)
else:
# run plmc binary
plmc_result = ct.run_plmc(
kwargs["alignment_file"],
outcfg["raw_ec_file"],
outcfg["model_file"],
focus_seq=kwargs["focus_sequence"],
alphabet=alphabet_setting,
theta=kwargs["theta"],
scale=kwargs["scale_clusters"],
ignore_gaps=kwargs["ignore_gaps"],
iterations=kwargs["iterations"],
lambda_h=kwargs["lambda_h"],
lambda_J=lambda_J,
lambda_g=kwargs["lambda_group"],
cpu=kwargs["cpu"],
binary=kwargs["plmc"],
)
# save iteration table to file
iter_table_file = prefix + "_iteration_table.csv"
plmc_result.iteration_table.to_csv(
iter_table_file
)
# turn namedtuple into dictionary to make
# restarting code nicer
plmc_result = dict(plmc_result._asdict())
# then replace table with filename so
# we can store results in config file
plmc_result["iteration_table"] = iter_table_file
# save results of search for possible restart
write_config_file(plm_outcfg_file, plmc_result)
# store useful information about model in outcfg
outcfg.update({
"num_sites": plmc_result["num_valid_sites"],
"num_sequences": plmc_result["num_valid_seqs"],
"effective_sequences": plmc_result["effective_samples"],
"region_start": plmc_result["region_start"],
})
# read and sort ECs
ecs = pairs.read_raw_ec_file(outcfg["raw_ec_file"])
# add mixture model probability
ecs = pairs.add_mixture_probability(ecs)
if segments is not None: # and (len(segments) > 1 or not kwargs["focus_mode"]):
# create index mapping
seg_mapper = mapping.SegmentIndexMapper(
kwargs["focus_mode"], outcfg["region_start"], *segments
)
# apply to EC table
ecs = mapping.segment_map_ecs(ecs, seg_mapper)
# write updated table to csv file
ecs.to_csv(outcfg["ec_file"], index=False)
# also store longrange ECs as convenience output
if kwargs["min_sequence_distance"] is not None:
outcfg["ec_longrange_file"] = prefix + "_CouplingScores_longrange.csv"
ecs_longrange = ecs.query(
"abs(i - j) >= {}".format(kwargs["min_sequence_distance"])
)
ecs_longrange.to_csv(outcfg["ec_longrange_file"], index=False)
# also create line-drawing script (for now, only for single segments)
if segments is None or len(segments) == 1:
outcfg["ec_lines_pml_file"] = prefix + "_draw_ec_lines.pml"
L = outcfg["num_sites"]
ec_lines_pymol_script(
ecs_longrange.iloc[:L, :],
outcfg["ec_lines_pml_file"]
)
# compute EC enrichment (for now, for single segments
# only since enrichment code cannot handle multiple segments)
if segments is None or len(segments) == 1:
outcfg["enrichment_file"] = prefix + "_enrichment.csv"
ecs_enriched = pairs.enrichment(ecs)
ecs_enriched.to_csv(outcfg["enrichment_file"], index=False)
# create corresponding enrichment pymol scripts
outcfg["enrichment_pml_files"] = []
for sphere_view, pml_suffix in [
(True, "_enrichment_spheres.pml"), (False, "_enrichment_sausage.pml")
]:
pml_file = prefix + pml_suffix
enrichment_pymol_script(ecs_enriched, pml_file, sphere_view=sphere_view)
outcfg["enrichment_pml_files"].append(pml_file)
# output EVzoom JSON file if we have stored model file
if outcfg.get("model_file", None) is not None:
outcfg["evzoom_file"] = prefix + "_evzoom.json"
with open(outcfg["evzoom_file"], "w") as f:
# load parameters
c = CouplingsModel(outcfg["model_file"])
# create JSON output and write to file
f.write(
evzoom_json(c) + "\n"
)
# dump output config to YAML file for debugging/logging
write_config_file(prefix + ".couplings_standard.outcfg", outcfg)
return outcfg
def mean_field(**kwargs):
"""
Protocol:
Infer ECs from alignment using mean field direct coupling analysis.
For now, mean field DCA can only be run in focus mode, gaps
included.
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required.
Returns
-------
outcfg : dict
Output configuration of the pipeline, including
the following fields:
* raw_ec_file
* model_file
* num_sites
* num_sequences
* effective_sequences
* focus_mode (passed through)
* focus_sequence (passed through)
* segments (passed through)
"""
check_required(
kwargs,
[
"prefix", "alignment_file", "segments",
"focus_mode", "focus_sequence", "theta",
"pseudo_count", "alphabet",
"min_sequence_distance", # "save_model",
]
)
if not kwargs["focus_mode"]:
raise InvalidParameterError(
"For now, mean field DCA can only be run in focus mode."
)
prefix = kwargs["prefix"]
# option to save model disabled
"""
if kwargs["save_model"]:
model = prefix + ".model"
else:
model = None
"""
model = prefix + ".model"
outcfg = {
"model_file": model,
"raw_ec_file": prefix + "_ECs.txt",
"ec_file": prefix + "_CouplingScores.csv",
# TODO: the following are passed through stage...
# keep this or unnecessary?
"focus_mode": kwargs["focus_mode"],
"focus_sequence": kwargs["focus_sequence"],
"segments": kwargs["segments"],
}
# make sure input alignment exists
alignment_file = kwargs["alignment_file"]
verify_resources(
"Input alignment does not exist",
kwargs["alignment_file"]
)
# make sure output directory exists
create_prefix_folders(prefix)
segments = kwargs["segments"]
if segments is not None:
segments = [
mapping.Segment.from_list(s) for s in segments
]
# determine alphabet
# default is protein
if kwargs["alphabet"] is None:
alphabet = ALPHABET_PROTEIN
else:
alphabet = kwargs["alphabet"]
# allow shortcuts for protein, DNA, RNA
if alphabet in ALPHABET_MAP:
alphabet = ALPHABET_MAP[alphabet]
# read in a2m alignment
with open(alignment_file) as f:
input_alignment = Alignment.from_file(
f, alphabet=alphabet,
format="fasta"
)
# init mean field direct coupling analysis
mf_dca = MeanFieldDCA(input_alignment)
# run mean field approximation
model = mf_dca.fit(
theta=kwargs["theta"],
pseudo_count=kwargs["pseudo_count"]
)
# write ECs to file
model.to_raw_ec_file(
outcfg["raw_ec_file"]
)
# write model file
if outcfg["model_file"] is not None:
model.to_file(
outcfg["model_file"],
file_format="plmc_v2"
)
# store useful information about model in outcfg
outcfg.update({
"num_sites": model.L,
"num_sequences": model.N_valid,
"effective_sequences": float(round(model.N_eff, 1)),
"region_start": int(model.index_list[0]),
})
# read and sort ECs
ecs = pd.read_csv(
outcfg["raw_ec_file"], sep=" ",
# for now, call the last two columns
# "fn" and "cn" to prevent compare
# stage from crashing
names=["i", "A_i", "j", "A_j", "fn", "cn"]
# names=["i", "A_i", "j", "A_j", "mi", "di"]
).sort_values(
by="cn",
ascending=False
)
# write the sorted ECs table to csv file
ecs.to_csv(outcfg["ec_file"], index=False)
# also store longrange ECs as convenience output
if kwargs["min_sequence_distance"] is not None:
outcfg["ec_longrange_file"] = prefix + "_CouplingScores_longrange.csv"
ecs_longrange = ecs.query(
"abs(i - j) >= {}".format(kwargs["min_sequence_distance"])
)
ecs_longrange.to_csv(outcfg["ec_longrange_file"], index=False)
# also create line-drawing script (for now, only for single segments)
if segments is None or len(segments) == 1:
outcfg["ec_lines_pml_file"] = prefix + "_draw_ec_lines.pml"
L = outcfg["num_sites"]
ec_lines_pymol_script(
ecs_longrange.iloc[:L, :],
outcfg["ec_lines_pml_file"],
score_column="cn" # "di
)
# compute EC enrichment (for now, for single segments
# only since enrichment code cannot handle multiple segments)
if segments is None or len(segments) == 1:
outcfg["enrichment_file"] = prefix + "_enrichment.csv"
ecs_enriched = pairs.enrichment(ecs, score="cn") # "di"
ecs_enriched.to_csv(outcfg["enrichment_file"], index=False)
# create corresponding enrichment pymol scripts
outcfg["enrichment_pml_files"] = []
for sphere_view, pml_suffix in [
(True, "_enrichment_spheres.pml"), (False, "_enrichment_sausage.pml")
]:
pml_file = prefix + pml_suffix
enrichment_pymol_script(ecs_enriched, pml_file, sphere_view=sphere_view)
outcfg["enrichment_pml_files"].append(pml_file)
# output EVzoom JSON file if we have stored model file
if outcfg.get("model_file", None) is not None:
outcfg["evzoom_file"] = prefix + "_evzoom.json"
with open(outcfg["evzoom_file"], "w") as f:
# create JSON output and write to file
f.write(
evzoom_json(model) + "\n"
)
# dump output config to YAML file for debugging/logging
write_config_file(prefix + ".couplings_standard.outcfg", outcfg)
return outcfg
def mean_field(**kwargs):
"""
Protocol:
Infer ECs from alignment using mean field direct coupling analysis.
For now, mean field DCA can only be run in focus mode, gaps
included.
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required.
Returns
-------
outcfg : dict
Output configuration of the pipeline, including
the following fields:
* raw_ec_file
* model_file
* num_sites
* num_sequences
* effective_sequences
* focus_mode (passed through)
* focus_sequence (passed through)
* segments (passed through)
"""
check_required(
kwargs,
[
"prefix", "alignment_file", "segments",
"focus_mode", "focus_sequence", "theta",
"pseudo_count", "alphabet",
"min_sequence_distance", # "save_model",
"ec_score_type",
]
)
if not kwargs["focus_mode"]:
raise InvalidParameterError(
"For now, mean field DCA can only be run in focus mode."
)
prefix = kwargs["prefix"]
# option to save model disabled
"""
if kwargs["save_model"]:
model = prefix + ".model"
else:
model = None
"""
model = prefix + ".model"
outcfg = {
"model_file": model,
"raw_ec_file": prefix + "_ECs.txt",
"ec_file": prefix + "_CouplingScores.csv",
# TODO: the following are passed through stage...
# keep this or unnecessary?
"focus_mode": kwargs["focus_mode"],
"focus_sequence": kwargs["focus_sequence"],
"segments": kwargs["segments"],
}
# make sure input alignment exists
alignment_file = kwargs["alignment_file"]
verify_resources(
"Input alignment does not exist",
kwargs["alignment_file"]
)
# make sure output directory exists
create_prefix_folders(prefix)
segments = kwargs["segments"]
if segments is not None:
segments = [
mapping.Segment.from_list(s) for s in segments
]
# determine alphabet
# default is protein
if kwargs["alphabet"] is None:
alphabet = ALPHABET_PROTEIN
else:
alphabet = kwargs["alphabet"]
# allow shortcuts for protein, DNA, RNA
if alphabet in ALPHABET_MAP:
alphabet = ALPHABET_MAP[alphabet]
# read in a2m alignment
with open(alignment_file) as f:
input_alignment = Alignment.from_file(
f, alphabet=alphabet,
format="fasta"
)
# init mean field direct coupling analysis
mf_dca = MeanFieldDCA(input_alignment)
# run mean field approximation
model = mf_dca.fit(
theta=kwargs["theta"],
pseudo_count=kwargs["pseudo_count"]
)
# write ECs to file
model.to_raw_ec_file(
outcfg["raw_ec_file"]
)
# write model file
if outcfg["model_file"] is not None:
model.to_file(
outcfg["model_file"],
file_format="plmc_v2"
)
# store useful information about model in outcfg
outcfg.update({
"num_sites": model.L,
"num_valid_sequences": model.N_valid,
"effective_sequences": float(round(model.N_eff, 1)),
"region_start": int(model.index_list[0]),
})
# read and sort ECs
# Note: this now deviates from the original EC format
# file because it has 4 score columns to accomodate
# MI (raw), MI (APC-corrected), DI, CN;
ecs = pd.read_csv(
outcfg["raw_ec_file"], sep=" ",
names=["i", "A_i", "j", "A_j", "mi_raw", "mi_apc", "di", "cn"]
)
# select target score;
# by default select CN score, since it allows to compute probabilities etc.
ec_score_type = kwargs.get("ec_score_type", "cn")
valid_ec_type_choices = ["cn", "di", "mi_raw", "mi_apc"]
if ec_score_type not in valid_ec_type_choices:
raise InvalidParameterError(
"Invalid choice for valid_ec_type: {}, valid options are: {}".format(
ec_score_type, ", ".join(valid_ec_type_choices)
)
)
# perform rescoring if CN score is selected, otherwise cannot rescore
# since all models are based on distribution shapes generated by CN score
if ec_score_type == "cn":
# perform EC rescoring starting from CN score output by plmc;
# outconfig update will be merged further down in final outcfg merge
# returned list is already sorted
ecs, rescorer_outcfg_update = rescore_cn_score_ecs(
ecs, segments, outcfg, kwargs, score="cn"
)
else:
# If MI or DI, cannot apply distribution-based rescoring approaches,
# so just set score column and add dummy probability value for compatibility
# with downstream code
ecs = ecs.assign(
score=ecs[ec_score_type],
probability=np.nan
).sort_values(
by="score",
ascending=False
)
# no additional values to be updated in outcfg in this case
rescorer_outcfg_update = {}
is_single_segment = segments is None or len(segments) == 1
outcfg = {
**outcfg,
**rescorer_outcfg_update,
**_postprocess_inference(
ecs, kwargs, model, outcfg, prefix,
generate_enrichment=is_single_segment,
generate_line_plot=is_single_segment,
score="score"
)
}
# dump output config to YAML file for debugging/logging
write_config_file(prefix + ".couplings_meanfield.outcfg", outcfg)
return outcfg
