def _create_mapping(r):
_, query_start, query_end = parse_header(ali.ids[0])
# create mapping from query into PDB Uniprot sequence
# A_i will be query sequence indices, A_j Uniprot sequence indices
m = map_indices(ali[0], query_start, query_end,
ali[r["alignment_id"]], r["alignment_start"],
r["alignment_end"])
# create mapping from PDB Uniprot into seqres numbering
# j will be Uniprot sequence index, k seqres index
n = pd.DataFrame({
"j":
list(range(r["uniprot_start"], r["uniprot_end"] + 1)),
"k":
list(range(r["resseq_start"], r["resseq_end"] + 1)),
})
# need to convert to strings since other mapping has indices as strings
n.loc[:, "j"] = n.j.astype(str)
n.loc[:, "k"] = n.k.astype(str)
# join over Uniprot indices (i.e. j);
# get rid of any position that is not aligned
mn = m.merge(n, on="j", how="inner").dropna()
# extract final mapping from seqres (k) to query (i)
map_ = dict(zip(mn.k, mn.i))
return map_, mn
def find_paralogs(target_id, annotation_data, identity_threshold):
"""
Finds all the sequences in the alignment that originate
from the same species as the target_id, if those sequences
are below the identity threshold (ie, are diverged from
the query)
Parameters
----------
target_id : str
Full identifier of the query sequence
annotation_data : pd.DataFrame
With columns id, species, identity_to_query.
The column species contains the species annotation to use.
The column identity_to_query contains the prcent identity to the target id.
identity_threshold : float
Sequences above this identity to the query are not considered paralogs
Returns
-------
pd.DataFrame
with columns id, species, identity_to_query
Entries are paralogs found in the same genome as the query id
"""
base_query = parse_header(target_id)
# get all the rows that have an id that contains the
# query id. This includes the focus sequence and its hit to
# itself in the database.
contains_annotation = [
True if base_query in x else False for x in annotation_data.id.str
]
query_hits = annotation_data.loc[contains_annotation, :]
# get the species annotation for the query sequence
query_species = list(query_hits.species.dropna())
# get all rows that are from the query species
paralogs = annotation_data.query("species == @query_species")
# confirm that paralogs are below the similarity threshold
# ie, are diverged in sequence space from the query
paralogs = paralogs.query("identity_to_query < @identity_threshold")
return paralogs
def find_paralogs(target_id, id_to_organism, similarities, identity_threshold):
"""
Finds all the sequences in the alignment that originate
from the same species as the target_id, if those sequences
are below the identity threshold (ie, are diverged from
the query)
Parameters
----------
target_id : str
Full identifier of the query sequence
similarities : pd.DataFrame
The contents of identities.csv
id_to_organism : pd.DataFrame
The contents of annotation.csv
identity_threshold : float
Sequences above this identity to the query are not considered paralogs
Returns
-------
pd.DataFrame
with columns id, species, identity_to_query
Entries are paralogs found in the same genome as the query id
"""
# output of parse_header is (ID, region_start, region_end)
base_query_id, _, _ = parse_header(target_id)
# get all the rows that have an id that contains the
# query id. This includes the focus sequence and its hit to
# itself in the database.
annotation_data = similarities.merge(id_to_organism, on="id")
contains_annotation = [base_query_id in x for x in annotation_data.id]
query_hits = annotation_data.loc[contains_annotation, :]
# get the species annotation for the query sequence
query_species = list(query_hits.species.dropna())
# get all rows that are from the query species
paralogs = annotation_data.query("species == @query_species")
# confirm that paralogs are below the similarity threshold
# ie, are diverged in sequence space from the query
paralogs = paralogs.query("identity_to_query < @identity_threshold")
return paralogs
def _compute_monomer_distance_maps(sifts_map, name_prefix, chain_name):
# prepare a sequence map to remap the structures we have found
verify_resources("Target sequence file does not exist",
kwargs[name_prefix + "_target_sequence_file"])
# create target sequence map for remapping structure
with open(kwargs[name_prefix + "_target_sequence_file"]) as f:
header, seq = next(read_fasta(f))
# create target sequence map for remapping structure
seq_id, seq_start, seq_end = parse_header(header)
seqmap = dict(zip(range(seq_start, seq_end + 1), seq))
# compute distance maps and save
# (but only if we found some structure)
if len(sifts_map.hits) > 0:
d_intra = intra_dists(sifts_map,
structures,
atom_filter=kwargs["atom_filter"],
output_prefix=aux_prefix + "_" +
name_prefix + "_distmap_intra")
d_intra.to_file(outcfg[name_prefix + "_distmap_monomer"])
# save contacts to separate file
outcfg[
name_prefix +
"_monomer_contacts_file"] = prefix + "_" + name_prefix + "_contacts_monomer.csv"
d_intra.contacts(kwargs["distance_cutoff"]).to_csv(
outcfg[name_prefix + "_monomer_contacts_file"], index=False)
# compute multimer distances, if requested;
# note that d_multimer can be None if there
# are no structures with multiple chains
if kwargs[name_prefix + "_compare_multimer"]:
d_multimer = multimer_dists(sifts_map,
structures,
atom_filter=kwargs["atom_filter"],
output_prefix=aux_prefix + "_" +
name_prefix + "_distmap_multimer")
else:
d_multimer = None
# if we have a multimer contact map, save it
if d_multimer is not None:
d_multimer.to_file(outcfg[name_prefix + "_distmap_multimer"])
outcfg[
name_prefix +
"_multimer_contacts_file"] = prefix + name_prefix + "_contacts_multimer.csv"
# save contacts to separate file
d_multimer.contacts(kwargs["distance_cutoff"]).to_csv(
outcfg[name_prefix + "_multimer_contacts_file"],
index=False)
else:
outcfg[name_prefix + "_distmap_multimer"] = None
# create remapped structures (e.g. for
# later comparison of folding results)
# remap structures, swap mapping index and filename in
# dictionary so we have a list of files in the dict keys
outcfg[name_prefix + "_remapped_pdb_files"] = {
filename: mapping_index
for mapping_index, filename in remap_chains(
sifts_map,
aux_prefix,
seqmap,
chain_name=chain_name,
raise_missing=kwargs["raise_missing"]).items()
}
else:
# if no structures, cannot compute distance maps
d_intra = None
d_multimer = None
outcfg[name_prefix + "_distmap_monomer"] = None
outcfg[name_prefix + "_distmap_multimer"] = None
outcfg[name_prefix + "remapped_pdb_files"] = None
return d_intra, d_multimer, seqmap
def standard(**kwargs):
"""
Protocol:
Compare ECs for single proteins (or domains)
to 3D structure information
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required
Returns
-------
outcfg : dict
Output configuration of the pipeline, including
the following fields:
* ec_file_compared_all
* ec_file_compared_all_longrange
* pdb_structure_hits
* distmap_monomer
* distmap_multimer
* contact_map_files
* remapped_pdb_files
"""
check_required(kwargs, [
"prefix",
"ec_file",
"min_sequence_distance",
"pdb_mmtf_dir",
"atom_filter",
"compare_multimer",
"distance_cutoff",
"target_sequence_file",
"scale_sizes",
])
prefix = kwargs["prefix"]
outcfg = {
"ec_compared_all_file": prefix + "_CouplingScoresCompared_all.csv",
"ec_compared_longrange_file":
prefix + "_CouplingScoresCompared_longrange.csv",
"pdb_structure_hits_file": prefix + "_structure_hits.csv",
"pdb_structure_hits_unfiltered_file":
prefix + "_structure_hits_unfiltered.csv",
# cannot have the distmap files end with "_file" because there are
# two files (.npy and .csv), which would cause problems with automatic
# checking if those files exist
"distmap_monomer": prefix + "_distance_map_monomer",
"distmap_multimer": prefix + "_distance_map_multimer",
}
# make sure EC file exists
verify_resources("EC file does not exist", kwargs["ec_file"])
# make sure output directory exists
create_prefix_folders(prefix)
# store auxiliary files here (too much for average user)
aux_prefix = insert_dir(prefix, "aux", rootname_subdir=False)
create_prefix_folders(aux_prefix)
# Step 1: Identify 3D structures for comparison
sifts_map, sifts_map_full = _identify_structures(
**{
**kwargs,
"prefix": aux_prefix,
})
# save selected PDB hits
sifts_map.hits.to_csv(outcfg["pdb_structure_hits_file"], index=False)
# also save full list of hits
sifts_map_full.hits.to_csv(outcfg["pdb_structure_hits_unfiltered_file"],
index=False)
# Step 2: Compute distance maps
# load all structures at once
structures = load_structures(sifts_map.hits.pdb_id,
kwargs["pdb_mmtf_dir"],
raise_missing=False)
# compute distance maps and save
# (but only if we found some structure)
if len(sifts_map.hits) > 0:
d_intra = intra_dists(sifts_map,
structures,
atom_filter=kwargs["atom_filter"],
output_prefix=aux_prefix + "_distmap_intra")
d_intra.to_file(outcfg["distmap_monomer"])
# save contacts to separate file
outcfg["monomer_contacts_file"] = prefix + "_contacts_monomer.csv"
d_intra.contacts(kwargs["distance_cutoff"]).to_csv(
outcfg["monomer_contacts_file"], index=False)
# compute multimer distances, if requested;
# note that d_multimer can be None if there
# are no structures with multiple chains
if kwargs["compare_multimer"]:
d_multimer = multimer_dists(sifts_map,
structures,
atom_filter=kwargs["atom_filter"],
output_prefix=aux_prefix +
"_distmap_multimer")
else:
d_multimer = None
# if we have a multimer contact mapin the end, save it
if d_multimer is not None:
d_multimer.to_file(outcfg["distmap_multimer"])
outcfg[
"multimer_contacts_file"] = prefix + "_contacts_multimer.csv"
# save contacts to separate file
d_multimer.contacts(kwargs["distance_cutoff"]).to_csv(
outcfg["multimer_contacts_file"], index=False)
else:
outcfg["distmap_multimer"] = None
# at this point, also create remapped structures (e.g. for
# later comparison of folding results)
verify_resources("Target sequence file does not exist",
kwargs["target_sequence_file"])
# create target sequence map for remapping structure
with open(kwargs["target_sequence_file"]) as f:
header, seq = next(read_fasta(f))
seq_id, seq_start, seq_end = parse_header(header)
seqmap = dict(zip(range(seq_start, seq_end + 1), seq))
# remap structures, swap mapping index and filename in
# dictionary so we have a list of files in the dict keys
outcfg["remapped_pdb_files"] = {
filename: mapping_index
for mapping_index, filename in remap_chains(
sifts_map, aux_prefix, seqmap).items()
}
else:
# if no structures, can not compute distance maps
d_intra = None
d_multimer = None
outcfg["distmap_monomer"] = None
outcfg["distmap_multimer"] = None
outcfg["remapped_pdb_files"] = None
# Step 3: Compare ECs to distance maps
ec_table = pd.read_csv(kwargs["ec_file"])
# identify number of sites in EC model
num_sites = len(
set.union(set(ec_table.i.unique()), set(ec_table.j.unique())))
for out_file, min_seq_dist in [
("ec_compared_longrange_file", kwargs["min_sequence_distance"]),
("ec_compared_all_file", 0),
]:
# compare ECs only if we minimally have intra distance map
if d_intra is not None:
coupling_scores_compared(ec_table,
d_intra,
d_multimer,
dist_cutoff=kwargs["distance_cutoff"],
output_file=outcfg[out_file],
min_sequence_dist=min_seq_dist)
else:
outcfg[out_file] = None
# also create line-drawing script if we made the csv
if outcfg["ec_compared_longrange_file"] is not None:
ecs_longrange = pd.read_csv(outcfg["ec_compared_longrange_file"])
outcfg[
"ec_lines_compared_pml_file"] = prefix + "_draw_ec_lines_compared.pml"
pairs.ec_lines_pymol_script(ecs_longrange.iloc[:num_sites, :],
outcfg["ec_lines_compared_pml_file"],
distance_cutoff=kwargs["distance_cutoff"])
# Step 4: Make contact map plots
# if no structures available, defaults to EC-only plot
outcfg["contact_map_files"] = _make_contact_maps(ec_table, d_intra,
d_multimer, **kwargs)
return outcfg
def existing(**kwargs):
"""
Protocol:
Use external sequence alignment and extract all relevant
information from there (e.g. sequence, region, etc.),
then apply gap & fragment filtering as usual
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required
Returns
-------
outcfg : dict
Output configuration of the pipeline, including
the following fields:
* sequence_id (passed through from input)
* alignment_file
* raw_focus_alignment_file
* statistics_file
* sequence_file
* first_index
* target_sequence_file
* annotation_file (None)
* frequencies_file
* identities_file
* focus_mode
* focus_sequence
* segments
"""
check_required(kwargs, [
"prefix", "input_alignment", "sequence_id", "first_index",
"extract_annotation"
])
prefix = kwargs["prefix"]
# make sure output directory exists
create_prefix_folders(prefix)
# this file is starting point of pipeline;
# check if input alignment actually exists
input_alignment = kwargs["input_alignment"]
verify_resources("Input alignment does not exist", input_alignment)
# first try to autodetect format of alignment
with open(input_alignment) as f:
format = detect_format(f)
if format is None:
raise InvalidParameterError(
"Format of input alignment {} could not be "
"automatically detected.".format(input_alignment))
with open(input_alignment) as f:
ali_raw = Alignment.from_file(f, format)
# save annotation in sequence headers (species etc.)
annotation_file = None
if kwargs["extract_annotation"]:
annotation_file = prefix + "_annotation.csv"
from_anno_line = (format == "stockholm")
annotation = extract_header_annotation(ali_raw,
from_annotation=from_anno_line)
annotation.to_csv(annotation_file, index=False)
# Target sequence of alignment
sequence_id = kwargs["sequence_id"]
if sequence_id is None:
raise InvalidParameterError("Parameter sequence_id must be defined")
# First, find focus sequence in alignment
focus_index = None
for i, id_ in enumerate(ali_raw.ids):
if id_.startswith(sequence_id):
focus_index = i
break
# if we didn't find it, cannot continue
if focus_index is None:
raise InvalidParameterError(
"Target sequence {} could not be found in alignment".format(
sequence_id))
# identify what columns (non-gap) to keep for focus
focus_seq = ali_raw[focus_index]
focus_cols = np.array([
c not in [ali_raw._match_gap, ali_raw._insert_gap] for c in focus_seq
])
# extract focus alignment
focus_ali = ali_raw.select(columns=focus_cols)
focus_seq_nogap = "".join(focus_ali[focus_index])
# determine region of sequence. If first_index is given,
# use that in any case, otherwise try to autodetect
full_focus_header = ali_raw.ids[focus_index]
focus_id = full_focus_header.split()[0]
# try to extract region from sequence header
id_, region_start, region_end = parse_header(focus_id)
# override with first_index if given
if kwargs["first_index"] is not None:
region_start = kwargs["first_index"]
region_end = region_start + len(focus_seq_nogap) - 1
if region_start is None or region_end is None:
raise InvalidParameterError(
"Could not extract region information " +
"from sequence header {} ".format(full_focus_header) +
"and first_index parameter is not given.")
# resubstitute full sequence ID from identifier
# and region information
header = "{}/{}-{}".format(id_, region_start, region_end)
focus_ali.ids[focus_index] = header
# write target sequence to file
target_sequence_file = prefix + ".fa"
with open(target_sequence_file, "w") as f:
write_fasta([(header, focus_seq_nogap)], f)
# apply sequence identity and fragment filters,
# and gap threshold
mod_outcfg, ali = modify_alignment(focus_ali, focus_index, id_,
region_start, **kwargs)
# generate output configuration of protocol
outcfg = {
**mod_outcfg,
"sequence_id": sequence_id,
"sequence_file": target_sequence_file,
"first_index": region_start,
"target_sequence_file": target_sequence_file,
"focus_sequence": header,
"focus_mode": True,
}
if annotation_file is not None:
outcfg["annotation_file"] = annotation_file
# dump config to YAML file for debugging/logging
write_config_file(prefix + ".align_existing.outcfg", outcfg)
# return results of protocol
return outcfg
def _format_alignment_for_hmmbuild(input_alignment_file, **kwargs):
# this file is starting point of pipeline;
# check if input alignment actually exists
verify_resources("Input alignment does not exist",
input_alignment_file)
# first try to autodetect format of alignment
with open(input_alignment_file) as f:
format = detect_format(f)
if format is None:
raise InvalidParameterError(
"Format of input alignment {} could not be "
"automatically detected.".format(input_alignment_file))
with open(input_alignment_file) as f:
ali_raw = Alignment.from_file(f, format)
# Target sequence of alignment
sequence_id = kwargs["sequence_id"]
if sequence_id is None:
raise InvalidParameterError(
"Parameter sequence_id must be defined")
# First, find focus sequence in alignment
focus_index = None
for i, id_ in enumerate(ali_raw.ids):
if id_.startswith(sequence_id):
focus_index = i
break
# if we didn't find it, cannot continue
if focus_index is None:
raise InvalidParameterError(
"Target sequence {} could not be found in alignment".format(
sequence_id))
# identify what columns (non-gap) to keep for focus
# this should be all columns in the raw_focus_alignment_file
# but checking anyway
focus_seq = ali_raw[focus_index]
focus_cols = np.array([
c not in [ali_raw._match_gap, ali_raw._insert_gap]
for c in focus_seq
])
# extract focus alignment
focus_ali = ali_raw.select(columns=focus_cols)
focus_seq_nogap = "".join(focus_ali[focus_index])
# determine region of sequence. If first_index is given,
# use that in any case, otherwise try to autodetect
full_focus_header = ali_raw.ids[focus_index]
focus_id = full_focus_header.split()[0]
# try to extract region from sequence header
id_, region_start, region_end = parse_header(focus_id)
# override with first_index if given
if kwargs["first_index"] is not None:
region_start = kwargs["first_index"]
region_end = region_start + len(focus_seq_nogap) - 1
if region_start is None or region_end is None:
raise InvalidParameterError(
"Could not extract region information " +
"from sequence header {} ".format(full_focus_header) +
"and first_index parameter is not given.")
# resubstitute full sequence ID from identifier
# and region information
header = "{}/{}-{}".format(id_, region_start, region_end)
focus_ali.ids[focus_index] = header
# write target sequence to file
target_sequence_file = prefix + ".fa"
with open(target_sequence_file, "w") as f:
write_fasta([(header, focus_seq_nogap)], f)
# swap target sequence to first position if it is not
# the first sequence in alignment;
# this is particularly important for hhfilter run
# because target sequence might otherwise be filtered out
if focus_index != 0:
indices = np.arange(0, len(focus_ali))
indices[0] = focus_index
indices[focus_index] = 0
focus_index = 0
focus_ali = focus_ali.select(sequences=indices)
# write the raw focus alignment for hmmbuild
focus_fasta_file = prefix + "_raw_focus_input.fasta"
with open(focus_fasta_file, "w") as f:
focus_ali.write(f, "fasta")
return focus_fasta_file, target_sequence_file, region_start, region_end
def secondary_structure(**kwargs):
"""
Predict or load secondary structure for an
input sequence
Parameters
----------
Mandatory kwargs arguments:
See list below in code where calling check_required
Returns
-------
residues : pandas.DataFrame
Table with sequence and secondary structure
in columns i, A_i and sec_struct_3state
"""
check_required(
kwargs,
[
"prefix", "target_sequence_file",
"segments", "sec_struct_method",
"sec_struct_file", "psipred",
]
)
prefix = kwargs["prefix"]
create_prefix_folders(prefix)
secstruct_file = kwargs["sec_struct_file"]
if secstruct_file is not None:
verify_resources(
"Secondary structure prediction file does not exist/is empty",
secstruct_file
)
residues = pd.read_csv(secstruct_file)
else:
# make sure target sequence file is there so we can
# predict secondary structure
target_seq_file = kwargs["target_sequence_file"]
verify_resources(
"Sequence file does not exist/is empty", target_seq_file
)
# we need to figure out what the index of the first residue
# in the target sequence is; obtain first index from segment
# information if possible
if kwargs["segments"] is not None:
s = Segment.from_list(kwargs["segments"][0])
first_index = s.region_start
else:
# otherwise try to get it from sequence file
first_index = None
with open(target_seq_file) as f:
header, _ = next(read_fasta(f))
if header is not None:
_, first_index, _ = parse_header(header)
# if we cannot identify first index from header,
# do not make guesses but fail
if first_index is None:
raise InvalidParameterError(
"Could not unambiguously identify sequence range from "
"FASTA header, needs to specified as id/start-end: {}".format(
header
)
)
# finally, run secondary structure prediction
if kwargs["sec_struct_method"] == "psipred":
# store psipred output in a separate directory
output_dir = path.join(path.dirname(prefix), "psipred")
# run psipred
ss2_file, horiz_file = run_psipred(
target_seq_file, output_dir, binary=kwargs["psipred"]
)
# parse output, renumber to first index
residues = read_psipred_prediction(
horiz_file, first_index=first_index
)
else:
raise InvalidParameterError(
"Secondary structure prediction method not implemented: "
"{}. Valid choices: psipred".format(kwargs["sec_struct_method"])
)
# return predicted table
return residues
def alignment_index_mapping(alignment_file,
format="stockholm",
target_seq=None):
"""
Create index mapping table between sequence positions
based on a sequence alignment.
Parameters
----------
alignment_file : str
Path of alignment file containing sequences for
which indices shoul dbe mapped
format : {"stockholm", "fasta"}
Format of alignment file
target_seq : str, optional (default: None)
Identifier of sequence around which the index
mapping will be centered. If None, first sequence
in alignment will be used.
Returns
-------
pandas.DataFrame
Mapping table containing assignment of
1. index in target sequence (i)
2. symbol in target sequence (A_i)
For all other sequences in alignment, the following
two columns:
3. index in second sequence (j_<sequence id>)
4. symbol in second sequence (A_j_<sequence_id>)
"""
# read alignment that is basis of mapping
with open(alignment_file) as a:
ali = Alignment.from_file(a, format)
# determine index of target sequence if necessary
# (default: first sequence in alignment)
if target_seq is None:
target_seq_index = 0
else:
for i, full_id in enumerate(ali.ids):
if full_id.startswith(target_seq):
target_seq_index = i
# get range and sequence of target
id_, target_start, target_end = parse_header(ali.ids[target_seq_index])
target_seq = ali.matrix[target_seq_index]
# now map from target numbering to hit numbering
full_map = None
for i, full_id in enumerate(ali.ids):
if i == target_seq_index:
continue
# extract information about sequence we are comparing to
id_, region_start, region_end = parse_header(full_id)
other_seq = ali.matrix[i]
# compute mapping table
map_df = map_indices(target_seq, target_start, target_end, other_seq,
region_start, region_end,
[ali._match_gap, ali._insert_gap])
# adjust column names for non-target sequence
map_df = map_df.rename(columns={
"j": "i_" + full_id,
"A_j": "A_i_" + full_id,
})
# add to full mapping table, left outer join
# so all positions in target sequence are kept
if full_map is None:
full_map = map_df
else:
full_map = full_map.merge(map_df, on=("i", "A_i"), how="left")
return full_map
