def check_hmm_signatures(signature_file, hmm_dir): sigs = get_signature_profiles(signature_file) for sig in sigs: hmm_file = os.path.abspath(os.path.join(hmm_dir, sig.path)) assert os.path.exists(hmm_file) name = None for line in open(hmm_file): if line.startswith("NAME"): name = line.split()[-1] assert name assert name == sig.name, "%s != %s" % (name, sig.name)
def get_signature_profiles() -> List[signature.HmmSignature]: """ Generates the HMM signature profiles from hmmdetails.txt Only does the processing once per python invocation, future runs access existing profiles """ # if already called once, then just reuse the cached results existing = getattr(get_signature_profiles, 'existing', None) if existing is not None: assert isinstance(existing, list) return existing # not cached yet, so generate profiles = signature.get_signature_profiles(path.get_full_path(__file__, "data", "hmmdetails.txt")) # cache this for future reuse, and silence mypy warnings because it can't handle it get_signature_profiles.existing = profiles # type: ignore return profiles
def get_sequence_counts(details_file: str) -> Dict[str, int]: """ Gets the number of sequences/seeds used to generate each HMM signature Arguments: detail_file: a file containing all HMMs Returns: a dictionary mapping HMM name to the number of sequences used to generate it """ result = {} for hmm in get_signature_profiles(details_file): for line in open(path.get_full_path(details_file, hmm.hmm_file), 'r'): if line.startswith('NSEQ '): result[hmm.name] = int(line[6:].strip()) break if hmm.name not in result: raise ValueError("Unknown number of seeds for hmm file: %s" % details_file) return result
def get_sequence_counts(details_file: str) -> Dict[str, str]: """ Gets the number of sequences/seeds used to generate each HMM signature Arguments: detail_file: a file containing all HMMs Returns: a dictionary mapping HMM name to the number of sequences used to generate it """ result = {} for hmm in get_signature_profiles(details_file): for line in open(path.get_full_path(details_file, hmm.hmm_file), 'r'): if line.startswith('NSEQ '): result[hmm.name] = line[6:].strip() break # TODO: ideally this shouldn't ever happen, clean up inputs and change to error if hmm.name not in result: result[hmm.name] = "?" return result
def detect_protoclusters_and_signatures(record: Record, signature_file: str, seeds_file: str, rule_files: List[str], valid_categories: Set[str], filter_file: str, tool: str, annotate_existing_subregions: bool = True) -> RuleDetectionResults: """ Compares all CDS features in a record with HMM signatures and generates Protocluster features based on those hits and the current protocluster detection rules. Arguments: record: the record to analyse signature_file: a tab separated file; each row being a single HMM reference with columns: label, description, minimum score cutoff, hmm path seeds_file: the file containing all HMM profiles rule_files: the files containing the rules to use for cluster definition valid_categories: a set containing valid rule category strings filter_file: a file containing equivalence sets of HMMs tool: the name of the tool providing the HMMs (e.g. rule_based_clusters) annotate_existing_subregions: if True, subregions already present in the record will have domains annotated even if no protocluster is found """ if not rule_files: raise ValueError("rules must be provided") full_fasta = fasta.get_fasta_from_record(record) # if there's no CDS features, don't try to do anything if not full_fasta: return RuleDetectionResults({}, tool, []) sig_by_name = {sig.name: sig for sig in get_signature_profiles(signature_file)} rules: List[rule_parser.DetectionRule] = [] aliases: Dict[str, List[rule_parser.Token]] = {} for rule_file in rule_files: rules = create_rules(rule_file, set(sig_by_name), valid_categories, aliases, rules) results = [] results_by_id: Dict[str, HSP] = {} runresults = run_hmmsearch(seeds_file, full_fasta, use_tempfile=True) for runresult in runresults: acc = runresult.accession.split('.')[0] # Store result if it is above cut-off for hsp in runresult.hsps: if hsp.query_id in sig_by_name: sig = sig_by_name[hsp.query_id] elif acc in sig_by_name: sig = sig_by_name[acc] else: raise ValueError('Failed to find signature for ID %s / ACC %s' % ( hsp.query_id, acc)) if hsp.bitscore > sig.cutoff: results.append(hsp) if hsp.hit_id not in results_by_id: results_by_id[hsp.hit_id] = [hsp] else: results_by_id[hsp.hit_id].append(hsp) # Filter results by comparing scores of different models (for PKS systems) results, results_by_id = filter_results(results, results_by_id, filter_file, set(sig_by_name)) # Filter multiple results of the same model in one gene results, results_by_id = filter_result_multiple(results, results_by_id) # Use rules to determine gene clusters cds_domains_by_cluster, cluster_type_hits = apply_cluster_rules(record, results_by_id, rules) # Find number of sequences on which each pHMM is based num_seeds_per_hmm = get_sequence_counts(signature_file) # annotate everything in detected protoclusters rules_by_name = {rule.name: rule for rule in rules} clusters = find_protoclusters(record, cluster_type_hits, rules_by_name) strip_inferior_domains(cds_domains_by_cluster, rules_by_name) def get_domains_for_cds(cds: CDSFeature) -> List[SecMetQualifier.Domain]: domains = [] for hsp in results_by_id.get(cds.get_name(), []): domains.append(SecMetQualifier.Domain(hsp.query_id, hsp.evalue, hsp.bitscore, num_seeds_per_hmm[hsp.query_id], tool)) return domains cds_results_by_cluster = {} cdses_with_annotations = set() for cluster in clusters: cds_results = [] for cds in record.get_cds_features_within_location(cluster.location): domains = get_domains_for_cds(cds) if domains: cds_results.append(CDSResults(cds, domains, cds_domains_by_cluster.get(cds.get_name(), {}))) cdses_with_annotations.add(cds) cds_results_by_cluster[cluster] = cds_results # add detected profile annotations for any existing subregions, if enabled cds_results_outside_clusters = [] if annotate_existing_subregions: for subregion in record.get_subregions(): for cds in subregion.cds_children: if cds in cdses_with_annotations: continue domains = get_domains_for_cds(cds) if domains: cds_results_outside_clusters.append(CDSResults(cds, domains, {})) cdses_with_annotations.add(cds) return RuleDetectionResults(cds_results_by_cluster, tool, cds_results_outside_clusters)
def detect_borders_and_signatures(record: Record, signature_file: str, seeds_file: str, rules_file: str, filter_file: str, tool: str) -> RuleDetectionResults: """ Compares all CDS features in a record with HMM signatures and generates Cluster features based on those hits and the current cluster detection rules. Arguments: record: the record to analyse signature_file: a tab separated file; each row being a single HMM reference with columns: label, description, minimum score cutoff, hmm path seeds_file: the file containing all HMM profiles rules_file: the file containing all the rules to use for cluster definition filter_file: a file containing equivalence sets of HMMs tool: the name of the tool providing the HMMs (e.g. clusterfinder, rule_based_clusters) """ full_fasta = fasta.get_fasta_from_record(record) # if there's no CDS features, don't try to do anything if not full_fasta: return None sig_by_name = { sig.name: sig for sig in get_signature_profiles(signature_file) } rules = create_rules(rules_file, set(sig_by_name)) results = [] results_by_id = {} # type: Dict[str, HSP] runresults = run_hmmsearch(seeds_file, full_fasta, use_tempfile=True) for runresult in runresults: acc = runresult.accession.split('.')[0] # Store result if it is above cut-off for hsp in runresult.hsps: if hsp.query_id in sig_by_name: sig = sig_by_name[hsp.query_id] elif acc in sig_by_name: sig = sig_by_name[acc] else: raise ValueError( 'Failed to find signature for ID %s / ACC %s' % (hsp.query_id, acc)) if hsp.bitscore > sig.cutoff: results.append(hsp) if hsp.hit_id not in results_by_id: results_by_id[hsp.hit_id] = [hsp] else: results_by_id[hsp.hit_id].append(hsp) # Filter results by comparing scores of different models (for PKS systems) results, results_by_id = filter_results(results, results_by_id, filter_file, set(sig_by_name)) # Filter multiple results of the same model in one gene results, results_by_id = filter_result_multiple(results, results_by_id) # Use rules to determine gene clusters cds_domains_by_cluster, cluster_type_hits = apply_cluster_rules( record, results_by_id, rules) # Find number of sequences on which each pHMM is based num_seeds_per_hmm = get_sequence_counts(signature_file) # Save final results to record rules_by_name = {rule.name: rule for rule in rules} clusters = find_clusters(record, cluster_type_hits, rules_by_name) strip_inferior_domains(cds_domains_by_cluster, rules_by_name) cds_results_by_cluster = {} for cluster in clusters: record.add_cluster_border(cluster) cds_results = [] cluster_extent = FeatureLocation( cluster.location.start - cluster.extent, cluster.location.end + cluster.extent) for cds in record.get_cds_features_within_location(cluster_extent): domains = [] for hsp in results_by_id.get(cds.get_name(), []): domains.append( SecMetQualifier.Domain(hsp.query_id, hsp.evalue, hsp.bitscore, num_seeds_per_hmm[hsp.query_id], tool)) if domains: cds_results.append( CDSResults(cds, domains, cds_domains_by_cluster.get(cds.get_name(), {}))) cds_results_by_cluster[cluster] = cds_results return RuleDetectionResults(cds_results_by_cluster, tool)