class DummyParser(ReturnParser): def __init__(self): super(DummyParser, self).__init__() # arguments self.arguments = ['taxid'] # output data self.dummy_nodes = NodeSet(['Dummy'], merge_keys=['sid']) self.fummy_nodes = NodeSet(['Fummy'], merge_keys=['sid']) self.dummy_knows_fummy = RelationshipSet('KNOWS', ['Dummy'], ['Fummy'], ['sid'], ['sid']) def run_with_mounted_arguments(self): self.run(self.taxid) def run(self, taxid): dummy_instance = self.get_instance_by_name('Dummy') dummyfile = dummy_instance.get_file('file.txt') target_sids = list(string.ascii_lowercase) # Fummy nodes for i in range(10): self.fummy_nodes.add_node({'sid': i, 'taxid': taxid}) with open(dummyfile) as f: for l in f: letter = l.strip() self.dummy_nodes.add_node({'sid': letter, 'taxid': taxid}) self.dummy_knows_fummy.add_relationship({'sid': letter}, {'sid': randint(0, 9)}, {'key': 'value'})
class HGNCParser(ReturnParser): def __init__(self): """ :param ncbigene_instance: NcbiGene Instance :type ncbigene_instance: DataSourceInstance :param taxid: """ super(HGNCParser, self).__init__() # output data self.genes = NodeSet(['Gene'], merge_keys=['sid']) self.gene_maps_gene = RelationshipSet('MAPS', ['Gene'], ['Gene'], ['sid'], ['sid']) self.gene_maps_genesymbol = RelationshipSet('MAPS', ['Gene'], ['GeneSymbol'], ['sid'], ['sid', 'taxid']) def run_with_mounted_arguments(self): self.run() def run(self): hgnc_instance = self.get_instance_by_name('HGNC') hgnc_complete_file = hgnc_instance.get_file('hgnc_complete_set.txt') self.parse_hgnc_complete_file(hgnc_complete_file) def parse_hgnc_complete_file(self, hgnc_complete_file): with open(hgnc_complete_file, 'rt') as f: header = next(f) for l in f: flds = l.strip().split('\t') sid = flds[0] gene_symbol = flds[1] ncbi_id = flds[18] if len(flds) > 18 else None ensembl_id = flds[19] if len(flds) > 19 else None all_props = dict(zip(header, flds)) all_props['sid'] = sid all_props['source'] = 'hgnc' self.genes.add_node(all_props) if ncbi_id: self.gene_maps_gene.add_relationship({'sid': sid}, {'sid': ncbi_id}, {'source': 'hgnc'}) if ensembl_id: self.gene_maps_gene.add_relationship({'sid': sid}, {'sid': ensembl_id}, {'source': 'hgnc'}) if gene_symbol: self.gene_maps_genesymbol.add_relationship({'sid': sid}, { 'sid': gene_symbol, 'taxid': '9606' }, {'source': 'hgnc'})
def load_wpp_data(base_path, graph): """ Load UN population data. :param base_path: Path where file was downloaded. """ un_wpp_csv_file = os.path.join(base_path, 'WPP2019_PopulationByAgeSex_Medium.csv') log.info('Parse UN population data file: {}'.format(un_wpp_csv_file)) country = NodeSet(['Country'], ['name']) age_group_nodes = NodeSet(['AgeGroup'], ['group']) country_total_group = RelationshipSet('CURRENT_TOTAL', ['Country'], ['AgeGroup'], ['name'], ['group']) country_male_group = RelationshipSet('CURRENT_MALE', ['Country'], ['AgeGroup'], ['name'], ['group']) country_female_group = RelationshipSet('CURRENT_FEMALE', ['Country'], ['AgeGroup'], ['name'], ['group']) countries_added = set() age_groups_added = set() with open(un_wpp_csv_file, 'rt') as f: csv_file = csv.reader(f, delimiter=',', quotechar='"') # skip header next(csv_file) for row in csv_file: # LocID,Location,VarID,Variant,Time,MidPeriod,AgeGrp,AgeGrpStart,AgeGrpSpan,PopMale,PopFemale,PopTotal loc_id = row[0] location = row[1] time = int(row[4]) age_group = row[6] age_group_start = int(row[7]) age_group_span = row[8] pop_male = int(float((row[9])) * 1000) pop_female = int(float((row[10])) * 1000) pop_total = int(float((row[11])) * 1000) # only take 2019 if time == 2019: if location not in countries_added: country.add_node({'name': location, 'un_id': loc_id}) countries_added.add(location) if age_group not in age_groups_added: age_group_nodes.add_node({'group': age_group, 'start': age_group_start, 'span': age_group_span}) country_total_group.add_relationship({'name': location}, {'group': age_group}, {'count': pop_total}) country_male_group.add_relationship({'name': location}, {'group': age_group}, {'count': pop_male}) country_female_group.add_relationship({'name': location}, {'group': age_group}, {'count': pop_female}) log.info('Load data to Neo4j') country.merge(graph) age_group_nodes.merge(graph) country_total_group.merge(graph) country_male_group.merge(graph) country_female_group.merge(graph)
class RootTestParser(ReturnParser): def __init__(self): super(RootTestParser, self).__init__() self.source = NodeSet(['Source'], merge_keys=['source_id']) self.target = NodeSet(['Target'], merge_keys=['target_id']) def run_with_mounted_arguments(self): self.run() def run(self): for i in range(100): self.source.add_node({'source_id': i}) self.target.add_node({'target_id': i})
class EnsemblLocusParser(ReturnParser): def __init__(self): """ :param ensembl_instance: The ENSEMBL DataSource instance. """ super(EnsemblLocusParser, self).__init__() # arguments self.arguments = ['taxid'] # NodeSets self.locus = NodeSet(['Locus'], merge_keys=['uuid'], default_props={'source': 'ensembl'}) def run_with_mounted_arguments(self): self.run(self.taxid) def run(self, taxid): ensembl_instance = self.get_instance_by_name('Ensembl') datasource_name = ensembl_instance.datasource.name ensembl_gtf_file_path = Ensembl.get_gtf_file_path( taxid, ensembl_instance) annotation = GffReader(ensembl_gtf_file_path) log.info("Start parsing ENSEMBL gtf file, taxid {}, {}".format( taxid, ensembl_gtf_file_path)) for r in annotation.records: # one line is one unique Locus props = { 'chr': r.chr, 'annotation_source': r.source, 'start': int(r.start), 'end': int(r.end), 'type': r.type, 'score': r.score, 'strand': r.strand, 'frame': r.frame, 'taxid': taxid, 'ref': 'h38', 'uuid': str(uuid4()) } props.update(r.attributes) self.locus.add_node(props) log.info("Finished parsing ENSEMBL gtf file.")
class NcbiLegacyGeneParser(ReturnParser): """ Parse legacy gene IDs from gene_history.gz #tax_id GeneID Discontinued_GeneID Discontinued_Symbol Discontinue_Date 9 - 1246494 repA1 20031113 9 - 1246495 repA2 20031113 9 - 1246496 leuA 20031113 """ def __init__(self): super(NcbiLegacyGeneParser, self).__init__() self.arguments = ['taxid'] self.legacy_genes = NodeSet(['Gene', 'Legacy'], merge_keys=['sid'], default_props={'source': 'ncbigene'}) self.legacy_gene_now_gene = RelationshipSet( 'REPLACED_BY', ['Gene', 'Legacy'], ['Gene'], ['sid'], ['sid'], default_props={'source': 'ncbigene'}) def run_with_mounted_arguments(self): self.run(self.taxid) def run(self, taxid): log.debug(f'Run parser {self.__class__.__name__} for taxID: {taxid}.') ncbigene_instance = self.get_instance_by_name('NcbiGene') gene_history_file = ncbigene_instance.get_file('gene_history.gz') with gzip.open(gene_history_file, 'rt') as f: # skip header next(f) for l in f: flds = l.strip().split('\t') this_taxid = flds[0] if this_taxid == taxid: new_gene_id = flds[1] discontinued_gene_id = flds[2] discontinued_symbol = flds[3] date = flds[4] self.legacy_genes.add_node({ 'sid': discontinued_gene_id, 'date': date, 'symbol': discontinued_symbol, 'taxid': taxid }) if new_gene_id != '-': self.legacy_gene_now_gene.add_relationship( {'sid': discontinued_gene_id}, {'sid': new_gene_id}, {})
class SomeParser(ReturnParser): def __init__(self): super(SomeParser, self).__init__() self.source = NodeSet(['Source'], merge_keys=['source_id']) self.target = NodeSet(['Target'], merge_keys=['target_id']) self.rels = RelationshipSet('FOO', ['Source'], ['Target'], ['source_id'], ['target_id']) def run_with_mounted_arguments(self): self.run() def run(self): for i in range(100): self.source.add_node({'source_id': i}) self.target.add_node({'target_id': i}) self.rels.add_relationship({'source_id': i}, {'target_id': i}, {'source': 'test'})
class BigWordListParser(ReturnParser): def __init__(self): """ """ super(BigWordListParser, self).__init__() # NodeSets self.words = NodeSet(['Word'], merge_keys=['value']) def run_with_mounted_arguments(self): self.run() def run(self): log.info("Run {}".format(self.__class__.__name__)) bigwordlist_instance = self.get_instance_by_name('BigWordList') # collect the words and a list of wordlists they are metioned in word_to_list = {} log.info("") for i in range(3, 13): match_file = bigwordlist_instance.get_file( 'wlist_match{}.txt'.format(i)) try: log.info("Open {}".format(match_file)) with open(match_file, 'r') as f: for l in f: word = l.strip() if word in word_to_list: word_to_list[word].append(i) else: word_to_list[word] = [i] except TypeError: log.info("Cannot open file {}".format(match_file)) for word, list_of_matches in word_to_list.items(): node_props = {'value': word} for i in list_of_matches: node_props["match{}".format(i)] = True self.words.add_node(node_props)
class ChebiParser(ReturnParser): def __init__(self): super(ChebiParser, self).__init__() # NodeSets self.metabolites = NodeSet(['Metabolite'], merge_keys=['sid'], default_props={'source': 'chebi'}) self.metabolite_isa_metabolite = RelationshipSet( 'IS_A', ['Metabolite'], ['Metabolite'], ['sid'], ['sid'], default_props={'source': 'chebi'}) self.metabolite_rel_metabolite = RelationshipSet( 'CHEBI_REL', ['Metabolite'], ['Metabolite'], ['sid'], ['sid'], default_props={'source': 'chebi'}) self.metabolite_maps_metabolite = RelationshipSet( 'MAPS', ['Metabolite'], ['Metabolite'], ['sid'], ['sid'], default_props={'source': 'chebi'}) def run_with_mounted_arguments(self): self.run() def run(self): chebi_instance = self.get_instance_by_name('Chebi') obo_file = chebi_instance.get_file('chebi.obo') cleaned_obo_file = clean_obo_file(obo_file) chebi_ontology = pronto.Ontology(cleaned_obo_file) reltypes = set() # iterate terms for term in chebi_ontology.terms(): term_sid = (term.id).split(':')[1] ontology_id = term.id self.metabolites.add_node({ 'name': (term.name), 'sid': term_sid, 'ontology_id': ontology_id, 'definition': term.definition, 'alt_ids': list(term.alternate_ids) }) for parent in term.superclasses(distance=1, with_self=False): self.metabolite_isa_metabolite.add_relationship( {'sid': term_sid}, {'sid': parent.id}, {}) ## other named relationships try: for reltype, targets in term.relationships.items(): for target in targets: self.metabolite_rel_metabolite.add_relationship( {'sid': term_sid}, {'sid': target.id}, {'type': reltype.id}) except KeyError as e: log.error(f"Cannot iterate relationshis of term {term_sid}") log.error(e) # metabolite-MAPS-metabolite for xref in term.xrefs: if 'HMDB:' in xref.id: hmdb_id = xref.id.strip().split('HMDB:')[1] self.metabolite_maps_metabolite.add_relationship( {'sid': term_sid}, {'sid': hmdb_id}, {})
class UniprotKnowledgebaseParser(ReturnParser): """ Uniprot has extensive mapping data to other data sources. Data is in the main Uniprot data file (referred to as Uniprot knowledge base). Ensembl: DR Ensembl; ENST00000353703; ENSP00000300161; ENSG00000166913. [P31946-1] DR Ensembl; ENST00000372839; ENSP00000361930; ENSG00000166913. [P31946-1] Refseq: DR RefSeq; NP_006752.1; NM_006761.4. [P62258-1] The mapping parser returns transcript-protein relationships for both ENSEMBL and RefSeq. """ def __init__(self): """ :param uniprot_instance: The Uniprot instance :param taxid: The taxid """ super(UniprotKnowledgebaseParser, self).__init__() # arguments self.arguments = ['taxid'] # NodeSet self.proteins = NodeSet(['Protein'], merge_keys=['sid'], default_props={'source': 'uniprot'}) # RelationshipSet self.protein_primary_protein = RelationshipSet('PRIMARY', ['Protein'], ['Protein'], ['sid'], ['sid'], default_props={'source': 'uniprot'}) self.transcript_codes_protein = RelationshipSet('CODES', ['Transcript'], ['Protein'], ['sid'], ['sid'], default_props={'source': 'uniprot'}) self.protein_maps_protein = RelationshipSet('MAPS', ['Protein'], ['Protein'], ['sid'], ['sid'], default_props={'source': 'uniprot'}) def run_with_mounted_arguments(self): self.run(self.taxid) def run(self, taxid): uniprot_instance = self.get_instance_by_name('Uniprot') knowledgebase_files = uniprot_instance.datasource.get_knowledgebase_files_for_taxid(taxid, uniprot_instance) datasource_name = uniprot_instance.datasource.name # get organims name from taxid os_string_id = TAXID_OS_NAME[taxid] check_protein = set() check_p_p_p = set() check_t_c_p = set() check_p_m_p = set() # for now we always run on SPROT and TREMBL for kb_file in knowledgebase_files: log.debug(f"Parsing {kb_file}") with gzip.open(kb_file, 'rt') as f: up_parser = EMBLReaderUniProt(f) for record in up_parser.records: # check taxon if os_string_id in record['OS']: # acc acc_list = record['AC'] primary_acc = acc_list[0] secondary = acc_list[1:] # (Protein) # make primary protein with full data desc = record['DE'] rec_name = desc.split(';')[0].split('Full=')[1] primary_props = {'sid': primary_acc, 'name': rec_name, 'desc': desc, 'category': 'primary', 'taxid': taxid} if primary_acc not in check_protein: self.proteins.add_node(primary_props) check_protein.add(primary_acc) for secondary_acc in secondary: if secondary_acc not in check_protein: self.proteins.add_node( {'sid': secondary_acc, 'category': 'secondary', 'taxid': taxid}) check_protein.add(secondary_acc) # (Protein)-[PRIMARY]-(Protein) if frozenset([primary_acc, secondary_acc]) not in check_p_p_p: self.protein_primary_protein.add_relationship( {'sid': primary_acc}, {'sid': secondary}, {} ) check_p_p_p.add(frozenset([primary_acc, secondary_acc])) # (Transcript)-[CODES]-(Protein) # (Protein)-[MAPS]-(Protein) ## RefSeq # ('RefSeq', ['NP_003395']), refseq_mappings = [x[1] for x in record['DR'] if x[0] == 'RefSeq'] for map in refseq_mappings: for refseq_id in map: # remove version from refseq ID refseq_id = refseq_id.split('.')[0] second_letter = refseq_id[1] # (Transcript)-[CODES]-(Protein) if second_letter == 'M' or second_letter == 'R': for uniprot_acc in acc_list: if refseq_id + uniprot_acc not in check_t_c_p: self.transcript_codes_protein.add_relationship( {'sid': refseq_id}, {'sid': uniprot_acc}, {'source': datasource_name} ) check_t_c_p.add(refseq_id + uniprot_acc) # (Protein)-[MAPS]-(Protein) if second_letter == 'P': for uniprot_acc in acc_list: if uniprot_acc + refseq_id not in check_p_m_p: self.protein_maps_protein.add_relationship( {'sid': uniprot_acc}, {'sid': refseq_id}, {} ) check_p_m_p.add(uniprot_acc + refseq_id) ## ensembl ensembl_mappings = [x[1] for x in record['DR'] if x[0] == 'Ensembl'] for map in ensembl_mappings: ensembl_transcript_id = map[0] ensembl_protein_id = map[1] for uniprot_acc in acc_list: # (Transcript)-[CODES]-(Protein) if ensembl_transcript_id + uniprot_acc not in check_t_c_p: self.transcript_codes_protein.add_relationship( {'sid': ensembl_transcript_id}, {'sid': uniprot_acc}, {} ) check_t_c_p.add(ensembl_transcript_id + uniprot_acc) # (Protein)-[MAPS]-(Protein) if ensembl_protein_id + uniprot_acc not in check_p_m_p: self.protein_maps_protein.add_relationship( {'sid': uniprot_acc}, {'sid': ensembl_protein_id}, {} ) check_p_m_p.add(ensembl_protein_id + uniprot_acc)
class NcbiGeneParser(ReturnParser): def __init__(self): super(NcbiGeneParser, self).__init__() # arguments self.arguments = ['taxid'] # output data # both gene IDs and GeneSymbols have the label 'Gene' # two different NodeSets are used because only the GeneSymbol nodes need taxid for uniqueness self.genes = NodeSet(['Gene'], merge_keys=['sid'], default_props={'source': 'ncbigene'}) self.genesymbols = NodeSet(['Gene'], merge_keys=['sid', 'taxid'], default_props={ 'source': 'ncbigene', 'type': 'symbol' }) self.genesymbol_synonym_genesymbol = RelationshipSet( 'SYNONYM', ['Gene'], ['Gene'], ['sid', 'taxid'], ['sid', 'taxid'], default_props={'source': 'ncbigene'}) self.gene_maps_genesymbol = RelationshipSet( 'MAPS', ['Gene'], ['Gene'], ['sid'], ['sid', 'taxid'], default_props={'source': 'ncbigene'}) def run_with_mounted_arguments(self): self.run(self.taxid) def run(self, taxid): log.info(f"Run {self.__class__.__name__}") ncbigene_instance = self.get_instance_by_name('NcbiGene') # get org specific gene_info file if available if taxid in TAXID_SPECIFIC_GENEINFO: gene_info_file = ncbigene_instance.get_file( TAXID_SPECIFIC_GENEINFO[taxid]) else: gene_info_file = ncbigene_instance.get_file('gene_info.gz') log.info(gene_info_file) self.parse_gene_info(gene_info_file, taxid) def parse_gene_info(self, gene_info_file, taxid): # check sets check_ids = set() check_ids_symbols = set() with gzip.open(gene_info_file, 'rt') as f: header = next(f) # account for different formatting of header line (leading "#Format: " or not) if header.startswith('#Format:'): header_fields = tuple( header.split(':')[1].split('(') [0].rstrip().lstrip().split()) elif header.startswith('#tax'): header_fields = tuple(header[1:].strip().split('\t')) else: raise AttributeError( "File header was reformatted: {0}".format(header)) for l in f: flds = l.rstrip().split('\t') this_taxid = flds[0] if this_taxid == taxid: # (Gene) entrez_gene_id = flds[1] if entrez_gene_id not in check_ids: props = {'sid': entrez_gene_id, 'taxid': taxid} # update with all fields props.update(zip(header_fields, flds)) check_ids.add(entrez_gene_id) self.genes.add_node(props) # (GeneSymbol) and (GeneSymbol)-[SYNONYM]-(GeneSymbol) primary_symbol = flds[2] synonym_symbols = flds[4].split('|') # add primary symbol node if primary_symbol not in check_ids_symbols and primary_symbol != '-': check_ids_symbols.add(primary_symbol) self.genesymbols.add_node({ 'sid': primary_symbol, 'taxid': taxid }) for synonym in synonym_symbols: # GeneSymbol-[SYNONYM]-GeneSymbol self.genesymbol_synonym_genesymbol.add_relationship( { 'sid': synonym, 'taxid': taxid }, { 'sid': primary_symbol, 'taxid': taxid }, {}) if synonym not in check_ids_symbols and synonym != '-': check_ids_symbols.add(synonym) self.genesymbols.add_node({ 'sid': synonym, 'status': 'synonym', 'taxid': taxid }) # (Gene)-[MAPS]-(GeneSymbol) # primary self.gene_maps_genesymbol.add_relationship( {'sid': entrez_gene_id}, { 'sid': primary_symbol, 'taxid': taxid }, {'status': 'primary'}) # synonym for symbol in synonym_symbols: self.gene_maps_genesymbol.add_relationship( {'sid': entrez_gene_id}, { 'sid': symbol, 'taxid': taxid }, {'status': 'synonym'})
class RefseqEntityParser(ReturnParser): """ Extract mRNA/ncRNA transcripts from RefSeq sequence catalog. The file contains one sequence item per line. Sequences can be nucleotides (DNA, RNA) and Proteins/Peptides. The prefix of the ID describes the type: NM_: mRNA (measured) XM_: mRNA (predicted) NR_: ncRNA (measured) XR_: ncRNA (predicted) NP_: Protein (measured) XP_: Protein (predicted) Example line: TaxID, organism, ID, taxonomy, status, length 9606 H**o sapiens NM_000035.3 complete|vertebrate_mammalian REVIEWED 2426 """ def __init__(self): super(RefseqEntityParser, self).__init__() # arguments self.arguments = ['taxid'] # define NodeSet and RelationshipSet self.transcripts = NodeSet(['Transcript'], merge_keys=['sid'], default_props={'source': 'refseq'}) self.proteins = NodeSet(['Protein'], merge_keys=['sid'], default_props={'source': 'refseq'}) def run_with_mounted_arguments(self): self.run(self.taxid) def run(self, taxid): refseq_instance = self.get_instance_by_name('Refseq') refseq_catalog_file = refseq_instance.datasource.get_catalog_file_path(refseq_instance) datasource_name = refseq_instance.datasource.name with gzip.open(refseq_catalog_file, 'rt') as f: check_transcripts = set() check_proteins = set() for l in f: flds = l.rstrip().split('\t') if len(flds) >= 6: this_taxid = flds[0] if this_taxid == taxid: refseq_acc, version = flds[2].split('.') status = flds[4] length = flds[5] # split by type # transcript if refseq_acc.startswith('NM') or refseq_acc.startswith('NR') or refseq_acc.startswith( "XM") or refseq_acc.startswith("XR"): if refseq_acc not in check_transcripts: self.transcripts.add_node( {'sid': refseq_acc, 'version': version, 'status': status, 'length': length, 'taxid': taxid} ) check_transcripts.add(refseq_acc) # protein if refseq_acc.startswith('NP') or refseq_acc.startswith('XP'): if refseq_acc not in check_proteins: self.proteins.add_node( {'sid': refseq_acc, 'version': version, 'status': status, 'length': length, 'taxid': taxid}) check_proteins.add(refseq_acc)
class RefseqRemovedRecordsParser(ReturnParser): """ Parse all removed records from the removed_records files. The mappings to gene IDs are in the accession2geneid of the *previous* release. Simple approach is to collect all files from all previous releases and collect all records (relationships are filtered locally). """ def __init__(self): super(RefseqRemovedRecordsParser, self).__init__() self.arguments = ['taxid'] self.legacy_ids = set() self.legacy_transcripts = NodeSet(['Transcript', 'Legacy'], merge_keys=['sid'], default_props={'source': 'refseq'}) self.legacy_transcript_now_transcript = RelationshipSet('REPLACED_BY', ['Transcript'], ['Transcript'], ['sid'], ['sid'], default_props={'source': 'refseq'}) self.legacy_proteins = NodeSet(['Protein', 'Legacy'], merge_keys=['sid'], default_props={'source': 'refseq'}) self.legacy_protein_now_protein = RelationshipSet('REPLACED_BY', ['Protein'], ['Protein'], ['sid'], ['sid'], default_props={'source': 'refseq'}) self.gene_codes_legacy_transcript = RelationshipSet('CODES', ['Gene'], ['Transcript', 'Legacy'], ['sid'], ['sid'], default_props={'source': 'refseq'}) self.legacy_transcript_codes_protein = RelationshipSet('CODES', ['Transcript', 'Legacy'], ['Protein'], ['sid'], ['sid'], default_props={'source': 'refseq'}) def run_with_mounted_arguments(self): self.run(self.taxid) def run(self, taxid): # get the nodes first, this also creates a set of all legacy IDs self.get_legacy_nodes(taxid) # then get the relationnships to gene IDs, this uses the set of legacy IDs to not recreate existing relationships self.get_legacy_gene_rels(taxid) def get_legacy_nodes(self, taxid): """ ========================================== release#.removed-records ========================================== Content: Tab-delimited report of records that were included in the previous release but are not included in the current release. Columns: 1. taxonomy ID 2. species name 3. accession.version 4. refseq release directory accession is included in complete + other directories '|' delimited 5. refseq status na - not available; status codes are not applied to most genomic records INFERRED PREDICTED PROVISIONAL VALIDATED REVIEWED MODEL UNKNOWN - status code not provided; however usually is provided for this type of record 6. length 7. removed status dead protein: protein was removed when genomic record was reloaded and protein was not found on the nucleotide update. This is an implied permanent suppress. temporarily suppressed: record was temporarily removed and may be restored at a later date. permanently suppressed: record was permanently removed. It is possible to restore this type of record however at the time of removal that action is not anticipated. replaced by accession: the accession in column 3 has become a secondary accession that cited in column 8. :param taxid: :return: """ refseq_instance = self.get_instance_by_name('Refseq') removed_records_files = refseq_instance.find_files(lambda x: 'removed-records' in x and x.endswith('.gz')) for file in removed_records_files: log.debug(f"Parse {file}") release = file.split('/')[-1].split('.')[0].replace('release', '') with gzip.open(file, 'rt') as f: for l in f: flds = l.strip().split('\t') this_taxid = flds[0] if this_taxid == taxid: refseq_acc, version = flds[2].split('.') reason = flds[-1] # transcript if refseq_acc.startswith('NM') or refseq_acc.startswith('NR') or refseq_acc.startswith( "XM") or refseq_acc.startswith("XR"): if refseq_acc not in self.legacy_ids: self.legacy_transcripts.add_node( {'sid': refseq_acc, 'version': version, 'status': 'removed', 'removed_in': release, 'reason': reason, 'taxid': taxid} ) self.legacy_ids.add(refseq_acc) if 'replaced by' in reason: # replaced by NM_022375 -> NM_022375 new_accession = (reason.rsplit(' ', 1)[1]).split('.')[0] self.legacy_transcript_now_transcript.add_relationship( {'sid': refseq_acc}, {'sid': new_accession}, {} ) # protein if refseq_acc.startswith('NP') or refseq_acc.startswith('XP'): if refseq_acc not in self.legacy_ids: self.legacy_proteins.add_node( {'sid': refseq_acc, 'version': version, 'status': 'removed', 'removed_in': release, 'reason': reason, 'taxid': taxid}) self.legacy_ids.add(refseq_acc) if 'replaced by' in reason: # replaced by NM_022375 -> NM_022375 new_accession = (reason.rsplit(' ', 1)[1]).split('.')[0] self.legacy_protein_now_protein.add_relationship( {'sid': refseq_acc}, {'sid': new_accession}, {} ) def get_legacy_gene_rels(self, taxid): """ Get the gene/protein relationships for the legacy Transcripts. ========================================== release#.accession2geneid ========================================== Content: Report of GeneIDs available at the time of the RefSeq release. Limited to GeneIDs that are associated with RNA or mRNA records with accession prefix N[M|R] and X[M|R]. Columns (tab delimited): 1: Taxonomic ID 2: Entrez GeneID 3: Transcript accession.version 4: Protein accession.version na if no data --for example, the NR_ accession prefix is used for RNA so there is no corresponding protein record :param taxid: :return: """ log.debug("Get relationships from legacy RefSeq IDs to genes.") refseq_instance = self.get_instance_by_name('Refseq') archived_accession2geneid = refseq_instance.find_files(lambda x: 'accession2geneid' in x and x.endswith('.gz')) check_set = set() for file in archived_accession2geneid: log.debug(f"Parse {file}") with gzip.open(file, 'rt') as f: for l in f: flds = l.strip().split('\t') this_taxid = flds[0] if this_taxid == taxid: """ 1: Taxonomic ID 2: Entrez GeneID 3: Transcript accession.version 4: Protein accession.version na if no data --for example, the NR_ accession prefix is used for RNA so there is no corresponding protein record """ gene_id = flds[1].strip() transcript_accession = flds[2].strip().split('.')[0] protein_accession = flds[3].strip().split('.')[0] if transcript_accession in self.legacy_ids: if (gene_id, transcript_accession) not in check_set: self.gene_codes_legacy_transcript.add_relationship( {'sid': gene_id}, {'sid': transcript_accession}, {} ) check_set.add((gene_id, transcript_accession)) if transcript_accession != 'na': if (transcript_accession, protein_accession) not in check_set: self.legacy_transcript_codes_protein.add_relationship( {'sid': transcript_accession}, {'sid': protein_accession}, {} ) check_set.add((transcript_accession, protein_accession))
class GtexMetadataParser(ReturnParser): def __init__(self): """ :param mesh_instance: NcbiGene Instance :type mesh_instance: DataSourceInstance """ super(GtexMetadataParser, self).__init__() # NodeSets self.tissues = NodeSet(['GtexTissue'], merge_keys=['name']) self.detailed_tissues = NodeSet(['GtexDetailedTissue'], merge_keys=['name']) self.sample = NodeSet(['GtexSample'], merge_keys=['sid']) self.sample_measures_tissue = RelationshipSet('MEASURES', ['GtexSample'], ['GtexTissue'], ['sid'], ['name']) self.sample_measures_detailed_tissue = RelationshipSet( 'MEASURES', ['GtexSample'], ['GtexDetailedTissue'], ['sid'], ['name']) self.tissue_parent_detailed_tissue = RelationshipSet( 'PARENT', ['GtexTissue'], ['GtexDetailedTissue'], ['name'], ['name']) self.tissue_parent_detailed_tissue.unique = True def run_with_mounted_arguments(self): self.run() def run(self): gtex_instance = self.get_instance_by_name('Gtex') gtext_sample_attribute_file = gtex_instance.get_file( 'GTEx_Analysis_v8_Annotations_SampleAttributesDS.txt') gtex_df = pandas.read_csv(gtext_sample_attribute_file, sep='\t', header=0, index_col=False, encoding="utf-8-sig") for row in gtex_df.itertuples(): sid = row.SAMPID tissue_name = row.SMTS detailed_tissue_name = row.SMTSD props = { 'sid': sid, 'SMATSSCR': row.SMATSSCR, 'SMCENTER': row.SMCENTER, 'SMPTHNTS': row.SMPTHNTS, 'SMRIN': row.SMRIN, 'SMTS': row.SMTS, 'SMTSD': row.SMTSD, 'SMUBRID': row.SMUBRID, 'SMTSISCH': row.SMTSISCH, 'SMTSPAX': row.SMTSPAX, 'SMNABTCH': row.SMNABTCH, 'SMNABTCHT': row.SMNABTCHT, 'SMNABTCHD': row.SMNABTCHD, 'SMGEBTCH': row.SMGEBTCH, 'SMGEBTCHD': row.SMGEBTCHD, 'SMGEBTCHT': row.SMGEBTCHT, 'SMAFRZE': row.SMAFRZE, 'SMGTC': row.SMGTC, 'SME2MPRT': row.SME2MPRT, 'SMCHMPRS': row.SMCHMPRS, 'SMNTRART': row.SMNTRART, 'SMNUMGPS': row.SMNUMGPS, 'SMMAPRT': row.SMMAPRT, 'SMEXNCRT': row.SMEXNCRT, 'SM550NRM': row.SM550NRM, 'SMGNSDTC': row.SMGNSDTC, 'SMUNMPRT': row.SMUNMPRT, 'SM350NRM': row.SM350NRM, 'SMRDLGTH': row.SMRDLGTH, 'SMMNCPB': row.SMMNCPB, 'SME1MMRT': row.SME1MMRT, 'SMSFLGTH': row.SMSFLGTH, 'SMESTLBS': row.SMESTLBS, 'SMMPPD': row.SMMPPD, 'SMNTERRT': row.SMNTERRT, 'SMRRNANM': row.SMRRNANM, 'SMRDTTL': row.SMRDTTL, 'SMVQCFL': row.SMVQCFL, 'SMMNCV': row.SMMNCV, 'SMTRSCPT': row.SMTRSCPT, 'SMMPPDPR': row.SMMPPDPR, 'SMCGLGTH': row.SMCGLGTH, 'SMGAPPCT': row.SMGAPPCT, 'SMUNPDRD': row.SMUNPDRD, 'SMNTRNRT': row.SMNTRNRT, 'SMMPUNRT': row.SMMPUNRT, 'SMEXPEFF': row.SMEXPEFF, 'SMMPPDUN': row.SMMPPDUN, 'SME2MMRT': row.SME2MMRT, 'SME2ANTI': row.SME2ANTI, 'SMALTALG': row.SMALTALG, 'SME2SNSE': row.SME2SNSE, 'SMMFLGTH': row.SMMFLGTH, 'SME1ANTI': row.SME1ANTI, 'SMSPLTRD': row.SMSPLTRD, 'SMBSMMRT': row.SMBSMMRT, 'SME1SNSE': row.SME1SNSE, 'SME1PCTS': row.SME1PCTS, 'SMRRNART': row.SMRRNART, 'SME1MPRT': row.SME1MPRT, 'SMNUM5CD': row.SMNUM5CD, 'SMDPMPRT': row.SMDPMPRT, 'SME2PCTS': row.SME2PCTS } self.sample.add_node(props) self.tissues.add_unique({'name': tissue_name}) self.detailed_tissues.add_unique({'name': detailed_tissue_name}) self.sample_measures_tissue.add_relationship({'sid': sid}, {'name': tissue_name}, {}) self.sample_measures_detailed_tissue.add_relationship( {'sid': sid}, {'name': detailed_tissue_name}, {}) self.tissue_parent_detailed_tissue.add_relationship( {'name': tissue_name}, {'name': detailed_tissue_name}, {})
class HmdbParser(ReturnParser): def __init__(self): super(HmdbParser, self).__init__() # NodeSets self.metabolites = NodeSet(['Metabolite'], merge_keys=['sid'], default_props={'source': 'hmdb'}) self.metabolite_map_metabolite = RelationshipSet( 'MAPS', ['Metabolite'], ['Metabolite'], ['sid'], ['sid'], default_props={'source': 'hmdb'}) self.metabolite_associates_protein = RelationshipSet( 'HAS_ASSOCIATION', ['Metabolite'], ['Protein'], ['sid'], ['sid'], default_props={'source': 'hmdb'}) def run_with_mounted_arguments(self): self.run() def run(self, *args, **kwargs): hmdb_instance = self.get_instance_by_name('Hmdb') all_metabolites_file = hmdb_instance.get_file('hmdb_metabolites.xml') all_metabolites = etree.parse(all_metabolites_file) for metabolite in all_metabolites.getroot(): # TODO just iterate over property list, this code snippet was copied from manually testing stuff in Spyder # TODO filter empty properties sid = metabolite.findtext('{http://www.hmdb.ca}accession') name = metabolite.findtext('{http://www.hmdb.ca}name') chebi_id = metabolite.findtext('{http://www.hmdb.ca}chebi_id') chemspider_id = metabolite.findtext( '{http://www.hmdb.ca}chemspider_id') cs_description = metabolite.findtext( '{http://www.hmdb.ca}cs_description') description = metabolite.findtext( '{http://www.hmdb.ca}description') chemical_formula = metabolite.findtext( '{http://www.hmdb.ca}chemical_formula') average_molecular_weight = metabolite.findtext( '{http://www.hmdb.ca}average_molecular_weight') iupac_name = metabolite.findtext('{http://www.hmdb.ca}iupac_name') cas_registry_number = metabolite.findtext( '{http://www.hmdb.ca}cas_registry_number') smiles = metabolite.findtext('{http://www.hmdb.ca}smiles') inchi = metabolite.findtext('{http://www.hmdb.ca}inchi') kegg_id = metabolite.findtext('{http://www.hmdb.ca}kegg_id') metabolite_properties = { 'sid': sid, 'name': name, 'chebi_id': chebi_id, 'chemspider_id': chemspider_id, 'cs_description': cs_description, 'description': description, 'chemical_formula': chemical_formula, 'average_molecular_weight': average_molecular_weight, 'iupac_name': iupac_name, 'cas_registry_number': cas_registry_number, 'smiles': smiles, 'inchi': inchi, 'kegg_id': kegg_id } self.metabolites.add_node(metabolite_properties) # add mapping to Chebi if chebi_id: self.metabolite_map_metabolite.add_relationship( {'sid': sid}, {'sid': chebi_id}, {}) # add association to Proteins for protein in metabolite.find( '{http://www.hmdb.ca}protein_associations'): uniprot_id = protein.findtext('{http://www.hmdb.ca}uniprot_id') self.metabolite_associates_protein.add_relationship( {'sid': sid}, {'sid': uniprot_id}, {})
class LncipediaParser(ReturnParser): """ Parse Lncipedia GFF file. 'lnc_RNA' entries contain gene and transcript IDs as well as mappings to ENSEMBL 'exon' entries don't have different IDs, they reuse the gene/transcript IDs from their parent 'lnc_RNA' entries chr16 lncipedia.org lnc_RNA 52005479 52026435 . - . ID=lnc-TOX3-1:20;gene_id=lnc-TOX3-1;transcript_id=lnc-TOX3-1:20;gene_alias_1=XLOC_011939;gene_alias_2=linc-SALL1-6;transcript_alias_1=TCONS_00025002;transcript_alias_2=NONHSAT142490; chr10 lncipedia.org exon 8052243 8052735 . - . Parent=GATA3-AS1:5;gene_id=GATA3-AS1;transcript_id=GATA3-AS1:5;gene_alias_1=XLOC_008724;gene_alias_2=linc-KIN-5;gene_alias_3=ENSG00000243350;gene_alias_4=RP11-379F12.3;gene_alias_5=ENSG00000243350.1;gene_alias_6=OTTHUMG00000017641.1;gene_alias_7=ENSG00000197308.9;gene_alias_8=GATA3-AS1;transcript_alias_1=TCONS_00017730;transcript_alias_2=ENST00000458727;transcript_alias_3=ENST00000458727.1;transcript_alias_4=RP11-379F12.3-001;transcript_alias_5=OTTHUMT00000046722.1;transcript_alias_6=NONHSAT011314;transcript_alias_7=NR_104327;transcript_alias_8=NR_104327.1; """ def __init__(self): super(LncipediaParser, self).__init__() self.genes = NodeSet(['Gene'], merge_keys=['sid']) self.transcripts = NodeSet(['Transcript'], merge_keys=['sid']) self.gene_codes_transcripts = RelationshipSet('CODES', ['Gene'], ['Transcript'], ['sid'], ['sid']) self.gene_maps_gene = RelationshipSet('MAPS', ['Gene'], ['Gene'], ['sid'], ['sid']) self.transcript_maps_transcript = RelationshipSet('MAPS', ['Transcript'], ['Transcript'], ['sid'], ['sid']) def run_with_mounted_arguments(self): self.run() def run(self): log.debug(f"Run {self.__class__.__name__}") lncipedia_instance = self.get_instance_by_name('Lncipedia') lncipedia_datasource_name = lncipedia_instance.datasource.name gff_file = lncipedia_instance.get_file('lncipedia_5_2_hg38.gff') annotation = GffReader(gff_file) check_ids = set() for r in annotation.records: if r.type == 'lnc_RNA': # create gene gene_id = r.attributes['gene_id'] if gene_id not in check_ids: self.genes.add_node({'sid': gene_id, 'source': lncipedia_datasource_name}) check_ids.add(gene_id) transcript_id = r.attributes['transcript_id'] if transcript_id not in check_ids: self.transcripts.add_node({'sid': transcript_id, 'source': lncipedia_datasource_name}) check_ids.add(transcript_id) if frozenset((gene_id, transcript_id)) not in check_ids: self.gene_codes_transcripts.add_relationship( {'sid': gene_id}, {'sid': transcript_id}, {} ) check_ids.add(frozenset((gene_id, transcript_id))) for k,v in r.attributes.items(): if k.startswith('gene_alias'): ref_gene_id = v.split('.')[0] # don't create MAPS relationship if same name like mapped entity if gene_id != ref_gene_id: if frozenset((gene_id, ref_gene_id)) not in check_ids: self.gene_maps_gene.add_relationship( {'sid': gene_id}, {'sid': ref_gene_id}, {'source': lncipedia_datasource_name} ) check_ids.add(frozenset((gene_id, ref_gene_id))) if k.startswith('transcript_alias'): ref_transcript_id = v.split('.')[0] # don't create MAPS relationship if same name like mapped entity if transcript_id != ref_transcript_id: if frozenset((transcript_id, ref_transcript_id)) not in check_ids: self.transcript_maps_transcript.add_relationship( {'sid': transcript_id}, {'sid': ref_transcript_id}, {'source': lncipedia_datasource_name} ) check_ids.add(frozenset((transcript_id, ref_transcript_id)))
class MeshParser(ReturnParser): def __init__(self): super(MeshParser, self).__init__() # NodeSets self.descriptor = NodeSet(['MeshDescriptor'], merge_keys=['sid']) self.qualifier = NodeSet(['MeshQualifier'], merge_keys=['sid']) self.concept = NodeSet(['MeshConcept'], merge_keys=['sid']) self.term = NodeSet(['MeshTerm'], merge_keys=['sid']) self.descriptor_allowed_qualifier = RelationshipSet('ALLOWED', ['MeshDescriptor'], ['MeshQualifier'], ['sid'], ['sid']) self.descriptor_has_concept = RelationshipSet('HAS', ['MeshDescriptor'], ['MeshConcept'], ['sid'], ['sid']) self.descriptor_has_concept.unique = True self.concept_has_term = RelationshipSet('HAS', ['MeshConcept'], ['MeshTerm'], ['sid'], ['sid']) self.concept_has_term.unique = True self.concept_related_concept = RelationshipSet('RELATED', ['MeshConcept'], ['MeshConcept'], ['sid'], ['sid']) self.concept_related_concept.unique = True def run_with_mounted_arguments(self): self.run() def run(self): self.parse_xml() def parse_xml(self): """ Parse descriptor XML file. """ mesh_instance = self.get_instance_by_name('Mesh') version = DataSourceVersion.version_from_string( mesh_instance.version ) descriptor_xml = mesh_instance.get_file('desc{}.xml'.format(str(version))) log.debug("XML file {}".format(descriptor_xml)) tree = ET.parse(descriptor_xml) root = tree.getroot() check_qualifier = set() check_concepts = set() check_terms = set() for descriptor_record in root.getchildren(): descriptor_ui = descriptor_record.find('DescriptorUI').text # <DescriptorName> # <String>Calcimycin</String> # </DescriptorName> descriptor_name = descriptor_record.find('.DescriptorName/String').text self.descriptor.add_node({'sid': descriptor_ui, 'name': descriptor_name}) # <AllowableQualifiersList> # <AllowableQualifier> # <QualifierReferredTo> # <QualifierUI>Q000302</QualifierUI> # <QualifierName> # <String>isolation & purification</String> # </QualifierName> # </QualifierReferredTo> # <Abbreviation>IP</Abbreviation> # </AllowableQualifier> # </AllowableQualifiersList> allowed_qualifiers = descriptor_record.findall( '.AllowableQualifiersList/AllowableQualifier/QualifierReferredTo') for qualifier in allowed_qualifiers: qualifier_ui = qualifier.find('.QualifierUI').text # add qualifier node id not exists if qualifier_ui not in check_qualifier: qualifier_name = qualifier.find('.QualifierName/String').text self.qualifier.add_node({'sid': qualifier_ui, 'name': qualifier_name}) check_qualifier.add(qualifier_ui) # add descriptor -> qualifier relationship self.descriptor_allowed_qualifier.add_relationship( {'sid': descriptor_ui}, {'sid': qualifier_ui}, {'source': 'mesh'} ) # <ConceptList> # <Concept PreferredConceptYN="Y"> # <ConceptUI>M0000001</ConceptUI> # <ConceptName> # <String>Calcimycin</String> # </ConceptName> # <CASN1Name>4-Benzoxazolecarboxylic acid, 5-(methylamino)-2-((3,9,11-trimethyl-8-(1-methyl-2-oxo-2-(1H-pyrrol-2-yl)ethyl)-1,7-dioxaspiro(5.5)undec-2-yl)methyl)-, (6S-(6alpha(2S*,3S*),8beta(R*),9beta,11alpha))-</CASN1Name> # <RegistryNumber>37H9VM9WZL</RegistryNumber> # <ScopeNote>An ionophorous, polyether antibiotic from Streptomyces chartreusensis. It binds and transports CALCIUM and other divalent cations across membranes and uncouples oxidative phosphorylation while inhibiting ATPase of rat liver mitochondria. The substance is used mostly as a biochemical tool to study the role of divalent cations in various biological systems. # </ScopeNote> # <RelatedRegistryNumberList> # <RelatedRegistryNumber>52665-69-7 (Calcimycin)</RelatedRegistryNumber> # </RelatedRegistryNumberList> # <ConceptRelationList> # <ConceptRelation RelationName="NRW"> # <Concept1UI>M0000001</Concept1UI> # <Concept2UI>M0353609</Concept2UI> # </ConceptRelation> # </ConceptRelationList> # <TermList> # <Term ConceptPreferredTermYN="Y" IsPermutedTermYN="N" LexicalTag="NON" RecordPreferredTermYN="Y"> # <TermUI>T000002</TermUI> # <String>Calcimycin</String> # <DateCreated> # <Year>1999</Year> # <Month>01</Month> # <Day>01</Day> # </DateCreated> # <ThesaurusIDlist> # <ThesaurusID>FDA SRS (2014)</ThesaurusID> # <ThesaurusID>NLM (1975)</ThesaurusID> # </ThesaurusIDlist> # </Term> # </TermList> # </Concept> concepts = descriptor_record.findall('.ConceptList/Concept') for concept in concepts: preferred_concept = concept.attrib['PreferredConceptYN'] concept_ui = concept.find('.ConceptUI').text # concept node if not exists if concept_ui not in check_concepts: concept_properties = {} concept_properties['sid'] = concept_ui concept_properties['name'] = concept.find('.ConceptName/String').text try: concept_properties['scope_note'] = concept.find('.ScopeNote').text except AttributeError as e: pass self.concept.add_node(concept_properties) check_concepts.add(concept_ui) # (Descriptor)--(Concept) relation self.descriptor_has_concept.add_relationship({'sid': descriptor_ui}, {'sid': concept_ui}, {'preferred': preferred_concept}) # concept relations for concept_relation in concept.findall('.ConceptRelationList/ConceptRelation'): left = concept_relation.find('.Concept1UI').text right = concept_relation.find('.Concept2UI').text name = concept_relation.attrib['RelationName'] self.concept_related_concept.add_relationship({'sid': left}, {'sid': right}, {'name': name}) # iterate Terms for concept for term in concept.findall('.TermList/Term'): term_ui = term.find('TermUI').text concept_preferred_term = term.attrib['ConceptPreferredTermYN'] # Term node if not exists if term_ui not in check_terms: term_name = term.find('.String').text self.term.add_node({'sid': term_ui, 'name': term_name}) check_terms.add(term_ui) # (Concept)--(Term) self.concept_has_term.add_relationship({'sid': concept_ui}, {'sid': term_ui}, {'preferred': concept_preferred_term})
class MirbaseParser(ReturnParser): def __init__(self): super(MirbaseParser, self).__init__() # NodeSets self.precursor_mirna = NodeSet(['PrecursorMirna'], merge_keys=['sid']) self.mature_mirna = NodeSet(['Mirna'], merge_keys=['sid']) # RelationshipSets self.precursor_codes_mature = RelationshipSet('PRE', ['PrecursorMirna'], ['Mirna'], ['sid'], ['sid']) self.transcript_codes_precursor = RelationshipSet( 'IS', ['Transcript'], ['PrecursorMirna'], ['sid'], ['sid']) self.gene_is_precursor = RelationshipSet('IS', ['Gene'], ['PrecursorMirna'], ['sid'], ['sid']) def run_with_mounted_arguments(self): self.run() def run(self): self.get_mature_mirnas() self.get_pre_mirnas() self.get_pre_mature_relationship() self.get_pre_transcript_relationships() self.get_gene_pre_relationships() @property def mirbase_instance(self): return self.get_instance_by_name('Mirbase') @property def pre_mirna_df(self): """ Ge precursor miRNA DataFrame from mirna.txt.gz mir_acc, mir_id, prev_mir_id, desc, sequence, comment, organism_key, dead_flag """ precursor_mirna_table_file = self.mirbase_instance.get_file( 'mirna.txt.gz') pre_mirs_df = pandas.read_csv(precursor_mirna_table_file, sep='\t', index_col=0, header=None) pre_mirs_df.columns = [ 'mir_acc', 'mir_id', 'prev_mir_id', 'desc', 'sequence', 'comment', 'organism_key', 'dead_flag' ] return pre_mirs_df @property def mature_mirna_df(self): """ Get mature miRNA DataFrame from: mirna_mature.txt.gz name, prev_name, mir_acc, evidence, ref, similarity, dead_flag """ mirna_table_file = self.mirbase_instance.get_file( 'mirna_mature.txt.gz') mirnas_df = pandas.read_csv(mirna_table_file, sep='\t', index_col=0, header=None) mirnas_df.columns = [ 'name', 'prev_name', 'mir_acc', 'evidence', 'ref', 'similarity', 'dead_flag' ] return mirnas_df @property def context_df(self): """ Get context DataFrame from: mirna_context.txt.gz auto_mirna transcript_id overlap_sense overlap_type number transcript_source transcript_name 64777 ENST00000545242 + intron 15 HGNC_trans_name ABLIM2-203 """ mirna_context_file = self.mirbase_instance.get_file( 'mirna_context.txt.gz') mirna_context_df = pandas.read_csv(mirna_context_file, sep='\t', index_col=0, header=None) mirna_context_df.columns = [ 'transcript_id', 'overlap_sense', 'overlap_type', 'number', 'transcript_source', 'transcript_name' ] return mirna_context_df @property def mirna_database_url_df(self): """ Database list from: mirna_database_url.txt.gz `auto_db`, `display_name`, `url` 5 EntrezGene https://www.ncbi.nlm.nih.gov/gene/<?> """ file = self.mirbase_instance.get_file('mirna_database_url.txt.gz') df = pandas.read_csv(file, sep='\t', index_col=0, header=None) df.columns = ['display_name', 'url'] return df @property def mirna_database_link_df(self): """ Database links from: mirna_database_links.txt.gz 'auto_mirna', 'auto_db', 'link', 'display_name' 64744 5 406883 MIRLET7A3 """ file = self.mirbase_instance.get_file('mirna_database_links.txt.gz') df = pandas.read_csv(file, sep='\t', index_col=0, header=None) df.columns = ['auto_db', 'link', 'display_name'] return df def get_mature_mirnas(self): """ Mature miRNAs are stored in a single table: mirna_mature.txt.gz name, prev_name, mir_acc, evidence, ref, similarity, dead_flag """ for row in self.mature_mirna_df.itertuples(): # add node self.mature_mirna.add_node({ 'sid': row.mir_acc, 'name': row.name, 'evidence': row.evidence }) def get_pre_mirnas(self): """ Precursor miRNAs are stored in a table: mirna.txt.gz mir_acc, mir_id, prev_mir_id, desc, sequence, comment, organism_key Organism identifier for the precursor miRNAs are in another table: mirna_species.txt.gz organism, division, org_name, taxonomy, genome_assembly, genome_accession, ensembl_db `org_name` is the long name of the organism, there is no taxonomy ID. :return: List of precursor miRNAs :rtype: list[Entity] """ organism_table_file = self.mirbase_instance.get_file( 'mirna_species.txt.gz') # load pre-miRNA table # load organism table orgs_df = pandas.read_csv(organism_table_file, sep='\t', index_col=0, header=None) orgs_df.columns = [ 'organism', 'division', 'org_name', 'taxon_id', 'taxonomy', 'genome_assembly', 'genome_accession', 'ensembl_db' ] merged_pre_mirs_org_df = pandas.merge(self.pre_mirna_df, orgs_df, on=None, left_on='organism_key', right_index=True) # add precursor miRNA nodes for row in merged_pre_mirs_org_df.itertuples(): props = { 'sid': row.mir_acc, 'name': row.mir_id, 'desc': row.desc, 'sequence': row.sequence, 'taxid': row.taxon_id, 'comment': str(row.comment) } self.precursor_mirna.add_node(props) def get_pre_mature_relationship(self): """ Mature miRNAs and precursor miRNAs are in the same files described in the respective parser function (above). Mapping is stored in a mapping table: mirna_pre_mature.txt.gz pre_dbid, mature_dbid, start, end It contains the primary key of mature and precursor miRNA tables and the start/end of the mature sequence within the precursor. :return: List of relationships between mature and precursor miRNAs :rtype: list[Relationship] """ mapping_table_file = self.mirbase_instance.get_file( 'mirna_pre_mature.txt.gz') # collect db_primary_key -> mirBase accession to later parse # the mature/precursor mapping table precursor_db_key_2_accession = {} for row in self.pre_mirna_df.itertuples(): precursor_db_key_2_accession[row.Index] = row.mir_acc # mature miRNAs # get miRNAs from miRNAs table first # they are mapped to pre-miRNAs with a mapping table that contains the position of the mature sequence # organims is also only stored for pre-miRNA # collect db_primary_key -> mirBase accession to later parse # the mature/precursor mapping table mature_db_key_2_accession = {} for row in self.mature_mirna_df.itertuples(): mature_db_key_2_accession[row.Index] = row.mir_acc # parse mappings # get mapping table pre_2_mature_df = pandas.read_csv(mapping_table_file, sep='\t', index_col=False, header=None) pre_2_mature_df.columns = ['pre_dbid', 'mature_dbid', 'start', 'end'] # iterate over mapping table and create relationships for row in pre_2_mature_df.itertuples(): mature_acc = mature_db_key_2_accession[row.mature_dbid] precursor_acc = precursor_db_key_2_accession[row.pre_dbid] self.precursor_codes_mature.add_relationship( {'sid': precursor_acc}, {'sid': mature_acc}, { 'start': int(row.start), 'end': int(row.end) }) def get_pre_transcript_relationships(self): """ MirBase provides the transcriptional context based on ENSEMBL transcipts. Context is stored in a single file: mirna_context.txt.gz auto_mirna transcript_id overlap_sense overlap_type number transcript_source transcript_name 64777 ENST00000545242 + intron 15 HGNC_trans_name ABLIM2-203 For mapping the auto_mirna KEY we need the precursor miRNAs from: mirna.txt.gz """ pre_2_context = pandas.merge(self.context_df, self.pre_mirna_df, how='left', on=None, left_index=True, right_index=True) for row in pre_2_context.itertuples(): self.transcript_codes_precursor.add_relationship( {'sid': row.transcript_id}, {'sid': row.mir_acc}, { 'overlap_type': row.overlap_type, 'number': row.number }) def get_gene_pre_relationships(self): """ Parse relationships from Gene to Mirna. MiRBase provides links to external databases in a table:mirna_database_links.txt.gz `auto_mirna`, `auto_db`, `link`, `display_name` 64743 5 406882 MIRLET7A2 auto_mirna is the mirna KEY from: mirna.txt.gz auto_db is the DB KEY from: mirna_database_url.txt.gz 5 EntrezGene https://www.ncbi.nlm.nih.gov/gene/<?> Example line: 64743 ENTREZGENE 406882 MIRLET7A2 :return: """ gene_pre_df = pandas.merge(self.mirna_database_link_df, self.mirna_database_url_df, how='left', left_on='auto_db', right_index=True) print(len(self.mirna_database_link_df), len(self.mirna_database_url_df)) print(len(gene_pre_df)) final_merge = pandas.merge(gene_pre_df, self.pre_mirna_df, how='left', on=None, left_index=True, right_index=True) for row in final_merge.itertuples(): if row.display_name_y == 'EntrezGene': self.gene_is_precursor.add_relationship( {'sid': row.link}, {'sid': row.mir_acc}, {'source': self.mirbase_instance.datasource.name})
class EnsemblEntityParser(ReturnParser): def __init__(self): """ :param ensembl_instance: The ENSEMBL DataSource instance. """ super(EnsemblEntityParser, self).__init__() # arguments self.arguments = ['taxid'] # NodeSets self.genes = NodeSet(['Gene'], merge_keys=['sid'], default_props={'source': 'ensembl'}) self.transcripts = NodeSet(['Transcript'], merge_keys=['sid'], default_props={'source': 'ensembl'}) self.proteins = NodeSet(['Protein'], merge_keys=['sid'], default_props={'source': 'ensembl'}) # RelationshipSets self.gene_codes_transcript = RelationshipSet( 'CODES', ['Gene'], ['Transcript'], ['sid'], ['sid'], default_props={'source': 'ensembl'}) self.transcript_codes_protein = RelationshipSet( 'CODES', ['Transcript'], ['Protein'], ['sid'], ['sid'], default_props={'source': 'ensembl'}) def run_with_mounted_arguments(self): self.run(self.taxid) def run(self, taxid): ensembl_instance = self.get_instance_by_name('Ensembl') datasource_name = ensembl_instance.datasource.name # try patched path, if not available take flat ensembl_gtf_file_path = Ensembl.get_gtf_file_path(taxid, ensembl_instance, patched=True) if not os.path.exists(ensembl_gtf_file_path): ensembl_gtf_file_path = Ensembl.get_gtf_file_path(taxid, ensembl_instance, patched=False) annotation = GffReader(ensembl_gtf_file_path) check_gene_ids = set() check_transcript_ids = set() check_protein_ids = set() check_gene_transcript_rels = set() check_transcript_protein_rels = set() log.info("Start parsing ENSEMBL gtf file, taxid {}, {}".format( taxid, ensembl_gtf_file_path)) for r in annotation.records: # add gene node gene_id = r.attributes['gene_id'] if gene_id not in check_gene_ids: props = { 'sid': gene_id, 'name': r.attributes['gene_name'], 'taxid': taxid } self.genes.add_node(props) check_gene_ids.add(gene_id) # add transcript node if r.type == 'transcript': transcript_id = r.attributes['transcript_id'] if transcript_id not in check_transcript_ids: props = {'sid': transcript_id, 'taxid': taxid} self.transcripts.add_node(props) check_transcript_ids.add(transcript_id) # add protein node if r.type == 'CDS': protein_id = r.attributes['protein_id'] if protein_id not in check_protein_ids: props = {'sid': protein_id, 'taxid': taxid} self.proteins.add_node(props) check_protein_ids.add(protein_id) # Gene-CODES-Transcript if r.type == 'transcript': transcript_id = r.attributes['transcript_id'] gene_id = r.attributes['gene_id'] # add gene-transcript rel if gene_id + transcript_id not in check_gene_transcript_rels: self.gene_codes_transcript.add_relationship( {'sid': gene_id}, {'sid': transcript_id}, {}) check_gene_transcript_rels.add(gene_id + transcript_id) # Transcript-CODES-Protein if r.type == 'CDS': protein_id = r.attributes['protein_id'] transcript_id = r.attributes['transcript_id'] # add transcript-protein rel if transcript_id + protein_id not in check_transcript_protein_rels: self.transcript_codes_protein.add_relationship( {'sid': transcript_id}, {'sid': protein_id}, {}) check_transcript_protein_rels.add(transcript_id + protein_id) log.info("Finished parsing ENSEMBL gtf file.")
for line in file: fields = line.strip().split('\t') ncbi_gene_id = fields[1] # get mapping to ENSEMBL Gene IDs mapped_ensembl_gene_ids = [] # get dbXrefs db_xrefs = fields[5] for mapped_element in db_xrefs.split('|'): if 'Ensembl' in mapped_element: ensembl_gene_id = mapped_element.split(':')[1] mapped_ensembl_gene_ids.append(ensembl_gene_id) # create nodes and relationships # add NCBI gene node ncbi_gene_nodes.add_node({'gene_id': ncbi_gene_id, 'db': 'ncbi'}) # add ENSEMBL gene nodes if they not exist already for ensembl_gene_id in mapped_ensembl_gene_ids: if ensembl_gene_id not in ensembl_gene_ids_added: ensembl_gene_nodes.add_node({ 'gene_id': ensembl_gene_id, 'db': 'ensembl' }) ensembl_gene_ids_added.add(ensembl_gene_id) # add (:Gene)-[:MAPS]->(:Gene) relationship for ensembl_gene_id in mapped_ensembl_gene_ids: gene_mapping_rels.add_relationship({'gene_id': ncbi_gene_id}, {'gene_id': ensembl_gene_id}, {'db': 'ncbi'})
class SwissLipidsParser(ReturnParser): def __init__(self): super(SwissLipidsParser, self).__init__() # define NodeSet and RelationshipSet self.lipids = NodeSet(['Lipid'], merge_keys=['sid']) self.lipid_fromclass_lipid = RelationshipSet('FROM_LIPID_CLASS', ['Lipid'], ['Lipid'], ['sid'], ['sid']) self.lipid_parent_lipid = RelationshipSet('HAS_PARENT', ['Lipid'], ['Lipid'], ['sid'], ['sid']) self.lipid_component_lipid = RelationshipSet('HAS_COMPONENT', ['Lipid'], ['Lipid'], ['sid'], ['sid']) self.lipid_maps_metabolite = RelationshipSet('MAPS', ['Lipid'], ['Metabolite'], ['sid'], ['sid']) self.lipid_associates_protein = RelationshipSet('HAS_ASSOCIATION', ['Lipid'], ['Protein'], ['sid'], ['sid']) def run_with_mounted_arguments(self): self.run() def run(self): swisslipids_instance = self.get_instance_by_name('SwissLipids') self.get_lipids(swisslipids_instance) self.get_lipid_to_protein(swisslipids_instance) def get_lipids(self, instance): """ Lipid ID Level Name Abbreviation* Synonyms* Lipid class* Parent Components* SMILES (pH7.3) InChI (pH7.3) InChI key (pH7.3) Formula (pH7.3) Charge (pH7.3) Mass (pH7.3) Exact Mass (neutral form) Exact m/z of [M.]+ Exact m/z of [M+H]+ Exact m/z of [M+K]+ Exact m/z of [M+Na]+ Exact m/z of [M+Li]+ Exact m/z of [M+NH4]+ Exact m/z of [M-H]- Exact m/z of [M+Cl]- Exact m/z of [M+OAc]- CHEBI LIPID MAPS HMDB PMID SLM:000000002 Class Ceramide (iso-d17:1(4E)) Cer(iso-d17:1(4E)) N-acyl-15-methylhexadecasphing-4-enine SLM:000399814 CC(C)CCCCCCCCC\C=C\[C@@H](O)[C@H](CO)NC([*])=O InChI=none C18H34NO3R 0 70846 14685263 | 21325339 | 9603947 | 21926990 SLM:000000003 Isomeric subspecies 15-methylhexadecasphing-4-enine SLM:000390097 CC(C)CCCCCCCCC\C=C\[C@@H](O)[C@@H]([NH3+])CO InChI=1S/C17H35NO2/c1-15(2)12-10-8-6-4-3-5-7-9-11-13-17(20)16(18)14-19/h11,13,15-17,19-20H,3-10,12,14,18H2,1-2H3/p+1/b13-11+/t16-,17+/m0/s1 InChIKey=LZKPPSAEINBHRP-KORIGIIASA-O C17H36NO2 1 286.473200 285.266779 285.266231 286.274056 324.229938 308.256000 292.282235 303.300605 284.259503 320.236181 344.280632 70771 19372430 Columns: - difficult to select column names, use index - the * means the field is a list - different field separators in list fields 0 Lipid ID 1 Level 2 Name 3 Abbreviation* 4 Synonyms* 5 Lipid class* 6 Parent 7 Components* 8 SMILES (pH7.3) 9 InChI (pH7.3) 10 InChI key (pH7.3) 11 Formula (pH7.3) 12 Charge (pH7.3) 13 Mass (pH7.3) 14 Exact Mass (neutral form) 15 Exact m/z of [M.]+ 16 Exact m/z of [M+H]+ 17 Exact m/z of [M+K]+ 18 Exact m/z of [M+Na]+ 19 Exact m/z of [M+Li]+ 20 Exact m/z of [M+NH4]+ 21 Exact m/z of [M-H]- 22 Exact m/z of [M+Cl]- 23 Exact m/z of [M+OAc]- 24 CHEBI 25 LIPID MAPS 26 HMDB 27 PMID """ lipids_file = instance.get_file('lipids.tsv.gz') # get header header = None with gzip.open(lipids_file, 'rt') as f: header = next(f) header = header.strip().split('\t') def safe_string(s): for char in [' ', '[', ']', '(', ')', '*', '/']: s = s.replace(char, '_') return s header_cypher_safe = [safe_string(s) for s in header] log.debug(header_cypher_safe) # iterate file with gzip.open(lipids_file, 'rt', errors="replace") as f: # skip header next(f) for l in f: flds = l.strip().split('\t') lipid_sid = flds[0] # (Lipid) node props = {'source': 'swisslipids'} props['sid'] = lipid_sid # add all properties, some are empty but contain whitespaces for i, fld in enumerate(flds): fld = fld.strip() if fld: props[header_cypher_safe[i]] = fld # # print( # dict(zip(header, flds)) # ) self.lipids.add_node(props) # (Lipid)-[FROM_LIPID_CLASS]-(Lipid) for lipid_class_sid in flds[5].strip().split('|'): # strip leading/trailing spaces, not always existing lipid_class_sid = lipid_class_sid.strip() self.lipid_fromclass_lipid.add_relationship( {'sid': lipid_sid}, {'sid': lipid_class_sid}, {'source': 'swisslipids'} ) # (Lipid)-[HAS_PARENT]-(Lipid) self.lipid_parent_lipid.add_relationship( {'sid': lipid_sid}, {'sid': flds[6].strip()}, {'source': 'swisslipids'} ) # (Lipid)-[COMPONENT]-(Lipid) ## e.g. SLM:000000510 (sn1) / SLM:000000418 (sn2) for lipid_component in flds[7].strip().split('/'): # get sid and type of lipid component, type does not always exist try: lipid_component_sid, lipid_component_type = lipid_component.strip().split(' ', 1) self.lipid_component_lipid.add_relationship( {'sid': lipid_sid}, {'sid': lipid_component_sid}, {'type': lipid_component_type} ) # some empty fields contain extra spaces except ValueError: pass # (Lipid)-[MAPS]-(Metabolite) try: chebi_id = flds[24].strip() if chebi_id: self.lipid_maps_metabolite.add_relationship( {'sid': lipid_sid}, {'sid': chebi_id}, {'source': 'swisslipids'} ) except IndexError: pass try: hmdb_id = flds[26].strip() if hmdb_id: self.lipid_maps_metabolite.add_relationship( {'sid': lipid_sid}, {'sid': hmdb_id}, {'source': 'swisslipids'} ) except IndexError: pass def get_lipid_to_protein(self, instance): """ File: lipids2uniprot.tsv.gz Columns: - difficult to select column names, use index 0 metabolite id 1 UniprotKB IDs 2 level 3 metabolite name 4 abbreviations 5 synonyms 6 lipid class 7 components 8 PMIDs 9 SMILES (pH7.3) 10 InChI (pH7.3) 11 InChI key (pH7.3) 12 Formula (pH7.3) 13 Mass (pH7.3) 14 Charge (pH7.3) 15 Exact Mass (neutral form) 16 Exact m/z of [M.]+ 17 Exact m/z of [M+H]+ 18 Exact m/z of [M+K]+Exact m/z of [M+Na]+ 19 Exact m/z of [M+Li]+ 20 Exact m/z of [M+NH4]+ 21 Exact m/z of [M-H]- 22 Exact m/z of [M+Cl]- 23 Exact m/z of [M+OAc]- 24 ChEBI 25 LipidMaps 26 HMDB 27 Mapping level :param instance: The datasource instance. """ lipids_2_protein_file = instance.get_file('lipids2uniprot.tsv.gz') # iterate file with gzip.open(lipids_2_protein_file, 'rt', errors="replace") as f: next(f) for l in f: flds = l.strip().split('\t') swisslipids_id = flds[0].strip() mapping_level = flds[27].strip() # collect UniProt IDs from uniprot fields, contains a '|' separated list # G5EC84 | O18037 | P91079 | Q09517 | Q10916 | Q20735 | Q21054 | Q23498 | Q9U3D4 # note: not always formatted with space: ' | ' uniprot_id_string = flds[1] uniprot_ids = set() for u in uniprot_id_string.split('|'): u = u.strip() if u: uniprot_ids.add(u) for up in uniprot_ids: self.lipid_associates_protein.add_relationship( {'sid': swisslipids_id}, {'sid': up}, {'source': 'swisslipids', 'level': mapping_level} )