def check(self): files = [ f for f in os.listdir(MARKERS_FOLDER) if isfile(join(MARKERS_FOLDER, f)) ] for marker_file in files: if not is_whitelist_file(marker_file): dendrogram_path = join(DENDROGRAMS_FOLDER, get_taxonomy_file_name(marker_file)) print("DEND:" + dendrogram_path) if os.path.exists(dendrogram_path): headers, marker_records = read_csv_to_dict(join( MARKERS_FOLDER, marker_file), delimiter="\t") if str(dendrogram_path).endswith(".json"): dend = dend_json_2_nodes_n_edges(dendrogram_path) else: dend = nomenclature_2_nodes_n_edges(dendrogram_path) dend_dict = index_dendrogram(dend) self.check_cluster_name(dend_dict, marker_records, marker_file) else: message = "Could not find taxonomy file '{}' for marker '{}'." \ .format(dendrogram_path, marker_file) self.reports.append(message)
def test_get_synonym_pairs(self): tree = dend_json_2_nodes_n_edges(PATH_DENDROGRAM_JSON) nodes = tree['nodes'] node3_pairs = get_synonym_pairs(nodes[3]).split(PAIR_SEPARATOR) self.assertEqual("CS202002013_120", nodes[3]["cell_set_accession"]) self.assertEqual(5, len(node3_pairs)) self.assertTrue("cell_set_preferred_alias:''" in node3_pairs) self.assertTrue("original_label:n4" in node3_pairs) self.assertTrue("cell_set_label:RNAseq 001-091" in node3_pairs) self.assertTrue("cell_set_aligned_alias:''" in node3_pairs) self.assertTrue("cell_set_additional_aliases:''" in node3_pairs) node20_pairs = get_synonym_pairs(nodes[20]).split(PAIR_SEPARATOR) self.assertEqual("CS202002013_6", nodes[20]["cell_set_accession"]) self.assertEqual(5, len(node20_pairs)) self.assertTrue( "cell_set_preferred_alias:Lamp5 Pdlim5_2" in node20_pairs) self.assertTrue("original_label:Lamp5 Pdlim5_2" in node20_pairs) self.assertTrue("cell_set_label:RNAseq 006" in node20_pairs) self.assertTrue("cell_set_aligned_alias:''" in node20_pairs) self.assertTrue("cell_set_additional_aliases:''" in node20_pairs) node50_pairs = get_synonym_pairs(nodes[50]).split(PAIR_SEPARATOR) self.assertEqual("CS202002013_146", nodes[50]["cell_set_accession"]) self.assertEqual(5, len(node50_pairs)) self.assertTrue("cell_set_preferred_alias:''" in node50_pairs) self.assertTrue("original_label:n30" in node50_pairs) self.assertTrue("cell_set_label:RNAseq 022-025" in node50_pairs) self.assertTrue("cell_set_aligned_alias:''" in node50_pairs) self.assertTrue("cell_set_additional_aliases:''" in node50_pairs)
def generate_cross_species_template(taxonomy_file_path, output_filepath): taxon = extract_taxonomy_name_from_path(taxonomy_file_path) taxonomy_config = read_taxonomy_config(taxon) if taxonomy_config: if str(taxonomy_file_path).endswith(".json"): dend = dend_json_2_nodes_n_edges(taxonomy_file_path) else: dend = nomenclature_2_nodes_n_edges(taxonomy_file_path) dend_tree = generate_dendrogram_tree(dend) subtrees = get_subtrees(dend_tree, taxonomy_config) cross_species_template = [] headers, cs_by_preferred_alias = read_csv_to_dict( CROSS_SPECIES_PATH, id_column_name="cell_set_preferred_alias", id_to_lower=True) headers, cs_by_aligned_alias = read_csv_to_dict( CROSS_SPECIES_PATH, id_column_name="cell_set_aligned_alias", id_to_lower=True) for o in dend['nodes']: if o['cell_set_accession'] in set.union( *subtrees) and (o['cell_set_preferred_alias'] or o['cell_set_additional_aliases']): cross_species_classes = set() if o["cell_set_aligned_alias"] and str( o["cell_set_aligned_alias"]).lower( ) in cs_by_aligned_alias: cross_species_classes.add(PCL_BASE + get_class_id( cs_by_aligned_alias[str(o["cell_set_aligned_alias"]). lower()]["cell_set_accession"])) if "cell_set_additional_aliases" in o and o[ "cell_set_additional_aliases"]: additional_aliases = str( o["cell_set_additional_aliases"]).lower().split( EXPRESSION_SEPARATOR) for additional_alias in additional_aliases: if additional_alias in cs_by_preferred_alias: cross_species_classes.add(PCL_BASE + get_class_id( cs_by_preferred_alias[additional_alias] ["cell_set_accession"])) if len(cross_species_classes): d = dict() d['defined_class'] = PCL_BASE + get_class_id( o['cell_set_accession']) d['cross_species_classes'] = EXPRESSION_SEPARATOR.join( cross_species_classes) cross_species_template.append(d) class_robot_template = pd.DataFrame.from_records( cross_species_template) class_robot_template.to_csv(output_filepath, sep="\t", index=False)
def generate_marker_gene_set_template(taxonomy_file_path, output_filepath): taxon = extract_taxonomy_name_from_path(taxonomy_file_path) taxonomy_config = read_taxonomy_config(taxon) if taxonomy_config: if str(taxonomy_file_path).endswith(".json"): dend = dend_json_2_nodes_n_edges(taxonomy_file_path) else: dend = nomenclature_2_nodes_n_edges(taxonomy_file_path) dend_tree = generate_dendrogram_tree(dend) subtrees = get_subtrees(dend_tree, taxonomy_config) if "Reference_gene_list" in taxonomy_config: gene_db_path = ENSEMBLE_PATH.format( str(taxonomy_config["Reference_gene_list"][0]).strip().lower()) gene_names = read_gene_data(gene_db_path) minimal_markers = read_markers( MARKER_PATH.format(taxon.replace("CCN", "").replace("CS", "")), gene_names) else: minimal_markers = {} class_seed = [ 'defined_class', 'Marker_set_of', 'Minimal_markers', 'Brain_region_abbv', 'Species_abbv', 'Brain_region', 'Parent' ] class_template = [] for o in dend['nodes']: if o['cell_set_accession'] in set.union( *subtrees) and (o['cell_set_preferred_alias'] or o['cell_set_additional_aliases']): if o['cell_set_accession'] in minimal_markers: d = dict() d['defined_class'] = PCL_BASE + get_marker_gene_set_id( o['cell_set_accession']) d['Marker_set_of'] = o['cell_set_preferred_alias'] d['Minimal_markers'] = minimal_markers[ o['cell_set_accession']] if 'Brain_region_abbv' in taxonomy_config: d['Brain_region_abbv'] = taxonomy_config[ 'Brain_region_abbv'][0] if 'Species_abbv' in taxonomy_config: d['Species_abbv'] = taxonomy_config['Species_abbv'][0] d['Brain_region'] = taxonomy_config['Brain_region'][0] d['Parent'] = "SO:0001260" # sequence collection for k in class_seed: if not (k in d.keys()): d[k] = '' class_template.append(d) class_robot_template = pd.DataFrame.from_records(class_template) class_robot_template.to_csv(output_filepath, sep="\t", index=False)
def main(): log.info("Dendrogram validation started.") files = [f for f in os.listdir(DENDROGRAM_FOLDER) if isfile(join(DENDROGRAM_FOLDER, f))] is_valid = True for file in files: filename, file_extension = os.path.splitext(file) if file_extension == ".json": dend = dend_json_2_nodes_n_edges(join(DENDROGRAM_FOLDER, file)) is_valid &= PrefAliasUniquenessChecker().check(filename, dend) if not is_valid: raise ValidationError("Dendrogram validation failed and issues logged.")
def generate_curated_class_template(dend_json_path, output_filepath): dend = dend_json_2_nodes_n_edges(dend_json_path) robot_class_curation_seed = { 'ID': 'ID', 'Label': 'LABEL', 'PrefLabel': 'A skos:prefLabel', 'Synonyms': 'A oboInOwl:has_exact_synonym', 'Exemplar': "EC CL:0000003 and 'has_exemplar' value %", 'Classification': 'SC %', 'Comment': 'A rdfs:comment', 'part of': "SC 'part of' some %", 'part evidence comment': '>A rdfs:comment', 'part evidence pub': ">A dc:reference SPLIT='|'", # Bundle this with dbxrefs? 'part evidence dbxref': ">A oio:hasDbXref SPLIT='|'", 'location': "SC 'located in' some %", 'location evidence comment': '>A rdfs:comment', 'location evidence pub': ">A dc:reference SPLIT='|'", # Bundle this with dbxrefs? 'location evidence dbxref': ">A oio:hasDbXref SPLIT='|'", 'has soma location': "SC 'has soma location' some %", 'soma location evidence comment': '>A rdfs:comment', 'soma location evidence pub': ">A dc:reference SPLIT='|'", # Bundle this with dbxrefs? 'soma location evidence dbxref': ">A oio:dbxref SPLIT='|'", 'cell morphology phenotype': "SC 'bearer of' some %", 'cell morphology evidence comment': '>A rdfs:comment', 'cell morphology evidence pub': ">A dc:reference SPLIT='|'", # Bundle this with dbxrefs? 'cell morphology evidence dbxref': ">A oio:hasDbXref SPLIT='|'" } class_template = [robot_class_curation_seed] # not bothering with stable IDs for now: for o in dend['nodes']: ID = 'http://brain_data_standards/scratch_' + str(uuid.uuid1()) d = dict() d['ID'] = ID d['Class_type'] = 'subclass' d['Exemplar'] = 'AllenDend:' + o['cell_set_accession'] if o['cell_set_label']: d['Label'] = o['cell_set_label'] if o['cell_set_preferred_alias']: d['PrefLabel'] = o['cell_set_preferred_alias'] for k in robot_class_curation_seed.keys(): if not (k in d.keys()): d[k] = '' class_template.append(d) class_robot_template = pd.DataFrame.from_records(class_template) class_robot_template.to_csv(output_filepath, sep="\t", index=False)
def generate_homologous_to_template(taxonomy_file_path, all_base_files, output_filepath): """ Homologous_to relations require a separate template. If this operation is driven by the nomenclature tables, some dangling classes may be generated due to root classes that don't have a class and should not be aligned. So, instead of nomenclature tables, base files are used for populating homologous to relations. This ensures all alignments has a corresponding class. Args: taxonomy_file_path: path of the taxonomy file all_base_files: paths of the all class template base files output_filepath: template output file path """ taxon = extract_taxonomy_name_from_path(taxonomy_file_path) taxonomy_config = read_taxonomy_config(taxon) other_taxonomy_aliases = index_base_files( [t for t in all_base_files if taxon not in t]) if taxonomy_config: if str(taxonomy_file_path).endswith(".json"): dend = dend_json_2_nodes_n_edges(taxonomy_file_path) else: dend = nomenclature_2_nodes_n_edges(taxonomy_file_path) dend_tree = generate_dendrogram_tree(dend) subtrees = get_subtrees(dend_tree, taxonomy_config) data_template = [] for o in dend['nodes']: if o['cell_set_accession'] in set.union( *subtrees) and (o['cell_set_preferred_alias'] or o['cell_set_additional_aliases']): d = dict() d['defined_class'] = PCL_BASE + get_class_id( o['cell_set_accession']) homologous_to = list() for other_aliases in other_taxonomy_aliases: if "cell_set_aligned_alias" in o and o["cell_set_aligned_alias"] \ and str(o["cell_set_aligned_alias"]).lower() in other_aliases: homologous_to.append(other_aliases[str( o["cell_set_aligned_alias"]).lower()] ["defined_class"]) d['homologous_to'] = "|".join(homologous_to) data_template.append(d) robot_template = pd.DataFrame.from_records(data_template) robot_template.to_csv(output_filepath, sep="\t", index=False)
def generate_marker_template(dend_json_path, output_filepath): dend = dend_json_2_nodes_n_edges(dend_json_path) robot_marker_template_seed = { 'ID': 'ID', 'Expresses': "TI 'expresses' % SPLIT = '|' ", 'Evidence': "^A rdfs:comment" } template = [robot_marker_template_seed] for o in dend['nodes']: d = dict() d['ID'] = 'AllenDend:' + o['cell_set_accession'] for k in robot_marker_template_seed.keys(): if not (k in d.keys()): d[k] = '' template.append(d) class_robot_template = pd.DataFrame.from_records(template) class_robot_template.to_csv(output_filepath, sep="\t", index=False)
def check(self): files = [ f for f in os.listdir(MARKERS_FOLDER) if isfile(join(MARKERS_FOLDER, f)) ] for marker_file in files: if not is_whitelist_file(marker_file): dendrogram_path = join(DENDROGRAMS_FOLDER, get_taxonomy_file_name(marker_file)) if os.path.exists(dendrogram_path): marker_records = read_tsv(join(MARKERS_FOLDER, marker_file)) if str(dendrogram_path).endswith(".json"): dend = dend_json_2_nodes_n_edges(dendrogram_path) else: dend = nomenclature_2_nodes_n_edges(dendrogram_path) self.check_all_nodes_exist(dend, marker_records, marker_file)
def generate_curated_class_template(taxonomy_file_path, output_filepath): taxon = extract_taxonomy_name_from_path(taxonomy_file_path) taxonomy_config = read_taxonomy_config(taxon) if taxonomy_config: if str(taxonomy_file_path).endswith(".json"): dend = dend_json_2_nodes_n_edges(taxonomy_file_path) else: dend = nomenclature_2_nodes_n_edges(taxonomy_file_path) dend_tree = generate_dendrogram_tree(dend) subtrees = get_subtrees(dend_tree, taxonomy_config) class_curation_seed = [ 'defined_class', 'Curated_synonyms', 'Classification', 'Classification_comment', 'Classification_pub', 'Expresses', 'Expresses_comment', 'Expresses_pub', 'Projection_type', 'Layers', 'Cross_species_text', 'Comment' ] class_template = [] for o in dend['nodes']: if o['cell_set_accession'] in set.union( *subtrees) and (o['cell_set_preferred_alias'] or o['cell_set_additional_aliases']): d = dict() d['defined_class'] = PCL_BASE + get_class_id( o['cell_set_accession']) if o['cell_set_preferred_alias']: d['prefLabel'] = o['cell_set_preferred_alias'] elif o['cell_set_additional_aliases']: d['prefLabel'] = str( o['cell_set_additional_aliases']).split( EXPRESSION_SEPARATOR)[0] for k in class_curation_seed: if not (k in d.keys()): d[k] = '' class_template.append(d) class_robot_template = pd.DataFrame.from_records(class_template) class_robot_template.to_csv(output_filepath, sep="\t", index=False)
def check(self): files = [ f for f in os.listdir(MARKERS_FOLDER) if isfile(join(MARKERS_FOLDER, f)) ] for marker_file in files: if not is_whitelist_file(marker_file): dendrogram_path = join(DENDROGRAMS_FOLDER, get_taxonomy_file_name(marker_file)) if os.path.exists(dendrogram_path): headers, marker_records = read_csv_to_dict(join( MARKERS_FOLDER, marker_file), delimiter="\t") if str(dendrogram_path).endswith(".json"): dend = dend_json_2_nodes_n_edges(dendrogram_path) else: dend = nomenclature_2_nodes_n_edges(dendrogram_path) dend_dict = index_dendrogram(dend) self.check_all_node_ids_valid(dend_dict, marker_records, marker_file) self.check_all_node_ids_unique(marker_file)
def generate_ind_template(dend_json_path, output_filepath): dend = dend_json_2_nodes_n_edges(dend_json_path) robot_template_seed = { 'ID': 'ID', 'Label': 'LABEL', 'PrefLabel': 'A skos:prefLabel', 'Entity Type': 'TI %', 'TYPE': 'TYPE', 'Property Assertions': "I BDSHELP:subcluster_of SPLIT='|'", 'Synonyms': 'A oboInOwl:has_exact_synonym', 'Function': 'TI capable_of some %' } dl = [robot_template_seed] synonym_properties = [ 'original_label', 'cell_set_aligned_alias', 'cell_set_additional_aliases' ] for o in dend['nodes']: d = dict() d['ID'] = 'AllenDend:' + o['cell_set_accession'] d['TYPE'] = 'owl:NamedIndividual' d['Label'] = o['cell_set_label'] + ' - ' + o['cell_set_accession'] d['PrefLabel'] = o['cell_set_preferred_alias'] + ' - ' + o[ 'cell_set_accession'] d['Entity Type'] = 'BDSHELP:Cluster' d['Synonyms'] = '|'.join([ o[prop] for prop in synonym_properties if prop in o.keys() and o[prop] ]) d['Property Assertions'] = '|'.join([ 'AllenDend:' + e[1] for e in dend['edges'] if e[0] == o['cell_set_accession'] ]) # There should only be one! dl.append(d) robot_template = pd.DataFrame.from_records(dl) robot_template.to_csv(output_filepath, sep="\t", index=False)
def generate_pattern_table_denormalised_markers(dend_json_path, output_filepath): path_parts = dend_json_path.split(os.path.sep) taxon = path_parts[len(path_parts) - 1].split(".")[0] taxonomy_config = read_taxonomy_config(taxon) dend = dend_json_2_nodes_n_edges(dend_json_path) dend_tree = read_dendrogram_tree(dend_json_path) marker_path = MARKER_PATH.format(str(taxon).replace("CCN", "")) gene_db_path = ENSEMBLE_PATH.format(str(taxonomy_config["Reference_gene_list"][0]).strip().lower()) if taxonomy_config: subtrees = get_subtrees(dend_tree, taxonomy_config) gene_names = read_gene_data(gene_db_path) denorm_markers = get_denorm_markers(taxon, gene_names) minimal_markers = read_markers(marker_path, gene_names) dl = [] for o in dend['nodes']: if o['cell_set_accession'] in set.union(*subtrees): d = dict() d['defined_class'] = ALLEN_DEND_ + o['cell_set_accession'] d['gross_cell_type'] = get_gross_cell_type(o['cell_set_accession'], subtrees, taxonomy_config) d['taxon'] = taxonomy_config['Species'][0] d['brain_region'] = taxonomy_config['Brain_region'][0] if o['cell_set_accession'] in denorm_markers: d['denorm_marker_list'] = denorm_markers[o['cell_set_accession']] d['minimal_marker_list'] = minimal_markers[o['cell_set_accession']] else: d['denorm_marker_list'] = '' d['minimal_marker_list'] = '' dl.append(d) robot_template = pd.DataFrame.from_records(dl) robot_template.to_csv(output_filepath, sep="\t", index=False)
def test_get_synonyms_from_taxonomy(self): tree = dend_json_2_nodes_n_edges(PATH_DENDROGRAM_JSON) nodes = tree['nodes'] node8_synonyms = get_synonyms_from_taxonomy( nodes[8]).split(OR_SEPARATOR) node8_synonyms = list(filter(None, node8_synonyms)) self.assertEqual("CS202002013_125", nodes[8]["cell_set_accession"]) self.assertEqual(1, len(node8_synonyms)) self.assertTrue("Lamp5" in node8_synonyms) node20_synonyms = get_synonyms_from_taxonomy( nodes[20]).split(OR_SEPARATOR) node20_synonyms = list(filter(None, node20_synonyms)) self.assertEqual("CS202002013_6", nodes[20]["cell_set_accession"]) self.assertEqual(1, len(node20_synonyms)) self.assertTrue("Lamp5 Pdlim5_2" in node20_synonyms) node50_synonyms = get_synonyms_from_taxonomy( nodes[50]).split(OR_SEPARATOR) node50_synonyms = list(filter(None, node50_synonyms)) self.assertEqual("CS202002013_146", nodes[50]["cell_set_accession"]) self.assertEqual(0, len(node50_synonyms))
def generate_app_specific_template(taxonomy_file_path, output_filepath): if str(taxonomy_file_path).endswith(".json"): dend = dend_json_2_nodes_n_edges(taxonomy_file_path) else: dend = nomenclature_2_nodes_n_edges(taxonomy_file_path) robot_template_seed = { 'ID': 'ID', 'TYPE': 'TYPE', 'cell_set_color': "A ALLENHELP:cell_set_color" } dl = [robot_template_seed] for o in dend['nodes']: if "cell_set_color" in o and o["cell_set_color"]: d = dict() d['ID'] = 'PCL:' + get_individual_id(o['cell_set_accession']) d['TYPE'] = 'owl:NamedIndividual' d['cell_set_color'] = str(o["cell_set_color"]).strip() dl.append(d) robot_template = pd.DataFrame.from_records(dl) robot_template.to_csv(output_filepath, sep="\t", index=False)
def generate_denormalised_marker_template(taxonomy_file_path, output_marker_path): """ Enriches existing marker file based on inheritance relations extracted from dendrogram file. New maker table, following the same format as the input marker table, with each node associated with a non-redundant list of all markers associated with the term in the input + all markers associated with parent terms. Args: dend_json_path: Path of the dendrogram file output_marker_path: Path of the new marker file """ path_parts = taxonomy_file_path.split(os.path.sep) taxon = path_parts[len(path_parts) - 1].split(".")[0] if str(taxonomy_file_path).endswith(".json"): dend = dend_json_2_nodes_n_edges(taxonomy_file_path) else: dend = nomenclature_2_nodes_n_edges(taxonomy_file_path) taxon = path_parts[len(path_parts) - 1].split(".")[0].replace("nomenclature_table_", "") taxonomy_config = read_taxonomy_config(taxon) root_nodes = get_root_nodes(taxonomy_config) marker_path = MARKER_PATH.format(str(taxon).replace("CCN", "").replace("CS", "")) generate_denormalised_marker(dend, marker_path, output_marker_path, root_nodes)
def generate_base_class_template(taxonomy_file_path, output_filepath): taxon = extract_taxonomy_name_from_path(taxonomy_file_path) taxonomy_config = read_taxonomy_config(taxon) if taxonomy_config: if str(taxonomy_file_path).endswith(".json"): dend = dend_json_2_nodes_n_edges(taxonomy_file_path) else: dend = nomenclature_2_nodes_n_edges(taxonomy_file_path) dend_tree = generate_dendrogram_tree(dend) subtrees = get_subtrees(dend_tree, taxonomy_config) if "Reference_gene_list" in taxonomy_config: gene_db_path = ENSEMBLE_PATH.format( str(taxonomy_config["Reference_gene_list"][0]).strip().lower()) gene_names = read_gene_data(gene_db_path) minimal_markers = read_markers( MARKER_PATH.format(taxon.replace("CCN", "").replace("CS", "")), gene_names) allen_markers = read_markers( ALLEN_MARKER_PATH.format( taxon.replace("CCN", "").replace("CS", "")), gene_names) else: minimal_markers = {} allen_markers = {} class_seed = [ 'defined_class', 'prefLabel', 'Alias_citations', 'Synonyms_from_taxonomy', 'Gross_cell_type', 'Taxon', 'Brain_region', 'Minimal_markers', 'Allen_markers', 'Individual', 'Brain_region_abbv', 'Species_abbv', 'Cluster_ID', 'part_of', 'has_soma_location', 'aligned_alias', 'marker_gene_set' ] class_template = [] for o in dend['nodes']: if o['cell_set_accession'] in set.union( *subtrees) and (o['cell_set_preferred_alias'] or o['cell_set_additional_aliases']): d = dict() d['defined_class'] = PCL_BASE + get_class_id( o['cell_set_accession']) if o['cell_set_preferred_alias']: d['prefLabel'] = o['cell_set_preferred_alias'] elif o['cell_set_additional_aliases']: d['prefLabel'] = str( o['cell_set_additional_aliases']).split( EXPRESSION_SEPARATOR)[0] d['Synonyms_from_taxonomy'] = get_synonyms_from_taxonomy(o) d['Gross_cell_type'] = get_gross_cell_type( o['cell_set_accession'], subtrees, taxonomy_config) d['Taxon'] = taxonomy_config['Species'][0] d['Brain_region'] = taxonomy_config['Brain_region'][0] d['Cluster_ID'] = o['cell_set_accession'] if 'cell_set_alias_citation' in o and o[ 'cell_set_alias_citation']: alias_citations = [ citation.strip() for citation in str( o["cell_set_alias_citation"]).split("|") if citation and citation.strip() ] d["Alias_citations"] = "|".join(alias_citations) if o['cell_set_accession'] in minimal_markers: d['Minimal_markers'] = minimal_markers[ o['cell_set_accession']] if o['cell_set_accession'] in allen_markers: d['Allen_markers'] = allen_markers[o['cell_set_accession']] else: d['Allen_markers'] = '' if 'Brain_region_abbv' in taxonomy_config: d['Brain_region_abbv'] = taxonomy_config[ 'Brain_region_abbv'][0] if 'Species_abbv' in taxonomy_config: d['Species_abbv'] = taxonomy_config['Species_abbv'][0] d['Individual'] = PCL_BASE + get_individual_id( o['cell_set_accession']) for index, subtree in enumerate(subtrees): if o['cell_set_accession'] in subtree: location_rel = taxonomy_config['Root_nodes'][index][ 'Location_relation'] if location_rel == "part_of": d['part_of'] = taxonomy_config['Brain_region'][0] d['has_soma_location'] = '' elif location_rel == "has_soma_location": d['part_of'] = '' d['has_soma_location'] = taxonomy_config[ 'Brain_region'][0] if "cell_set_aligned_alias" in o and o[ "cell_set_aligned_alias"]: d['aligned_alias'] = o["cell_set_aligned_alias"] if o['cell_set_accession'] in minimal_markers: d['marker_gene_set'] = PCL_PREFIX + get_marker_gene_set_id( o['cell_set_accession']) for k in class_seed: if not (k in d.keys()): d[k] = '' class_template.append(d) class_robot_template = pd.DataFrame.from_records(class_template) class_robot_template.to_csv(output_filepath, sep="\t", index=False)
def normalize_raw_markers(raw_marker): """ Raw marker files has different structure than the expected. Needs these modifications: - Extract Taxonomy_node_ID: clusterName matches cell_set_aligned_alias of the dendrogram. - Resolve markers: convert marker names to ensemble IDs from local DBs Args: raw_marker: """ taxonomy_config = get_taxonomy_config(raw_marker) taxonomy_id = taxonomy_config["Taxonomy_id"] print("Taxonomy ID: " + taxonomy_id) if taxonomy_id == "CS1908210": print("Read dendrogram: " + taxonomy_id) dend = dend_json_2_nodes_n_edges(DENDROGRAM.format(taxonomy_id)) nomenclature_indexes = [ index_dendrogram(dend, id_field_name="cell_set_preferred_alias", id_to_lower=True), # index_dendrogram(dend, id_field_name="cell_set_aligned_alias", id_to_lower=True), index_dendrogram(dend, id_field_name="cell_set_accession", id_to_lower=True), index_dendrogram(dend, id_field_name="original_label", id_to_lower=True), index_dendrogram(dend, id_field_name="cell_set_additional_aliases", id_to_lower=True) ] else: print("Read nomenclature table: " + taxonomy_id) nomenclature_indexes = [read_csv_to_dict(NOMENCLATURE.format(taxonomy_id), id_column_name="cell_set_preferred_alias", id_to_lower=True)[1], read_csv_to_dict(NOMENCLATURE.format(taxonomy_id), id_column_name="cell_set_aligned_alias", id_to_lower=True)[1], read_csv_to_dict(NOMENCLATURE.format(taxonomy_id), id_column_name="cell_set_accession", id_to_lower=True)[1], read_csv_to_dict(NOMENCLATURE.format(taxonomy_id), id_column_name="original_label", id_to_lower=True)[1], read_csv_to_dict(NOMENCLATURE.format(taxonomy_id), id_column_name="cell_set_additional_aliases", id_to_lower=True)[1], ] gene_db_path = GENE_DB_PATH.format(str(taxonomy_config["Reference_gene_list"][0]).strip().lower()) headers, genes_by_name = read_csv_to_dict(gene_db_path, id_column=2, delimiter="\t", id_to_lower=True) species_abbv = get_species_for_gene_db(gene_db_path).lower() unmatched_markers = set() normalized_markers = [] if raw_marker.endswith(".csv"): headers, raw_marker_data = read_csv_to_dict(raw_marker, id_column_name="clusterName") else: headers, raw_marker_data = read_csv_to_dict(raw_marker, id_column_name="clusterName", delimiter="\t") for cluster_name in raw_marker_data: normalized_data = {} row = raw_marker_data[cluster_name] cluster_name_variants = [cluster_name.lower(), cluster_name.lower().replace("-", "/"), cluster_name.replace("Micro", "Microglia").lower()] nomenclature_node = search_terms_in_index(cluster_name_variants, nomenclature_indexes) if nomenclature_node: node_id = nomenclature_node["cell_set_accession"] marker_names = get_marker_names(row) marker_ids = [] for name in marker_names: if name: if species_abbv + " " + name.lower() in genes_by_name: marker_ids.append(str(genes_by_name[species_abbv + " " + name.lower()]["ID"])) elif species_abbv + " " + name.lower().replace("_", "-") in genes_by_name: marker_ids.append(str(genes_by_name[species_abbv + " " + name.lower().replace("_", "-")]["ID"])) else: unmatched_markers.add(name) normalized_data["Taxonomy_node_ID"] = node_id normalized_data["clusterName"] = nomenclature_node["cell_set_preferred_alias"] normalized_data["Markers"] = "|".join(marker_ids) normalized_markers.append(normalized_data) else: log.error("Node with cluster name '{}' couldn't be found in the nomenclature.".format(cluster_name)) # raise Exception("Node with cluster name {} couldn't be found in the nomenclature.".format(cluster_name)) class_robot_template = pd.DataFrame.from_records(normalized_markers) class_robot_template.to_csv(OUTPUT_MARKER.format(taxonomy_id.replace("CCN", "").replace("CS", "")), sep="\t", index=False) log.error("Following markers could not be found in the db ({}): {}".format(len(unmatched_markers), str(unmatched_markers)))
def generate_ind_template(taxonomy_file_path, output_filepath): path_parts = taxonomy_file_path.split(os.path.sep) taxon = path_parts[len(path_parts) - 1].split(".")[0] if str(taxonomy_file_path).endswith(".json"): dend = dend_json_2_nodes_n_edges(taxonomy_file_path) else: dend = nomenclature_2_nodes_n_edges(taxonomy_file_path) taxon = path_parts[len(path_parts) - 1].split(".")[0].replace( "nomenclature_table_", "") dend_tree = generate_dendrogram_tree(dend) taxonomy_config = read_taxonomy_config(taxon) allen_descriptions = read_allen_descriptions( ALLEN_DESCRIPTIONS_PATH, taxonomy_config['Species_abbv'][0]) subtrees = get_subtrees(dend_tree, taxonomy_config) robot_template_seed = { 'ID': 'ID', 'Label': 'LABEL', 'PrefLabel': 'A skos:prefLabel', 'Entity Type': 'TI %', 'TYPE': 'TYPE', 'Property Assertions': "I 'subcluster of' SPLIT=|", 'Synonyms': 'A oboInOwl:hasExactSynonym SPLIT=|', 'Cluster_ID': "A 'cluster id'", 'Function': 'TI capable_of some %', 'cell_set_preferred_alias': "A n2o:cell_set_preferred_alias", 'original_label': "A n2o:original_label", 'cell_set_label': "A n2o:cell_set_label", 'cell_set_aligned_alias': "A n2o:cell_set_aligned_alias", 'cell_set_additional_aliases': "A n2o:cell_set_additional_aliases SPLIT=|", 'cell_set_alias_assignee': "A n2o:cell_set_alias_assignee SPLIT=|", 'cell_set_alias_citation': "A n2o:cell_set_alias_citation SPLIT=|", 'Metadata': "A n2o:node_metadata", 'Exemplar_of': "TI 'exemplar data of' some %", 'Comment': "A rdfs:comment", 'Aliases': "A oboInOwl:hasRelatedSynonym SPLIT=|", 'Rank': "A 'cell_type_rank' SPLIT=|" } dl = [robot_template_seed] synonym_properties = [ 'cell_set_aligned_alias', 'cell_set_additional_aliases' ] for o in dend['nodes']: d = dict() d['ID'] = 'PCL:' + get_individual_id(o['cell_set_accession']) d['TYPE'] = 'owl:NamedIndividual' d['Label'] = o['cell_set_label'] + ' - ' + o['cell_set_accession'] if 'cell_set_preferred_alias' in o and o['cell_set_preferred_alias']: d['PrefLabel'] = o['cell_set_preferred_alias'] else: d['PrefLabel'] = o['cell_set_accession'] d['Entity Type'] = 'PCL:0010001' # Cluster d['Metadata'] = json.dumps(o) d['Synonyms'] = '|'.join([ o[prop] for prop in synonym_properties if prop in o.keys() and o[prop] ]) d['Property Assertions'] = '|'.join( sorted([ 'PCL:' + get_individual_id(e[1]) for e in dend['edges'] if e[0] == o['cell_set_accession'] ])) meta_properties = [ 'cell_set_preferred_alias', 'original_label', 'cell_set_label', 'cell_set_aligned_alias', 'cell_set_additional_aliases', 'cell_set_alias_assignee', 'cell_set_alias_citation' ] for prop in meta_properties: if prop in o.keys(): d[prop] = '|'.join([ prop_val.strip() for prop_val in str(o[prop]).split("|") if prop_val ]) else: d[prop] = '' d['Cluster_ID'] = o['cell_set_accession'] if o['cell_set_accession'] in set().union( *subtrees) and o['cell_set_preferred_alias']: d['Exemplar_of'] = PCL_BASE + get_class_id(o['cell_set_accession']) if "cell_type_card" in o: d['Rank'] = '|'.join([ cell_type.strip().replace("No", "None") for cell_type in str(o["cell_type_card"]).split(",") ]) if o['cell_set_accession'] in allen_descriptions: allen_data = allen_descriptions[o['cell_set_accession']] d['Comment'] = allen_data["summary"][0] if allen_data["aliases"][0]: d['Aliases'] = '|'.join([ alias.strip() for alias in str(allen_data["aliases"][0]).split("|") ]) # There should only be one! dl.append(d) robot_template = pd.DataFrame.from_records(dl) robot_template.to_csv(output_filepath, sep="\t", index=False)