def generate_cross_species_template(taxonomy_file_path, output_filepath): taxon = extract_taxonomy_name_from_path(taxonomy_file_path) taxonomy_config = read_taxonomy_config(taxon) if taxonomy_config: if str(taxonomy_file_path).endswith(".json"): dend = dend_json_2_nodes_n_edges(taxonomy_file_path) else: dend = nomenclature_2_nodes_n_edges(taxonomy_file_path) dend_tree = generate_dendrogram_tree(dend) subtrees = get_subtrees(dend_tree, taxonomy_config) cross_species_template = [] headers, cs_by_preferred_alias = read_csv_to_dict( CROSS_SPECIES_PATH, id_column_name="cell_set_preferred_alias", id_to_lower=True) headers, cs_by_aligned_alias = read_csv_to_dict( CROSS_SPECIES_PATH, id_column_name="cell_set_aligned_alias", id_to_lower=True) for o in dend['nodes']: if o['cell_set_accession'] in set.union( *subtrees) and (o['cell_set_preferred_alias'] or o['cell_set_additional_aliases']): cross_species_classes = set() if o["cell_set_aligned_alias"] and str( o["cell_set_aligned_alias"]).lower( ) in cs_by_aligned_alias: cross_species_classes.add(PCL_BASE + get_class_id( cs_by_aligned_alias[str(o["cell_set_aligned_alias"]). lower()]["cell_set_accession"])) if "cell_set_additional_aliases" in o and o[ "cell_set_additional_aliases"]: additional_aliases = str( o["cell_set_additional_aliases"]).lower().split( EXPRESSION_SEPARATOR) for additional_alias in additional_aliases: if additional_alias in cs_by_preferred_alias: cross_species_classes.add(PCL_BASE + get_class_id( cs_by_preferred_alias[additional_alias] ["cell_set_accession"])) if len(cross_species_classes): d = dict() d['defined_class'] = PCL_BASE + get_class_id( o['cell_set_accession']) d['cross_species_classes'] = EXPRESSION_SEPARATOR.join( cross_species_classes) cross_species_template.append(d) class_robot_template = pd.DataFrame.from_records( cross_species_template) class_robot_template.to_csv(output_filepath, sep="\t", index=False)
def nomenclature_2_nodes_n_edges(taxonomy_file_path): out = dict() out['nodes'] = [] out['edges'] = set() child_cell_sets = list() headers, nomenclature_records = read_csv_to_dict( taxonomy_file_path, id_column_name='cell_set_accession') for node_cell_set_accession in nomenclature_records: record_dict = nomenclature_records[node_cell_set_accession] node = {prop: record_dict[prop] for prop in headers} out['nodes'].append(node) children_str = record_dict['child_cell_set_accessions'] if children_str: children = set(children_str.strip().split('|')) else: children = {node_cell_set_accession} node = { "node_cell_set_accession": node_cell_set_accession, "children": children } child_cell_sets.append(node) sorted_child_cell_sets = sorted(child_cell_sets, key=lambda x: len(x["children"])) for child_cell_sets in sorted_child_cell_sets: parent_node = find_next_inclusive_node(sorted_child_cell_sets, child_cell_sets) if parent_node: out['edges'].add((child_cell_sets["node_cell_set_accession"], parent_node["node_cell_set_accession"])) fix_multi_inheritance_relations(out, sorted_child_cell_sets) return out
def check(self): files = [ f for f in os.listdir(MARKERS_FOLDER) if isfile(join(MARKERS_FOLDER, f)) ] for marker_file in files: if not is_whitelist_file(marker_file): dendrogram_path = join(DENDROGRAMS_FOLDER, get_taxonomy_file_name(marker_file)) print("DEND:" + dendrogram_path) if os.path.exists(dendrogram_path): headers, marker_records = read_csv_to_dict(join( MARKERS_FOLDER, marker_file), delimiter="\t") if str(dendrogram_path).endswith(".json"): dend = dend_json_2_nodes_n_edges(dendrogram_path) else: dend = nomenclature_2_nodes_n_edges(dendrogram_path) dend_dict = index_dendrogram(dend) self.check_cluster_name(dend_dict, marker_records, marker_file) else: message = "Could not find taxonomy file '{}' for marker '{}'." \ .format(dendrogram_path, marker_file) self.reports.append(message)
def add_cluster_name_to_marker(marker_path): path_parts = marker_path.split(os.path.sep) taxonomy_id = path_parts[len(path_parts) - 1].split("_")[0].replace("CS", "CCN") marker_data = read_csv_to_dict(marker_path, id_column_name="Taxonomy_node_ID", delimiter="\t")[1] nomenclature = read_csv_to_dict(NOMENCLATURE.format(taxonomy_id), id_column_name="cell_set_accession")[1] normalized_markers = [] for accession_id in marker_data: normalized_data = {"Taxonomy_node_ID": accession_id, "clusterName": nomenclature[accession_id]["original_label"], "Markers": marker_data[accession_id]["Markers"]} normalized_markers.append(normalized_data) class_robot_template = pd.DataFrame.from_records(normalized_markers) class_robot_template.to_csv(OUTPUT_MARKER.format(taxonomy_id.replace("CCN", "").replace("CS", "")), sep="\t", index=False)
def index_base_files(base_files): index = list() for base_file in base_files: headers, records = read_csv_to_dict(base_file, delimiter="\t", id_column_name="aligned_alias", id_to_lower=True) index.append(records) return index
def fix_gene_database_species(gene_db_path): headers, genes_by_id = read_csv_to_dict(gene_db_path, id_column=0, delimiter="\t") species_abbv = get_species_for_gene_db(gene_db_path) with open(gene_db_path.replace(".tsv", "_2.tsv"), mode='w') as out: writer = csv.writer(out, delimiter="\t", quotechar='"') writer.writerow(["ID", "TYPE", "NAME"]) writer.writerow(["ID", "SC %", "A rdfs:label"]) print(headers) for gene in genes_by_id: writer.writerow([genes_by_id[gene]["ID"], genes_by_id[gene]["TYPE"], genes_by_id[gene]["NAME"] + " (" + species_abbv + ")"])
def fix_gene_database(gene_db_path, gene_prefix): headers, genes_by_name = read_csv_to_dict(gene_db_path, id_column=1, delimiter="\t") species_abbv = get_species_for_gene_db(gene_db_path) with open(gene_db_path.replace(".tsv", "_2.tsv"), mode='w') as out: writer = csv.writer(out, delimiter="\t", quotechar='"') writer.writerow(["ID", "TYPE", "NAME"]) writer.writerow(["ID", "SC %", "A rdfs:label"]) print(headers) for gene in genes_by_name: writer.writerow([gene_prefix + gene.replace("\"", ""), "SO:0000704", genes_by_name[gene]["gene_name"] + " (" + species_abbv + ")"])
def check(self): files = [ f for f in os.listdir(MARKERS_FOLDER) if isfile(join(MARKERS_FOLDER, f)) ] for marker_file in files: if not is_whitelist_file(marker_file): dendrogram_path = join(DENDROGRAMS_FOLDER, get_taxonomy_file_name(marker_file)) if os.path.exists(dendrogram_path): headers, marker_records = read_csv_to_dict(join( MARKERS_FOLDER, marker_file), delimiter="\t") self.check_marker_names(marker_records, marker_file)
def check(self): files = [ f for f in os.listdir(MARKERS_FOLDER) if isfile(join(MARKERS_FOLDER, f)) ] for file in files: if not is_whitelist_file(file): header_row, records = read_csv_to_dict(join( MARKERS_FOLDER, file), delimiter="\t") if header_row != self.expected_headers: message = "Invalid column names: {} in file {}. Expected columns are: {}" \ .format(header_row, file, self.expected_headers) self.reports.append(message)
def generate_datasets_template(dataset_metadata_path, output_filepath): path_parts = dataset_metadata_path.split(os.path.sep) taxonomy_id = path_parts[len(path_parts) - 1].split("_")[0] headers, dataset_metadata = read_csv_to_dict(dataset_metadata_path, generated_ids=True) robot_template_seed = { 'ID': 'ID', 'TYPE': 'TYPE', 'Entity Type': 'TI %', 'Label': 'LABEL', 'Taxonomy': 'AI schema:includedInDataCatalog', 'Cell Count': "AT 'cell_count'^^xsd:integer", 'Nuclei Count': "AT 'nuclei_count'^^xsd:integer", 'Description': "A rdfs:comment", 'Download Link': "A schema:archivedAt", 'Explore Link': "A schema:discussionUrl" } dl = [robot_template_seed] dataset_index = 0 for dataset in dataset_metadata: d = dict() d['ID'] = 'PCL:' + get_dataset_id(taxonomy_id, dataset_index) d['TYPE'] = 'owl:NamedIndividual' d['Entity Type'] = 'schema:Dataset' # Taxonomy d['Label'] = dataset_metadata[dataset]['Dataset'] d['Taxonomy'] = 'PCL:' + get_taxonomy_id(taxonomy_id) cells_nuclei = dataset_metadata[dataset]['cells/nuclei'] if 'nuclei' in cells_nuclei: d['Nuclei Count'] = int(''.join(c for c in cells_nuclei if c.isdigit())) elif 'cells' in cells_nuclei: d['Cell Count'] = int(''.join(c for c in cells_nuclei if c.isdigit())) d['Description'] = dataset_metadata[dataset]['text'] d['Download Link'] = dataset_metadata[dataset]['download_link'] d['Explore Link'] = dataset_metadata[dataset]['explore_link'] dataset_index += 1 dl.append(d) robot_template = pd.DataFrame.from_records(dl) robot_template.to_csv(output_filepath, sep="\t", index=False)
def generate_taxonomies_template(taxonomy_metadata_path, output_filepath): taxon_configs = read_taxonomy_details_yaml() headers, taxonomies_metadata = read_csv_to_dict(taxonomy_metadata_path) robot_template_seed = { 'ID': 'ID', 'TYPE': 'TYPE', 'Entity Type': 'TI %', 'Label': 'LABEL', 'Number of Cell Types': "A 'cell_types_count'", 'Number of Cell Subclasses': "A 'cell_subclasses_count'", 'Number of Cell Classes': "A 'cell_classes_count'", 'Anatomic Region': "A 'has_brain_region'", 'Species Label': "A skos:prefLabel", 'Age': "A 'has_age'", 'Sex': "A 'has_sex'", 'Primary Citation': "A oboInOwl:hasDbXref" } dl = [robot_template_seed] for taxon_config in taxon_configs: d = dict() d['ID'] = 'PCL:' + get_taxonomy_id(taxon_config["Taxonomy_id"]) d['TYPE'] = 'owl:NamedIndividual' d['Entity Type'] = 'PCL:0010002' # Taxonomy d['Label'] = taxon_config["Taxonomy_id"] d['Anatomic Region'] = taxon_config['Brain_region'][0] d['Primary Citation'] = taxon_config['PMID'][0] if taxon_config["Taxonomy_id"] in taxonomies_metadata: taxonomy_metadata = taxonomies_metadata[ taxon_config["Taxonomy_id"]] d['Number of Cell Types'] = taxonomy_metadata["Cell Types"] d['Number of Cell Subclasses'] = taxonomy_metadata[ "Cell Subclasses"] d['Number of Cell Classes'] = taxonomy_metadata["Cell Classes"] d['Species Label'] = taxonomy_metadata["Species"] d['Age'] = taxonomy_metadata["Age"] d['Sex'] = taxonomy_metadata["Sex"] dl.append(d) robot_template = pd.DataFrame.from_records(dl) robot_template.to_csv(output_filepath, sep="\t", index=False)
def check(self): files = [ f for f in os.listdir(MARKERS_FOLDER) if isfile(join(MARKERS_FOLDER, f)) ] for marker_file in files: if not is_whitelist_file(marker_file): dendrogram_path = join(DENDROGRAMS_FOLDER, get_taxonomy_file_name(marker_file)) if os.path.exists(dendrogram_path): headers, marker_records = read_csv_to_dict(join( MARKERS_FOLDER, marker_file), delimiter="\t") if str(dendrogram_path).endswith(".json"): dend = dend_json_2_nodes_n_edges(dendrogram_path) else: dend = nomenclature_2_nodes_n_edges(dendrogram_path) dend_dict = index_dendrogram(dend) self.check_all_node_ids_valid(dend_dict, marker_records, marker_file) self.check_all_node_ids_unique(marker_file)
def normalize_raw_markers(raw_marker): """ Raw marker files has different structure than the expected. Needs these modifications: - Extract Taxonomy_node_ID: clusterName matches cell_set_aligned_alias of the dendrogram. - Resolve markers: convert marker names to ensemble IDs from local DBs Args: raw_marker: """ taxonomy_config = get_taxonomy_config(raw_marker) taxonomy_id = taxonomy_config["Taxonomy_id"] print("Taxonomy ID: " + taxonomy_id) if taxonomy_id == "CS1908210": print("Read dendrogram: " + taxonomy_id) dend = dend_json_2_nodes_n_edges(DENDROGRAM.format(taxonomy_id)) nomenclature_indexes = [ index_dendrogram(dend, id_field_name="cell_set_preferred_alias", id_to_lower=True), # index_dendrogram(dend, id_field_name="cell_set_aligned_alias", id_to_lower=True), index_dendrogram(dend, id_field_name="cell_set_accession", id_to_lower=True), index_dendrogram(dend, id_field_name="original_label", id_to_lower=True), index_dendrogram(dend, id_field_name="cell_set_additional_aliases", id_to_lower=True) ] else: print("Read nomenclature table: " + taxonomy_id) nomenclature_indexes = [read_csv_to_dict(NOMENCLATURE.format(taxonomy_id), id_column_name="cell_set_preferred_alias", id_to_lower=True)[1], read_csv_to_dict(NOMENCLATURE.format(taxonomy_id), id_column_name="cell_set_aligned_alias", id_to_lower=True)[1], read_csv_to_dict(NOMENCLATURE.format(taxonomy_id), id_column_name="cell_set_accession", id_to_lower=True)[1], read_csv_to_dict(NOMENCLATURE.format(taxonomy_id), id_column_name="original_label", id_to_lower=True)[1], read_csv_to_dict(NOMENCLATURE.format(taxonomy_id), id_column_name="cell_set_additional_aliases", id_to_lower=True)[1], ] gene_db_path = GENE_DB_PATH.format(str(taxonomy_config["Reference_gene_list"][0]).strip().lower()) headers, genes_by_name = read_csv_to_dict(gene_db_path, id_column=2, delimiter="\t", id_to_lower=True) species_abbv = get_species_for_gene_db(gene_db_path).lower() unmatched_markers = set() normalized_markers = [] if raw_marker.endswith(".csv"): headers, raw_marker_data = read_csv_to_dict(raw_marker, id_column_name="clusterName") else: headers, raw_marker_data = read_csv_to_dict(raw_marker, id_column_name="clusterName", delimiter="\t") for cluster_name in raw_marker_data: normalized_data = {} row = raw_marker_data[cluster_name] cluster_name_variants = [cluster_name.lower(), cluster_name.lower().replace("-", "/"), cluster_name.replace("Micro", "Microglia").lower()] nomenclature_node = search_terms_in_index(cluster_name_variants, nomenclature_indexes) if nomenclature_node: node_id = nomenclature_node["cell_set_accession"] marker_names = get_marker_names(row) marker_ids = [] for name in marker_names: if name: if species_abbv + " " + name.lower() in genes_by_name: marker_ids.append(str(genes_by_name[species_abbv + " " + name.lower()]["ID"])) elif species_abbv + " " + name.lower().replace("_", "-") in genes_by_name: marker_ids.append(str(genes_by_name[species_abbv + " " + name.lower().replace("_", "-")]["ID"])) else: unmatched_markers.add(name) normalized_data["Taxonomy_node_ID"] = node_id normalized_data["clusterName"] = nomenclature_node["cell_set_preferred_alias"] normalized_data["Markers"] = "|".join(marker_ids) normalized_markers.append(normalized_data) else: log.error("Node with cluster name '{}' couldn't be found in the nomenclature.".format(cluster_name)) # raise Exception("Node with cluster name {} couldn't be found in the nomenclature.".format(cluster_name)) class_robot_template = pd.DataFrame.from_records(normalized_markers) class_robot_template.to_csv(OUTPUT_MARKER.format(taxonomy_id.replace("CCN", "").replace("CS", "")), sep="\t", index=False) log.error("Following markers could not be found in the db ({}): {}".format(len(unmatched_markers), str(unmatched_markers)))