def load_taneja_data(data_dir): data = init_study_data() study_key = 'Taneja_RA_Normal_Relatives' data['metadata'] = STUDY_METADATA[study_key] csv_fn = data_dir + PATIENT_DATA_URLS[study_key]['dest_filename'] data['attributes'] = file_utils.csv_file_column_names(csv_fn) sample_mappings = load_mappings(data_dir) sample_key_col = 'SampleID' raw_patients_data = file_utils.csv_file_to_nested_dict(csv_fn, sample_key_col) for raw_sample_key in raw_patients_data: raw_patient_data = raw_patients_data[raw_sample_key] #the spreadsheet has sample_ids that start with MWR and HLTY that do #not have entries in the sample_mappings and therefore(?) dont't have #any OTU data in the biom file to associate with. #the rows that have a SampleId that starts with 'V' are the good ones if raw_sample_key[0] == 'V': study_sample_key = 's' + (str(raw_sample_key)).lower() tornado_sample_key = sample_mappings[study_key][study_sample_key] patient_data = dict() patient_data['study_sample_key'] = study_sample_key patient_data['tornado_sample_key'] = tornado_sample_key relation_id = raw_patient_data['Relation'] if relation_id == raw_sample_key: patient_data['is_patient'] = True patient_data['is_family'] = False else: patient_data['is_patient'] = False patient_data['is_family'] = True data['patients'][tornado_sample_key] = patient_data return data
def load_crc_adenoma_data(data_dir): data = init_study_data() study_key = 'CRC_adenoma' data['metadata'] = STUDY_METADATA[study_key] csv_fn = data_dir + PATIENT_DATA_URLS[study_key]['dest_filename'] data['attributes'] = file_utils.csv_file_column_names(csv_fn) sample_mappings = load_mappings(data_dir) sample_key_col = 'SampleID' raw_patients_data = file_utils.csv_file_to_nested_dict(csv_fn, sample_key_col) for raw_sample_key in raw_patients_data: study_sample_key = 's' + (str(raw_sample_key)).lower() raw_patient_data = raw_patients_data[raw_sample_key] tornado_sample_key = sample_mappings[study_key][study_sample_key] patient_data = dict() patient_data['study_sample_key'] = study_sample_key patient_data['tornado_sample_key'] = tornado_sample_key diagnosis = raw_patient_data['Diagnosis'].strip() if diagnosis in ['cancer', 'adenoma', 'normal']: patient_data['Diagnosis'] = diagnosis else: raise Exception("Unexpected value in CRC Diagnosis Column: " + str(diagnosis)) polyps = raw_patient_data['POLYP.1=Y'] if polyps in ['1', '0']: patient_data['POLYP.1=Y'] = polyps else: raise Exception("Unexpected value in CRC POLYP.1=Y Column: " + str(polyps)) data['patients'][tornado_sample_key] = patient_data return data
def load_cdiff_data(data_dir): data = init_study_data() study_key = 'Cdiff' data['metadata'] = STUDY_METADATA[study_key] csv_fn = data_dir + PATIENT_DATA_URLS[study_key]['dest_filename'] data['attributes'] = file_utils.csv_file_column_names(csv_fn) sample_mappings = load_mappings(data_dir) sample_key_col = 'SampleID' raw_patients_data = file_utils.csv_file_to_nested_dict(csv_fn, sample_key_col) for raw_sample_key in raw_patients_data: study_sample_key = 'ssx' + (str(raw_sample_key)).lower() raw_patient_data = raw_patients_data[raw_sample_key] tornado_sample_key = sample_mappings[study_key][study_sample_key] patient_data = dict() patient_data['study_sample_key'] = study_sample_key patient_data['tornado_sample_key'] = tornado_sample_key cdiff_slug = raw_patient_data['Description'].strip() if cdiff_slug == 'Normal': cdiff_clean = 'normal' elif cdiff_slug in ['C-diff +', 'C-diff+']: cdiff_clean = 'positive' elif cdiff_slug in ['C-diff -', 'C-diff-'] : cdiff_clean = 'negative' else: raise Exception("Unknown cdiff Description: " + str(cdiff_slug)) patient_data['cdiff_level'] = cdiff_clean data['patients'][tornado_sample_key] = patient_data return data
def load_mappings(data_dir): """ nested dict of {study_key: {study_sample_key: tornado_sample_key}} """ fn = data_dir + MAPPINGS_FILE_DEST raw_mappings = file_utils.csv_file_to_nested_dict(fn, 'SampleID') clean_mappings = dict() for raw_sample_key in raw_mappings: raw_atts = raw_mappings[raw_sample_key] study_sample_key = raw_sample_key.lower() #tornado_sample_key = raw_atts['FileName'] tornado_sample_key = raw_sample_key study_key = raw_atts['Source_mapping_file'] if not study_key in clean_mappings: clean_mappings[study_key] = dict() clean_mappings[study_key][study_sample_key] = tornado_sample_key return clean_mappings
def construct_otu_metadata(taxa_filename, otu_ids): id_field = 'label' taxonomies = file_utils.csv_file_to_nested_dict(taxa_filename, id_field) for label in taxonomies: taxonomies[label]['species'] = 'unclassified' #the needed format for the biom format is like 'k__Bacteria', #except unclassifed is just 'unclassified' regardless of level #see biom_etl.extract_taxonomy_of_otu metadata = list() for otu_id in otu_ids: formatted_tax = list() for taxa_level in otus.TAXONOMY_LEVELS: taxa_name = taxonomies[otu_id][taxa_level] if taxa_name == 'unclassifed': taxa_formatted = taxa_name else: taxa_formatted = taxa_level[0] + '__' + taxa_name formatted_tax.append(taxa_formatted) otu_md = dict() otu_md['taxonomy'] = formatted_tax metadata.append(otu_md) return metadata
def load_lambert_vaccine_data(data_dir): data = init_study_data() study_key = 'Lambert_Nathaniel' data['metadata'] = STUDY_METADATA[study_key] csv_fn = data_dir + PATIENT_DATA_URLS[study_key]['dest_filename'] data['attributes'] = file_utils.csv_file_column_names(csv_fn) sample_mappings = load_mappings(data_dir) sample_key_col = 'SampleID' raw_patients_data = file_utils.csv_file_to_nested_dict(csv_fn, sample_key_col) for raw_sample_key in raw_patients_data: study_sample_key = 's' + (str(raw_sample_key)).lower() raw_patient_data = raw_patients_data[raw_sample_key] tornado_sample_key = sample_mappings[study_key][study_sample_key] patient_data = dict() patient_data['study_sample_key'] = study_sample_key patient_data['tornado_sample_key'] = tornado_sample_key days_since_vaccine_slug = raw_patient_data['Day'].strip() patient_data['days_since_vaccine'] = days_since_vaccine_slug data['patients'][tornado_sample_key] = patient_data return data
def write_and_read_a_csv_file(jcx): #when reading/writing csvs in python, the table is treated as a list of #rows, and every row is a dict of key-value pairs, where the key is a #column name and the value is the value of the table's cell. The columns #are unordered until you actually write the table to a file with #a specified column ordered. #Here, I am making a table with 2 columns named 'id' and 'color' #that looks like: # # id,color # 0,red # 1,blue # 2,green list_of_rows = list() #make the rows with just an 'id' column for row_idx in range(0, 3): data_row = dict() data_row['id'] = row_idx list_of_rows.append(data_row) #add another 'column' called 'color' to the three rows list_of_rows[0]['color'] = 'red' list_of_rows[1]['color'] = 'blue' list_of_rows[2]['color'] = 'green' #make a filename for our csv tmp_relative_fn = 'color_table.csv' #wix provides access to two data directories through the JobContext: # - 'get_run_local_data_dir' is a directory that only this run of the job # can access. It's intended for results files. # - 'get_job_global_data_dir' is a directory that each run of the same type # of job can access. It's intended for raw data that will be # processed on multiple runs and results that are cached between # runs (so they aren't recomputed each run) data_dir = jcx.get_run_local_data_dir() #the file goes in the working directory that wix gave us tmp_absolute_fn = os.path.join(data_dir, tmp_relative_fn) #the order we want the columns to be in the csv file column_names = ['id', 'color'] #use our file_utils to write the csv file #unfortunately, you really need to use my file_utils or things can go #wrong involving docker and file permissions jcx_user = jcx.get_container_user() file_utils.dump_csv(tmp_absolute_fn, column_names, list_of_rows, file_owner=jcx_user, ensure_dir=True) #now read it back in, completing the roundtrip. this method actually #returns a dict where the key is the 'id' row, and the value should be #identical to the row we wrote out roundtrip_rows = file_utils.csv_file_to_nested_dict(tmp_absolute_fn, 'id') #note that everything comes back as strings when you read them back in, #which is why the 'id' that is read back in is not an int jcx.log("The color of row '1' before and after roundtrip: " + list_of_rows[1]['color'] + ", " + roundtrip_rows['1']['color']) return
def load_fst_results_from_csv(comparison_key, data_dir): fn = filename_of_fst_results(comparison_key, data_dir) #corresponds to the tornado_observation_key, which is just a number x for OTU-x key_col = 'index' results_by_otu_num = file_utils.csv_file_to_nested_dict(fn, key_col) return results_by_otu_num