def parse_file(project_id, bq_dataset, bucket_name, file_data, filename,
outfilename, metadata, cloudsql_tables):
print 'Begin processing {0}.'.format(filename)
# connect to the cloud bucket
gcs = GcsConnector(project_id, bucket_name)
#main steps: download, convert to df, cleanup, transform, add metadata
filebuffer = gcs.download_blob_to_file(filename)
# convert blob into dataframe
data_df = convert_file_to_dataframe(filebuffer, skiprows=0, header=0)
# Get basic column information depending on datatype
column_mapping = get_column_mapping(metadata['DataType'])
data_df = cleanup_dataframe(data_df)
data_df.rename(columns=column_mapping, inplace=True)
# Get barcodes and update metadata_data table
# Assuming second scenario where each file is a different platform/pipeline combination
# TODO: Put in functionality for other scenario where all lists are in one file.
sample_barcodes = list(
[k for d, k in data_df['SampleBarcode'].iteritems()])
file_list = list([k for d, k in data_df['filenamepath'].iteritems()])
sample_metadata_list = []
for idx, barcode in enumerate(sample_barcodes):
new_metadata = metadata.copy()
new_metadata['sample_barcode'] = barcode
new_metadata['file_path'] = file_list[idx].replace('gs://', '')
sample_metadata_list.append(new_metadata)
update_metadata_data_list(cloudsql_tables['METADATA_DATA'],
sample_metadata_list)
def get_sdrf_info(project_id, bucket_name, disease_codes, header, set_index_col, search_patterns):
client = storage.Client(project_id)
bucket = client.get_bucket(bucket_name)
# connect to google cloud storage
gcs = GcsConnector(project_id, bucket_name)
sdrf_info = pd.DataFrame()
for disease_code in disease_codes:
for blob in bucket.list_blobs(prefix=disease_code):
sdrf_filename = blob.name
if not all(x in sdrf_filename for x in search_patterns):
continue
print(sdrf_filename)
filebuffer = gcs.download_blob_to_file(sdrf_filename)
# convert to a dataframe
sdrf_df = convert_file_to_dataframe(filebuffer, skiprows=0)
sdrf_df = cleanup_dataframe(sdrf_df)
sdrf_df["Study"] = disease_code
try:
sdrf_df = sdrf_df.set_index(set_index_col)
except:
sdrf_df = sdrf_df.set_index("Derived_Array_Data_File")
sdrf_info = sdrf_info.append(sdrf_df)
print("Done loading SDRF files.")
return sdrf_info
def get_sdrf_info(project_id, bucket_name, disease_codes, header,
set_index_col, search_patterns):
client = storage.Client(project_id)
bucket = client.get_bucket(bucket_name)
# connect to google cloud storage
gcs = GcsConnector(project_id, bucket_name)
sdrf_info = pd.DataFrame()
for disease_code in disease_codes:
for blob in bucket.list_blobs(prefix=disease_code):
sdrf_filename = blob.name
if not all(x in sdrf_filename for x in search_patterns):
continue
print(sdrf_filename)
filebuffer = gcs.download_blob_to_file(sdrf_filename)
# convert to a dataframe
sdrf_df = convert_file_to_dataframe(filebuffer, skiprows=0)
sdrf_df = cleanup_dataframe(sdrf_df)
sdrf_df['Study'] = disease_code
try:
sdrf_df = sdrf_df.set_index(set_index_col)
except:
sdrf_df = sdrf_df.set_index("Derived_Array_Data_File")
sdrf_info = sdrf_info.append(sdrf_df)
print("Done loading SDRF files.")
return sdrf_info
def parse_file(project_id, bq_dataset, bucket_name, file_data, filename, outfilename, metadata, cloudsql_tables):
print 'Begin processing {0}.'.format(filename)
# connect to the cloud bucket
gcs = GcsConnector(project_id, bucket_name)
#main steps: download, convert to df, cleanup, transform, add metadata
filebuffer = gcs.download_blob_to_file(filename)
# convert blob into dataframe
data_df = convert_file_to_dataframe(filebuffer, skiprows=0, header=0)
# Get basic column information depending on datatype
column_mapping = get_column_mapping(metadata['DataType'])
data_df = cleanup_dataframe(data_df)
data_df.rename(columns=column_mapping, inplace=True)
# Get barcodes and update metadata_data table
# Assuming second scenario where each file is a different platform/pipeline combination
# TODO: Put in functionality for other scenario where all lists are in one file.
sample_barcodes = list([k for d, k in data_df['SampleBarcode'].iteritems()])
file_list = list([k for d, k in data_df['filenamepath'].iteritems()])
sample_metadata_list = []
for idx, barcode in enumerate(sample_barcodes):
new_metadata = metadata.copy()
new_metadata['sample_barcode'] = barcode
new_metadata['file_path'] = file_list[idx].replace('gs://', '')
sample_metadata_list.append(new_metadata)
update_metadata_data_list(cloudsql_tables['METADATA_DATA'], sample_metadata_list)
def download_antibody_annotation_files(config, log):
object_key_template = config['protein']['aa_object_key_template']
aa_file_dir = config['protein']['aa_file_dir']
gcs = GcsConnector(config['project_id'], config['buckets']['open'])
studies = config['all_tumor_types']
nonrppa_studies = config['protein']['nonrppa']
log.info('\tstart downloading antibody annotation files to %s from %s:%s' %
(aa_file_dir, config['project_id'], config['buckets']['open']))
if not os.path.isdir(aa_file_dir):
os.makedirs(aa_file_dir)
for study in studies:
if study in nonrppa_studies:
continue
keypath = object_key_template % (study.lower(), study.upper())
log.info('\t\tdownloading %s' % (keypath))
tmpfile = gcs.download_blob_to_file(keypath)
with open(aa_file_dir + keypath[keypath.rindex('/'):], 'w') as outfile:
outfile.write(tmpfile.getvalue())
log.info('\tdone downloading antibody annotation files')
def generate_oncotator_inputfiles(project_id, bucket_name, filename,
outputfilename, oncotator_columns):
print(filename)
# NEW connection
gcs = GcsConnector(project_id, bucket_name)
filebuffer = gcs.download_blob_to_file(filename)
# convert blob into dataframe
try:
maf_df = convert_file_to_dataframe(filebuffer)
except:
print 'problem converting %s to a dataframe' % (filename)
raise
# clean-up dataframe
maf_df = cleanup_dataframe(maf_df)
print maf_df.columns
# lowercase the column names (WHY?)
maf_df.columns = map(lambda x: x.lower(), maf_df.columns)
#--------------------------------------------
# data - manipulation
#--------------------------------------------
maf_df["ncbi_build"] = maf_df["ncbi_build"].replace({
'hg19': '37',
'GRCh37': '37',
'GRCh37-lite': '37'
})
#---------------------------------------------
## Filters
## remember all the column names are lowercase
#---------------------------------------------
filters = {
"chromosome": map(str, range(1, 23)) + ['X', 'Y'],
"mutation_status": ['somatic', 'Somatic'],
"sequencer": ['Illumina HiSeq', 'Illumina GAIIx', 'Illumina MiSeq'],
"ncbi_build": ['37']
}
filter_checklist_df = maf_df.isin(filters)
filter_string = ((filter_checklist_df["chromosome"] == True)
& (filter_checklist_df["mutation_status"] == True)
& (filter_checklist_df["sequencer"] == True)
& (filter_checklist_df["ncbi_build"] == True))
maf_df = maf_df[filter_string]
#---------------------
#Oncotator part: generate intermediate files for Oncotator
#---------------------
# oncotator needs these columns
replace_column_names = {
"ncbi_build": 'build',
'chromosome': 'chr',
'start_position': 'start',
'end_position': 'end',
'reference_allele': 'ref_allele',
'tumor_seq_allele1': 'tum_allele1',
'tumor_seq_allele2': 'tum_allele2',
'tumor_sample_barcode': 'tumor_barcode',
'matched_norm_sample_barcode': 'normal_barcode'
}
# replace columns with new headings; just name change
for rcol in replace_column_names:
maf_df.columns = [
replace_column_names[x] if x == rcol else x for x in maf_df.columns
]
oncotator_columns = [
replace_column_names[y] if y == rcol else y
for y in oncotator_columns
]
# remove/mangle any duplicate columns ( we are naming line a, a.1, a.2 etc)
maf_df.columns = mangle_dupe_cols(maf_df.columns.values)
#---------------------
#Oncotator part: generate intermediate files for Oncotator
#---------------------
oncotator_df = maf_df[oncotator_columns]
print "df_columns", len(oncotator_df.columns)
df_stringIO = oncotator_df.to_csv(sep='\t',
index=False,
columns=oncotator_columns)
# upload the file
gcs.upload_blob_from_string(outputfilename, df_stringIO)
return True
def generate_oncotator_inputfiles(project_id, bucket_name, filename, outputfilename, oncotator_columns):
print (filename)
# NEW connection
gcs = GcsConnector(project_id, bucket_name)
filebuffer = gcs.download_blob_to_file(filename)
# convert blob into dataframe
maf_df = convert_file_to_dataframe(filebuffer)
# clean-up dataframe
maf_df = cleanup_dataframe(maf_df)
print maf_df.columns
# lowercase the column names (WHY?)
maf_df.columns = map(lambda x: x.lower(), maf_df.columns)
#--------------------------------------------
# data - manipulation
#--------------------------------------------
maf_df["ncbi_build"] = maf_df["ncbi_build"].replace({ 'hg19': '37'
,'GRCh37': '37'
,'GRCh37-lite': '37'
})
#---------------------------------------------
## Filters
## remember all the column names are lowercase
#---------------------------------------------
filters = {
"chromosome" : map(str,range(1,23)) + ['X', 'Y']
,"mutation_status": ['somatic', 'Somatic']
,"sequencer": ['Illumina HiSeq', 'Illumina GAIIx', 'Illumina MiSeq']
,"ncbi_build" : ['37']
}
filter_checklist_df = maf_df.isin(filters)
filter_string = (
(filter_checklist_df["chromosome"] == True)
&
(filter_checklist_df["mutation_status"] == True)
&
(filter_checklist_df["sequencer"] == True)
&
(filter_checklist_df["ncbi_build"] == True)
)
maf_df = maf_df[filter_string]
#---------------------
#Oncotator part: generate intermediate files for Oncotator
#---------------------
# oncotator needs these columns
replace_column_names = {
"ncbi_build" : 'build'
,'chromosome' : 'chr'
,'start_position' : 'start'
,'end_position' : 'end'
,'reference_allele' : 'ref_allele'
,'tumor_seq_allele1' : 'tum_allele1'
,'tumor_seq_allele2' : 'tum_allele2'
,'tumor_sample_barcode': 'tumor_barcode'
,'matched_norm_sample_barcode': 'normal_barcode'
}
# replace columns with new headings; just name change
for rcol in replace_column_names:
maf_df.columns = [replace_column_names[x] if x==rcol else x for x in maf_df.columns]
oncotator_columns = [replace_column_names[y] if y==rcol else y for y in oncotator_columns]
# remove/mangle any duplicate columns ( we are naming line a, a.1, a.2 etc)
maf_df.columns = mangle_dupe_cols(maf_df.columns.values)
#---------------------
#Oncotator part: generate intermediate files for Oncotator
#---------------------
oncotator_df = maf_df[oncotator_columns]
print "df_columns", len(oncotator_df.columns)
df_stringIO = oncotator_df.to_csv(sep='\t', index=False, columns= oncotator_columns)
# upload the file
gcs.upload_blob_from_string(outputfilename, df_stringIO)
return True
def main():
"""Parse GCT file, merge with barcodes info, melt(tidy)
and load to Google Storage and BigQuery
"""
project_id = ''
bucket_name = ''
# example file in bucket
filename = 'ccle/mRNA-gene-exp/CCLE_Expression_Entrez_2012-09-29.gct'
outfilename = 'test'
writer = ExcelWriter('ccle.xlsx')
# connect to the google cloud bucket
gcs = GcsConnector(project_id, bucket_name)
#------------------------------
# load the GCT file
# * the file has duplicate samples (columns)
#------------------------------
# To remove duplicates, load the first few lines of the file only. Get columns, unique and select dataframe
# this is a hack, but I cant find a elegant way to remove duplciates
gct_df = pd.read_table('CCLE_Expression_Entrez_2012-09-29.gct',
sep='\t',
skiprows=2,
mangle_dupe_cols=False,
nrows=2)
unqiue_columns = np.unique(gct_df.columns)
gct_df = pd.read_table('CCLE_Expression_Entrez_2012-09-29.gct',
sep='\t',
skiprows=2,
mangle_dupe_cols=True)
# clean-up the dataset/dataframe
gct_df = cleanup_dataframe(gct_df)
gct_df = gct_df[unqiue_columns]
# remove any gene_id starting with 'AFFX-'
gct_df[gct_df['Name'].str.startswith('AFFX-')].to_excel(
writer, sheet_name="affy_info")
gct_df = gct_df[~gct_df['Name'].str.startswith('AFFX-')]
#------------------------------
# HGNC validation
#-----------------------------
hgnc_df = hgnc_validation.get_hgnc_map()
hgnc_df.to_excel(writer, sheet_name="hgnc_info")
hgnc_dict = dict(zip(hgnc_df.entrez_id, hgnc_df.symbol))
gct_df['HGNC_gene_symbol'] = gct_df['Name'].map(
lambda gene_id: hgnc_dict.get(gene_id.replace('_at', ''), np.nan))
gct_df[['HGNC_gene_symbol', 'Name',
'Description']].to_excel(writer, sheet_name="gene_info")
gct_df['Name'] = gct_df['Name'].map(
lambda gene_id: gene_id.replace('_at', ''))
#------------------------------
# barcodes info
#------------------------------
barcodes_filename = 'ccle/mRNA-gene-exp/mRNA_names.out.tsv'
filebuffer = gcs.download_blob_to_file(barcodes_filename)
barcodes_df = pd.read_table(
filebuffer,
header=None,
names=['ParticipantBarcode', 'SampleBarcode',
'CCLE_long_name']) # convert into dataframe
barcodes_df = cleanup_dataframe(barcodes_df) # clean-up dataframe
#------------------------------
# ignore (drop) all of the columns from the gene-expression matrix
#that don't have corresponding Participant and Sample barcodes,
#------------------------------
columns_df = pd.DataFrame(unqiue_columns)
columns_df.columns = ['CCLE_long_name']
samples_map_df = pd.merge(columns_df,
barcodes_df,
on='CCLE_long_name',
how='inner')
samples_map_df.to_excel(writer, sheet_name="sample_info")
# select columns that are overlapping
overlapping_samples = samples_map_df['CCLE_long_name'].tolist()
overlapping_samples = overlapping_samples + [
'Name', 'Description', 'HGNC_gene_symbol'
]
gct_df = gct_df[overlapping_samples]
print gct_df
# melt the matrix
value_vars = [
col for col in gct_df.columns
if col not in ['Name', 'Description', 'HGNC_gene_symbol']
]
melted_df = pd.melt(gct_df,
id_vars=['Name', 'Description', 'HGNC_gene_symbol'],
value_vars=value_vars)
melted_df = melted_df.rename(
columns={
'Name': 'gene_id',
'Description': 'original_gene_symbol',
'variable': 'CCLE_long_name',
'value': 'RMA_normalized_expression'
})
# merge to get barcode information
# changed from outer join to inner join. In this case it shouldnt matter, since we already did a inner join
# while select the samples above.
data_df = pd.merge(melted_df,
samples_map_df,
on='CCLE_long_name',
how='inner')
data_df['Platform'] = "Affymetrix U133 Plus 2.0"
# reorder columns
col_order = [
"ParticipantBarcode", "SampleBarcode", "CCLE_long_name", "gene_id",
"HGNC_gene_symbol", "original_gene_symbol", "Platform",
"RMA_normalized_expression"
]
data_df = data_df[col_order]
# upload the contents of the dataframe in CSV format
print "Convert to CSV"
outfilename = "tcga/intermediary/CCLE_mrna_expr/bq_data_files/ccle_mrna_expr.csv"
df_string = data_df.to_csv(index=False, header=False)
status = gcs.upload_blob_from_string(outfilename, df_string)
print status
# save the excel file
writer.save()
def process_user_gen_files(project_id, user_project_id, study_id, bucket_name, bq_dataset, cloudsql_tables, files):
print 'Begin processing user_gen files.'
# connect to the cloud bucket
gcs = GcsConnector(project_id, bucket_name)
data_df = pd.DataFrame()
# Collect all columns that get passed in for generating BQ schema later
all_columns = []
# For each file, download, convert to df
for idx, file in enumerate(files):
blob_name = file['FILENAME'].split('/')[1:]
all_columns += file['COLUMNS']
metadata = {
'sample_barcode': file.get('SAMPLEBARCODE', ''),
'participant_barcode': file.get('PARTICIPANTBARCODE', ''),
'study_id': study_id,
'platform': file.get('PLATFORM', ''),
'pipeline': file.get('PIPELINE', ''),
'file_path': file['FILENAME'],
'file_name': file['FILENAME'].split('/')[-1],
'data_type': file['DATATYPE']
}
# download, convert to df
filebuffer = gcs.download_blob_to_file(blob_name)
# Get column mapping
column_mapping = get_column_mapping(file['COLUMNS'])
if idx == 0:
data_df = convert_file_to_dataframe(filebuffer, skiprows=0, header=0)
data_df = cleanup_dataframe(data_df)
data_df.rename(columns=column_mapping, inplace=True)
# Generate Metadata for this file
insert_metadata(data_df, metadata, cloudsql_tables['METADATA_DATA'])
else:
# convert blob into dataframe
new_df = convert_file_to_dataframe(filebuffer, skiprows=0, header=0)
new_df = cleanup_dataframe(new_df)
new_df.rename(columns=column_mapping, inplace=True)
# Generate Metadata for this file
insert_metadata(new_df, metadata, cloudsql_tables['METADATA_DATA'])
# TODO: Write function to check for participant barcodes, for now, we assume each file contains SampleBarcode Mapping
data_df = pd.merge(data_df, new_df, on='sample_barcode', how='outer')
# For complete dataframe, create metadata_samples rows
print 'Inserting into data into {0}.'.format(cloudsql_tables['METADATA_SAMPLES'])
data_df = cleanup_dataframe(data_df)
data_df['has_mrna'] = 0
data_df['has_mirna'] = 0
data_df['has_protein'] = 0
data_df['has_meth'] = 0
insert_metadata_samples(data_df, cloudsql_tables['METADATA_SAMPLES'])
# Update and create bq table file
temp_outfile = cloudsql_tables['METADATA_SAMPLES'] + '.out'
tmp_bucket = os.environ.get('tmp_bucket')
gcs.convert_df_to_njson_and_upload(data_df, temp_outfile, tmp_bucket=tmp_bucket)
# Using temporary file location (in case we don't have write permissions on user's bucket?
source_path = 'gs://' + tmp_bucket + '/' + temp_outfile
schema = generate_bq_schema(all_columns)
table_name = 'cgc_user_{0}_{1}'.format(user_project_id, study_id)
load_data_from_file.run(
project_id,
bq_dataset,
table_name,
schema,
source_path,
source_format='NEWLINE_DELIMITED_JSON',
write_disposition='WRITE_APPEND',
is_schema_file=False)
# Generate feature_defs
feature_defs = generate_feature_defs(study_id, project_id, bq_dataset, table_name, schema)
# Update feature_defs table
insert_feature_defs_list(cloudsql_tables['FEATURE_DEFS'], feature_defs)
# Delete temporary files
print 'Deleting temporary file {0}'.format(temp_outfile)
gcs = GcsConnector(project_id, tmp_bucket)
gcs.delete_blob(temp_outfile)
def parse_file(project_id, bq_dataset, bucket_name, file_data, filename, outfilename, metadata, cloudsql_tables):
print 'Begin processing {0}.'.format(filename)
# connect to the cloud bucket
gcs = GcsConnector(project_id, bucket_name)
#main steps: download, convert to df, cleanup, transform, add metadata
filebuffer = gcs.download_blob_to_file(filename)
# convert blob into dataframe
data_df = convert_file_to_dataframe(filebuffer, skiprows=0, header=0)
# clean-up dataframe
data_df = cleanup_dataframe(data_df)
new_df_data = []
map_values = {}
# Get basic column information depending on datatype
column_map = get_column_mapping(metadata['data_type'])
# Column headers are sample ids
for i, j in data_df.iteritems():
if i in column_map.keys():
map_values[column_map[i]] = [k for d, k in j.iteritems()]
else:
for k, m in j.iteritems():
new_df_obj = {}
new_df_obj['sample_barcode'] = i # Normalized to match user_gen
new_df_obj['project_id'] = metadata['project_id']
new_df_obj['study_id'] = metadata['study_id']
new_df_obj['Platform'] = metadata['platform']
new_df_obj['Pipeline'] = metadata['pipeline']
# Optional values
new_df_obj['Symbol'] = map_values['Symbol'][k] if 'Symbol' in map_values.keys() else ''
new_df_obj['ID'] = map_values['ID'][k] if 'ID' in map_values.keys() else ''
new_df_obj['TAB'] = map_values['TAB'][k] if 'TAB' in map_values.keys() else ''
new_df_obj['Level'] = m
new_df_data.append(new_df_obj)
new_df = pd.DataFrame(new_df_data)
# Get unique barcodes and update metadata_data table
sample_barcodes = list(set([k for d, k in new_df['sample_barcode'].iteritems()]))
sample_metadata_list = []
for barcode in sample_barcodes:
new_metadata = metadata.copy()
new_metadata['sample_barcode'] = barcode
sample_metadata_list.append(new_metadata)
update_metadata_data_list(cloudsql_tables['METADATA_DATA'], sample_metadata_list)
# Update metadata_samples table
update_molecular_metadata_samples_list(cloudsql_tables['METADATA_SAMPLES'], metadata['data_type'], sample_barcodes)
# Generate feature names and bq_mappings
table_name = file_data['BIGQUERY_TABLE_NAME']
feature_defs = generate_feature_Defs(metadata['data_type'], metadata['study_id'], project_id, bq_dataset, table_name, new_df)
# Update feature_defs table
insert_feature_defs_list(cloudsql_tables['FEATURE_DEFS'], feature_defs)
# upload the contents of the dataframe in njson format
tmp_bucket = os.environ.get('tmp_bucket')
gcs.convert_df_to_njson_and_upload(new_df, outfilename, metadata=metadata, tmp_bucket=tmp_bucket)
# Load into BigQuery
# Using temporary file location (in case we don't have write permissions on user's bucket?)
source_path = 'gs://' + tmp_bucket + '/' + outfilename
schema = get_molecular_schema()
load_data_from_file.run(
project_id,
bq_dataset,
table_name,
schema,
source_path,
source_format='NEWLINE_DELIMITED_JSON',
write_disposition='WRITE_APPEND',
is_schema_file=False)
# Delete temporary files
print 'Deleting temporary file {0}'.format(outfilename)
gcs = GcsConnector(project_id, tmp_bucket)
gcs.delete_blob(outfilename)
def parse_file(project_id, bq_dataset, bucket_name, file_data, filename,
outfilename, metadata, cloudsql_tables):
print 'Begin processing {0}.'.format(filename)
# connect to the cloud bucket
gcs = GcsConnector(project_id, bucket_name)
#main steps: download, convert to df, cleanup, transform, add metadata
filebuffer = gcs.download_blob_to_file(filename)
# convert blob into dataframe
data_df = convert_file_to_dataframe(filebuffer, skiprows=0, header=0)
# clean-up dataframe
data_df = cleanup_dataframe(data_df)
new_df_data = []
map_values = {}
# Get basic column information depending on datatype
column_map = get_column_mapping(metadata['data_type'])
# Column headers are sample ids
for i, j in data_df.iteritems():
if i in column_map.keys():
map_values[column_map[i]] = [k for d, k in j.iteritems()]
else:
for k, m in j.iteritems():
new_df_obj = {}
new_df_obj[
'sample_barcode'] = i # Normalized to match user_gen
new_df_obj['Project'] = metadata['project_id']
new_df_obj['Study'] = metadata['study_id']
new_df_obj['Platform'] = metadata['platform']
new_df_obj['Pipeline'] = metadata['pipeline']
# Optional values
new_df_obj['Symbol'] = map_values['Symbol'][
k] if 'Symbol' in map_values.keys() else ''
new_df_obj['ID'] = map_values['ID'][
k] if 'ID' in map_values.keys() else ''
new_df_obj['TAB'] = map_values['TAB'][
k] if 'TAB' in map_values.keys() else ''
new_df_obj['Level'] = m
new_df_data.append(new_df_obj)
new_df = pd.DataFrame(new_df_data)
# Get unique barcodes and update metadata_data table
sample_barcodes = list(
set([k for d, k in new_df['SampleBarcode'].iteritems()]))
sample_metadata_list = []
for barcode in sample_barcodes:
new_metadata = metadata.copy()
new_metadata['sample_barcode'] = barcode
sample_metadata_list.append(new_metadata)
update_metadata_data_list(cloudsql_tables['METADATA_DATA'],
sample_metadata_list)
# Update metadata_samples table
update_molecular_metadata_samples_list(cloudsql_tables['METADATA_SAMPLES'],
metadata['data_type'],
sample_barcodes)
# Generate feature names and bq_mappings
table_name = file_data['BIGQUERY_TABLE_NAME']
feature_defs = generate_feature_Defs(metadata['data_type'],
metadata['study_id'], project_id,
bq_dataset, table_name, new_df)
# Update feature_defs table
insert_feature_defs_list(cloudsql_tables['FEATURE_DEFS'], feature_defs)
# upload the contents of the dataframe in njson format
tmp_bucket = os.environ.get('tmp_bucket_location')
gcs.convert_df_to_njson_and_upload(new_df,
outfilename,
metadata=metadata,
tmp_bucket=tmp_bucket)
# Load into BigQuery
# Using temporary file location (in case we don't have write permissions on user's bucket?)
source_path = 'gs://' + tmp_bucket + '/' + outfilename
schema = get_molecular_schema()
load_data_from_file.run(project_id,
bq_dataset,
table_name,
schema,
source_path,
source_format='NEWLINE_DELIMITED_JSON',
write_disposition='WRITE_APPEND',
is_schema_file=False)
# Delete temporary files
print 'Deleting temporary file {0}'.format(outfilename)
gcs = GcsConnector(project_id, tmp_bucket)
gcs.delete_blob(outfilename)
def process_user_gen_files(project_id, user_project_id, study_id, bucket_name, bq_dataset, cloudsql_tables, files):
print 'Begin processing user_gen files.'
# connect to the cloud bucket
gcs = GcsConnector(project_id, bucket_name)
data_df = pd.DataFrame()
# Collect all columns that get passed in for generating BQ schema later
all_columns = []
# For each file, download, convert to df
for idx, file in enumerate(files):
blob_name = file['FILENAME'].split('/')[1:]
all_columns += file['COLUMNS']
metadata = {
'sample_barcode': file.get('SAMPLEBARCODE', ''),
'participant_barcode': file.get('PARTICIPANTBARCODE', ''),
'study_id': study_id,
'platform': file.get('PLATFORM', ''),
'pipeline': file.get('PIPELINE', ''),
'file_path': file['FILENAME'],
'file_name': file['FILENAME'].split('/')[-1],
'data_type': file['DATATYPE']
}
# download, convert to df
filebuffer = gcs.download_blob_to_file(blob_name)
# Get column mapping
column_mapping = get_column_mapping(file['COLUMNS'])
if idx == 0:
data_df = convert_file_to_dataframe(filebuffer, skiprows=0, header=0)
data_df = cleanup_dataframe(data_df)
data_df.rename(columns=column_mapping, inplace=True)
# Generate Metadata for this file
insert_metadata(data_df, metadata, cloudsql_tables['METADATA_DATA'])
else:
# convert blob into dataframe
new_df = convert_file_to_dataframe(filebuffer, skiprows=0, header=0)
new_df = cleanup_dataframe(new_df)
new_df.rename(columns=column_mapping, inplace=True)
# Generate Metadata for this file
insert_metadata(new_df, metadata, cloudsql_tables['METADATA_DATA'])
# TODO: Write function to check for participant barcodes, for now, we assume each file contains SampleBarcode Mapping
data_df = pd.merge(data_df, new_df, on='sample_barcode', how='outer')
# For complete dataframe, create metadata_samples rows
print 'Inserting into data into {0}.'.format(cloudsql_tables['METADATA_SAMPLES'])
data_df = cleanup_dataframe(data_df)
data_df['has_mrna'] = 0
data_df['has_mirna'] = 0
data_df['has_protein'] = 0
data_df['has_meth'] = 0
insert_metadata_samples(data_df, cloudsql_tables['METADATA_SAMPLES'])
# Update and create bq table file
temp_outfile = cloudsql_tables['METADATA_SAMPLES'] + '.out'
tmp_bucket = os.environ.get('tmp_bucket_location')
gcs.convert_df_to_njson_and_upload(data_df, temp_outfile, tmp_bucket=tmp_bucket)
# Using temporary file location (in case we don't have write permissions on user's bucket?
source_path = 'gs://' + tmp_bucket + '/' + temp_outfile
schema = generate_bq_schema(all_columns)
table_name = 'cgc_user_{0}_{1}'.format(user_project_id, study_id)
load_data_from_file.run(
project_id,
bq_dataset,
table_name,
schema,
source_path,
source_format='NEWLINE_DELIMITED_JSON',
write_disposition='WRITE_APPEND',
is_schema_file=False)
# Generate feature_defs
feature_defs = generate_feature_defs(study_id, project_id, bq_dataset, table_name, schema)
# Update feature_defs table
insert_feature_defs_list(cloudsql_tables['FEATURE_DEFS'], feature_defs)
# Delete temporary files
print 'Deleting temporary file {0}'.format(temp_outfile)
gcs = GcsConnector(project_id, tmp_bucket)
gcs.delete_blob(temp_outfile)
