def rename_analysis_results(analysis):
"""Task for renaming files in file_store after download"""
logger.debug("analysis_manager.rename_analysis_results called")
# rename file_store items to new name updated from galaxy file_ids
analysis_results = AnalysisResult.objects.filter(
analysis_uuid=analysis.uuid)
for result in analysis_results:
# new name to load
new_file_name = result.file_name
# workaround for FastQC reports downloaded from Galaxy as zip archives
(root, ext) = os.path.splitext(new_file_name)
item = FileStoreItem.objects.get_item(uuid=result.file_store_uuid)
if ext == '.html' and item.get_filetype() == ZIP:
new_file_name = root + '.zip'
rename(result.file_store_uuid, new_file_name)
def addIGVSamples(fields, results_samp, annot_samples=None):
"""creates phenotype file for IGV
:param samples: Solr results for samples to be included
:type samples: Array.
:param annot_samples: includes annotation files included with solr results
:type annot_samples: Array
"""
# creates human readable indexes of fields to iterate over
fields_dict = {}
for i in fields:
find_index = i.find("_Characteristics_")
if find_index > -1:
new_key = i.split("_Characteristics_")[0]
fields_dict[i] = new_key
# Creating temp file to enter into file_store
tempsampname = tempfile.NamedTemporaryFile(delete=False)
# writing header to sample file
tempsampname.write("#sampleTable" + "\n")
# writing column names to sample file
col_names = "Linking_id"
for k, v in fields_dict.iteritems():
col_names = col_names + '\t' + v
tempsampname.write(col_names + "\n")
# iterating over sample files
pheno_results = get_sample_lines(fields_dict, results_samp)
tempsampname.write(pheno_results)
# if annotations are not null
if annot_samples:
pheno_annot = get_sample_lines(fields_dict, annot_samples)
tempsampname.write(pheno_annot)
# closing temp file
tempsampname.close()
# getting file_store_uuid
filestore_uuid = create(tempsampname.name, permanent=True, filetype="txt")
filestore_item = import_file(filestore_uuid, permanent=True, refresh=True)
# file to rename
temp_file = filestore_item.datafile.name.split('/')
temp_file = temp_file[len(temp_file) - 1] + '.txt'
# rename file by way of file_store
filestore_item = rename(filestore_uuid, temp_file)
# getting file information based on file_uuids
curr_fs = FileStoreItem.objects.get(uuid=filestore_uuid)
# full path to selected UUID File
curr_url = get_full_url(curr_fs.get_datafile_url())
# delete temp file
os.unlink(tempsampname.name)
return curr_url
def rename_analysis_results(analysis):
""" Task for renaming files in file_store after download
"""
logger.debug("analysis_manager.rename_analysis_results called")
# rename file_store items to new name updated from galaxy file_ids
#TODO: handle Django exceptions
analysis_results = AnalysisResult.objects.filter(analysis_uuid=analysis.uuid)
for result in analysis_results:
# new name to load
new_file_name = result.file_name
# rename file by way of file_store
filestore_item = rename(result.file_store_uuid, new_file_name)
def rename_analysis_results(analysis):
""" Task for renaming files in file_store after download
"""
logger.debug("analysis_manager.rename_analysis_results called")
# rename file_store items to new name updated from galaxy file_ids
#TODO: handle Django exceptions
analysis_results = AnalysisResult.objects.filter(
analysis_uuid=analysis.uuid)
for result in analysis_results:
# new name to load
new_file_name = result.file_name
# rename file by way of file_store
filestore_item = rename(result.file_store_uuid, new_file_name)
def create_igv_session(genome, uuids, is_file_uuid=False):
""" Creates session file for selected file uuids, returns newly created
filestore uuid
:param is_file_uuid:
:param genome: Genome to be used in session file i.e. hg18, dm3
:type genome: string.
:param uuids: Array of UUIDs to be used
:type uuids: array.
:param uuids: Host URL i.e. 127.0.0.1:8000
:type uuids: string
"""
# Create IGV Session file and put into Filestore
"""
http://www.postneo.com/projects/pyxml/
<?xml version="1.0" encoding="UTF-8"?>
<Global genome="hg18" locus="EGFR" version="3">
<Resources>
<Resource name="RNA Genes"
path="http://www.broadinstitute.org/igvdata/tcga/gbm/GBM_batch1-8_level3_exp.txt.recentered.080820.gct.tdf"/>
<Resource name="RNA Genes"
path="http://www.broadinstitute.org/igvdata/annotations/hg18/rna_genes.bed"/>
<Resource name="sno/miRNA"
path="http://www.broadinstitute.org/igvdata/tcga/gbm/Sample_info.txt"/>
</Resources>
</Global>
"""
logger.debug("visualization_manager.create_igv_session called")
# Create the minidom document
doc = Document()
# Create the <wml> base element
xml = doc.createElement("Global")
xml.setAttribute("genome", genome)
xml.setAttribute("locus", "All")
xml.setAttribute("version", "4")
doc.appendChild(xml)
# Add Resources
xml_resources = doc.createElement("Resources")
xml.appendChild(xml_resources)
# get paths to url
for samp in uuids:
# gets filestore item
curr_name, curr_url = get_file_name(samp, is_file_uuid=is_file_uuid)
logger.debug('New resource: ' + curr_name + ' - ' + curr_url)
# What to do if fs does not exist?
if curr_name:
# creates Resource element
res = doc.createElement("Resource")
res.setAttribute("name", curr_name)
res.setAttribute("path", curr_url)
xml_resources.appendChild(res)
# Creating temp file to enter into file_store
tempfilename = tempfile.NamedTemporaryFile(delete=False)
tempfilename.write(doc.toprettyxml(indent=" "))
tempfilename.close()
# getting file_store_uuid
filestore_uuid = create(tempfilename.name, filetype="xml")
filestore_item = import_file(filestore_uuid, refresh=True)
# file to rename
temp_name = filestore_item.datafile.name.split('/')
temp_name = temp_name[len(temp_name) - 1] + '.xml'
# rename file by way of file_store
filestore_item = rename(filestore_uuid, temp_name)
# delete temp file
os.unlink(tempfilename.name)
# Url for session file
fs_url = get_full_url(filestore_item.get_datafile_url())
# IGV url for automatic launch of Java Webstart
igv_url = "http://www.broadinstitute.org/igv/projects/current/igv.php" \
"?sessionURL=" + fs_url
return igv_url
def add_igv_samples(fields, results_samp, annot_samples=None):
"""creates phenotype file for IGV
:param samples: Solr results for samples to be included
:type samples: Array.
:param annot_samples: includes annotation files included with solr results
:type annot_samples: Array
"""
# creates human readable indexes of fields to iterate over
fields_dict = {}
for i in fields:
find_index = i.find("_Characteristics_")
if find_index > -1:
new_key = i.split("_Characteristics_")[0]
fields_dict[i] = new_key
# Creating temp file to enter into file_store
temp_sample_name = tempfile.NamedTemporaryFile(delete=False)
# writing header to sample file
temp_sample_name.write("#sampleTable" + "\n")
# writing column names to sample file
col_names = "Linking_id"
for k, v in fields_dict.iteritems():
col_names = col_names + '\t' + v
temp_sample_name.write(col_names + "\n")
# iterating over sample files
pheno_results = get_sample_lines(fields_dict, results_samp)
try:
temp_sample_name.write(pheno_results)
except UnicodeEncodeError as e:
logger.error("Could not write results to file: %s. "
"Trying again with the content to write encoded "
"properly." % e)
temp_sample_name.write(pheno_results.encode("utf-8"))
# if annotations are not null
if annot_samples:
pheno_annot = get_sample_lines(fields_dict, annot_samples)
temp_sample_name.write(pheno_annot)
# closing temp file
temp_sample_name.close()
# getting file_store_uuid
filestore_uuid = create(temp_sample_name.name, filetype="txt")
filestore_item = import_file(filestore_uuid, refresh=True)
# file to rename
temp_file = filestore_item.datafile.name.split('/')
temp_file = temp_file[len(temp_file) - 1] + '.txt'
# rename file by way of file_store
filestore_item = rename(filestore_uuid, temp_file)
# getting file information based on file_uuids
curr_fs = FileStoreItem.objects.get(uuid=filestore_uuid)
# full path to selected UUID File
curr_url = get_full_url(curr_fs.get_datafile_url())
# delete temp file
os.unlink(temp_sample_name.name)
return curr_url
def create_igv_session_annot(genome, uuids, annot_uuids=None, samp_file=None):
"""Creates session file for selected file uuids, returns newly created
filestore uuid
:param genome: Genome to be used in session file i.e. hg18, dm3
:type genome: string.
:param uuids: Array of UUIDs to be used
:type uuids: array.
:param uuids: Host URL i.e. 127.0.0.1:8000
:type uuids: string
"""
# Create IGV Session file and put into Filestore
"""
http://www.postneo.com/projects/pyxml/
<?xml version="1.0" encoding="UTF-8"?>
<Global genome="hg18" locus="EGFR" version="3">
<Resources>
<Resource name="RNA Genes"
path="http://www.broadinstitute.org/igvdata/tcga/gbm/GBM_batch1-8_level3_exp.txt.recentered.080820.gct.tdf"/>
<Resource name="RNA Genes"
path="http://www.broadinstitute.org/igvdata/annotations/hg18/rna_genes.bed"/>
<Resource name="sno/miRNA"
path="http://www.broadinstitute.org/igvdata/tcga/gbm/Sample_info.txt"/>
</Resources>
</Global>
"""
# Create the minidom document
doc = Document()
# Create the <wml> base element
xml = doc.createElement("Global")
xml.setAttribute("genome", genome)
xml.setAttribute("locus", "All")
xml.setAttribute("version", "4")
doc.appendChild(xml)
# Add Resources
xml_resources = doc.createElement("Resources")
xml.appendChild(xml_resources)
# adding selected samples to xml file
add_igv_resource(uuids["node_uuid"], xml_resources, doc)
if annot_uuids:
# adding selected samples to xml file
add_igv_resource(annot_uuids["node_uuid"], xml_resources, doc)
# adds sample information file to IGV session file
if samp_file:
# <Resource name="Sample Information"
# path="http://igv.broadinstitute.org/data/hg18/tcga/gbm/gbmsubtypes/sampleTable.txt.gz"/>
# creates Resource element
res = doc.createElement("Resource")
res.setAttribute("name", "Sample Information")
res.setAttribute("path", samp_file)
xml_resources.appendChild(res)
# <HiddenAttributes>
# <Attribute name="DATA FILE"/>
# <Attribute name="Linking_id"/>
# <Attribute name="DATA TYPE"/>
# </HiddenAttributes>
# Adding parameters to hide basic unnecessary sample info
hidden_attr = doc.createElement("HiddenAttributes")
xml.appendChild(hidden_attr)
attr = doc.createElement("Attribute")
attr.setAttribute("name", "DATA FILE")
hidden_attr.appendChild(attr)
attr = doc.createElement("Attribute")
attr.setAttribute("name", "Linking_id")
hidden_attr.appendChild(attr)
attr = doc.createElement("Attribute")
attr.setAttribute("name", "DATA TYPE")
hidden_attr.appendChild(attr)
# Creating temp file to enter into file_store
tempfilename = tempfile.NamedTemporaryFile(delete=False)
tempfilename.write(doc.toprettyxml(indent=" "))
tempfilename.close()
# getting file_store_uuid
filestore_uuid = create(tempfilename.name, filetype="xml")
filestore_item = import_file(filestore_uuid, refresh=True)
# file to rename
temp_name = filestore_item.datafile.name.split('/')
temp_name = temp_name[len(temp_name) - 1] + '.xml'
# rename file by way of file_store
filestore_item = rename(filestore_uuid, temp_name)
# delete temp file
os.unlink(tempfilename.name)
# Url for session file
sessionfile_url = get_full_url(filestore_item.get_datafile_url())
# IGV url for automatic launch of Java Webstart
igv_url = "http://www.broadinstitute.org/igv/projects/current/igv.php" \
"?sessionURL=" + sessionfile_url
return igv_url
def createIGVsession(genome, uuids, is_file_uuid=False):
""" Creates session file for selected file uuids, returns newly created filestore uuid
:param genome: Genome to be used in session file i.e. hg18, dm3
:type genome: string.
:param uuids: Array of UUIDs to be used
:type uuids: array.
:param uuids: Host URL i.e. 127.0.0.1:8000
:type uuids: string
"""
# Create IGV Session file and put into Filestore
"""
http://www.postneo.com/projects/pyxml/
<?xml version="1.0" encoding="UTF-8"?>
<Global genome="hg18" locus="EGFR" version="3">
<Resources>
<Resource name="RNA Genes" path="http://www.broadinstitute.org/igvdata/tcga/gbm/GBM_batch1-8_level3_exp.txt.recentered.080820.gct.tdf"/>
<Resource name="RNA Genes" path="http://www.broadinstitute.org/igvdata/annotations/hg18/rna_genes.bed"/>
<Resource name="sno/miRNA" path="http://www.broadinstitute.org/igvdata/tcga/gbm/Sample_info.txt"/>
</Resources>
</Global>
"""
logger.debug("visualization_manager.createIGVsession called")
# Create the minidom document
doc = Document()
# Create the <wml> base element
xml = doc.createElement("Global")
xml.setAttribute("genome", genome)
xml.setAttribute("locus", "All")
xml.setAttribute("version", "4")
doc.appendChild(xml)
# Add Resources
xml_resources = doc.createElement("Resources")
xml.appendChild(xml_resources)
# get paths to url
for samp in uuids:
# gets filestore item
curr_name, curr_url = get_file_name(samp, is_file_uuid=is_file_uuid)
logger.debug( 'New resource: ' + curr_name + ' - ' + curr_url )
# What to do if fs does not exist?
if (curr_name):
# creates Resource element
res = doc.createElement("Resource")
res.setAttribute("name", curr_name)
res.setAttribute("path", curr_url)
xml_resources.appendChild(res)
# Creating temp file to enter into file_store
tempfilename = tempfile.NamedTemporaryFile(delete=False)
tempfilename.write(doc.toprettyxml(indent=" "))
tempfilename.close()
# getting file_store_uuid
filestore_uuid = create(tempfilename.name, permanent=True, filetype="xml")
filestore_item = import_file(filestore_uuid, permanent=True, refresh=True)
# file to rename
temp_name = filestore_item.datafile.name.split('/')
temp_name = temp_name[len(temp_name)-1] + '.xml'
# rename file by way of file_store
filestore_item = rename(filestore_uuid, temp_name)
# delete temp file
os.unlink(tempfilename.name)
# Print our newly created XML
#print doc.toprettyxml(indent=" ")
#print filestore_item.datafile.url
# Url for session file
fs_url = filestore_item.get_full_url()
# IGV url for automatic launch of Java Webstart
igv_url = "http://www.broadinstitute.org/igv/projects/current/igv.php?sessionURL=" + fs_url
return igv_url
def addIGVSamples(fields, results_samp, annot_samples=None):
""" creates phenotype file for IGV
:param samples: Solr results for samples to be included
:type samples: Array.
:param annot_samples: includes annotation files included with solr results
:type annot_samples: Array
"""
#logger.debug("visualization_manager.views addIGVSamples called, fields=%s" % fields)
# creates human readable indexes of fields to iterate over
fields_dict = {}
for i in fields:
find_index = i.find("_Characteristics_")
if find_index > -1:
new_key = i.split("_Characteristics_")[0]
fields_dict[i] = new_key
# Creating temp file to enter into file_store
tempsampname = tempfile.NamedTemporaryFile(delete=False)
# writing header to sample file
tempsampname.write("#sampleTable" + "\n")
# writing column names to sample file
col_names = "Linking_id"
for k,v in fields_dict.iteritems():
col_names = col_names + '\t' + v
tempsampname.write(col_names + "\n")
# iterating over sample files
pheno_results = get_sample_lines(fields_dict, results_samp)
tempsampname.write(pheno_results)
# if annotations are not null
if annot_samples:
#results_annot = annot_samples["response"]["docs"]
pheno_annot = get_sample_lines(fields_dict, annot_samples)
tempsampname.write(pheno_annot)
# closing temp file
tempsampname.close()
# getting file_store_uuid
filestore_uuid = create(tempsampname.name, permanent=True, filetype="txt")
filestore_item = import_file(filestore_uuid, permanent=True, refresh=True)
# file to rename
temp_file = filestore_item.datafile.name.split('/')
temp_file = temp_file[len(temp_file)-1] + '.txt'
# rename file by way of file_store
filestore_item = rename(filestore_uuid, temp_file)
# getting file information based on file_uuids
curr_fs = FileStoreItem.objects.get(uuid=filestore_uuid)
curr_name = curr_fs.datafile.name
# full path to selected UUID File
curr_url = curr_fs.get_full_url()
# delete temp file
os.unlink(tempsampname.name)
return curr_url
def createIGVsessionAnnot(genome, uuids, annot_uuids=None, samp_file=None):
""" Creates session file for selected file uuids, returns newly created filestore uuid
:param genome: Genome to be used in session file i.e. hg18, dm3
:type genome: string.
:param uuids: Array of UUIDs to be used
:type uuids: array.
:param uuids: Host URL i.e. 127.0.0.1:8000
:type uuids: string
"""
# Create IGV Session file and put into Filestore
"""
http://www.postneo.com/projects/pyxml/
<?xml version="1.0" encoding="UTF-8"?>
<Global genome="hg18" locus="EGFR" version="3">
<Resources>
<Resource name="RNA Genes" path="http://www.broadinstitute.org/igvdata/tcga/gbm/GBM_batch1-8_level3_exp.txt.recentered.080820.gct.tdf"/>
<Resource name="RNA Genes" path="http://www.broadinstitute.org/igvdata/annotations/hg18/rna_genes.bed"/>
<Resource name="sno/miRNA" path="http://www.broadinstitute.org/igvdata/tcga/gbm/Sample_info.txt"/>
</Resources>
</Global>
"""
#logger.debug("visualization_manager.views createIGVsessionAnnot called")
# Create the minidom document
doc = Document()
# Create the <wml> base element
xml = doc.createElement("Global")
xml.setAttribute("genome", genome)
xml.setAttribute("locus", "All")
xml.setAttribute("version", "4")
doc.appendChild(xml)
# Add Resources
xml_resources = doc.createElement("Resources")
xml.appendChild(xml_resources)
# adding selected samples to xml file
addIGVResource(uuids["node_uuid"], xml_resources, doc)
if annot_uuids:
# adding selected samples to xml file
addIGVResource(annot_uuids["node_uuid"], xml_resources, doc)
# adds sample information file to IGV session file
if samp_file:
#<Resource name="Sample Information" path="http://igv.broadinstitute.org/data/hg18/tcga/gbm/gbmsubtypes/sampleTable.txt.gz"/>
# creates Resource element
res = doc.createElement("Resource")
res.setAttribute("name", "Sample Information")
res.setAttribute("path", samp_file)
xml_resources.appendChild(res)
#<HiddenAttributes>
# <Attribute name="DATA FILE"/>
# <Attribute name="Linking_id"/>
# <Attribute name="DATA TYPE"/>
#</HiddenAttributes>
# Adding parameters to hide basic unnecessary sample info
hidden_attr = doc.createElement("HiddenAttributes")
xml.appendChild(hidden_attr)
attr = doc.createElement("Attribute")
attr.setAttribute("name", "DATA FILE")
hidden_attr.appendChild(attr)
attr = doc.createElement("Attribute")
attr.setAttribute("name", "Linking_id")
hidden_attr.appendChild(attr)
attr = doc.createElement("Attribute")
attr.setAttribute("name", "DATA TYPE")
hidden_attr.appendChild(attr)
# Creating temp file to enter into file_store
tempfilename = tempfile.NamedTemporaryFile(delete=False)
tempfilename.write(doc.toprettyxml(indent=" "))
tempfilename.close()
# getting file_store_uuid
filestore_uuid = create(tempfilename.name, permanent=True, filetype="xml")
filestore_item = import_file(filestore_uuid, permanent=True, refresh=True)
# file to rename
temp_name = filestore_item.datafile.name.split('/')
temp_name = temp_name[len(temp_name)-1] + '.xml'
# rename file by way of file_store
filestore_item = rename(filestore_uuid, temp_name)
# delete temp file
os.unlink(tempfilename.name)
# Print our newly created XML
#logger.info( doc.toprettyxml(indent=" "))
#print filestore_item.datafile.url
# Url for session file
fs_url = filestore_item.get_full_url()
# IGV url for automatic launch of Java Webstart
igv_url = "http://www.broadinstitute.org/igv/projects/current/igv.php?sessionURL=" + fs_url
return igv_url
def add_igv_samples(fields, results_samp, annot_samples=None):
"""creates phenotype file for IGV
:param samples: Solr results for samples to be included
:type samples: Array.
:param annot_samples: includes annotation files included with solr results
:type annot_samples: Array
"""
# creates human readable indexes of fields to iterate over
fields_dict = {}
for i in fields:
find_index = i.find("_Characteristics_")
if find_index > -1:
new_key = i.split("_Characteristics_")[0]
fields_dict[i] = new_key
# Creating temp file to enter into file_store
temp_sample_name = tempfile.NamedTemporaryFile(delete=False)
# writing header to sample file
temp_sample_name.write("#sampleTable" + "\n")
# writing column names to sample file
col_names = "Linking_id"
for k, v in fields_dict.iteritems():
col_names = col_names + "\t" + v
temp_sample_name.write(col_names + "\n")
# iterating over sample files
pheno_results = get_sample_lines(fields_dict, results_samp)
try:
temp_sample_name.write(pheno_results)
except UnicodeEncodeError as e:
logger.error(
"Could not write results to file: %s. " "Trying again with the content to write encoded " "properly.", e
)
temp_sample_name.write(pheno_results.encode("utf-8"))
# if annotations are not null
if annot_samples:
pheno_annot = get_sample_lines(fields_dict, annot_samples)
temp_sample_name.write(pheno_annot)
# closing temp file
temp_sample_name.close()
# getting file_store_uuid
filestore_uuid = create(temp_sample_name.name, filetype="txt")
filestore_item = import_file(filestore_uuid, refresh=True)
# file to rename
temp_file = filestore_item.datafile.name.split("/")
temp_file = temp_file[len(temp_file) - 1] + ".txt"
# rename file by way of file_store
filestore_item = rename(filestore_uuid, temp_file)
# getting file information based on file_uuids
curr_fs = FileStoreItem.objects.get(uuid=filestore_uuid)
# full path to selected UUID File
curr_url = get_full_url(curr_fs.get_datafile_url())
# delete temp file
os.unlink(temp_sample_name.name)
return curr_url
