# CHOOSE CONVERTER
tool_id = choose_converter(input_file_format,output_file_format)
# INITIALIZE GALAXY
galaxy_instance = GalaxyInstance(url=base_url, key=apikey)
history_client = HistoryClient(galaxy_instance)
tool_client = ToolClient(galaxy_instance)
dataset_client = DatasetClient(galaxy_instance)
history = history_client.create_history('tmp')
# UPLOAD FILES
input_file_1 = tool_client.upload_file(input_file_full, history['id'], type='txt')
input_file_2 = tool_client.upload_file(input_file_full, history['id'], type='txt')
params = {'input_numbers_001':{'src': 'hda', 'id': input_file_1['outputs'][0]['id']},'input_numbers_002':{'src': 'hda', 'id': input_file_2['outputs'][0]['id']}}
wait_4_process(history['id'],"uploading files")
# RUN CONVERSION
runtool_output = tool_client.run_tool(history_id=history['id'], tool_id=tool_id, tool_inputs=params)
wait_4_process(history['id'],"running tool")
# DOWNLOAD CONVERTED FILE
download_output = dataset_client.download_dataset(runtool_output['jobs'][0]['id'],output_file_full, use_default_filename=False)
print download_output
class DataManagers:
def __init__(self, galaxy_instance, configuration):
"""
:param galaxy_instance: A GalaxyInstance object (import from bioblend.galaxy)
:param configuration: A dictionary. Examples in the ephemeris documentation.
"""
self.gi = galaxy_instance
self.config = configuration
self.tool_data_client = ToolDataClient(self.gi)
self.tool_client = ToolClient(self.gi)
self.possible_name_keys = ['name',
'sequence_name'] # In order of importance!
self.possible_value_keys = ['value', 'sequence_id',
'dbkey'] # In order of importance!
self.data_managers = self.config.get('data_managers')
self.genomes = self.config.get('genomes', '')
self.source_tables = DEFAULT_SOURCE_TABLES
self.fetch_jobs = []
self.skipped_fetch_jobs = []
self.index_jobs = []
self.skipped_index_jobs = []
def initiate_job_lists(self):
"""
Determines which data managers should be run to populate the data tables.
Distinguishes between fetch jobs (download files) and index jobs.
:return: populate self.fetch_jobs, self.skipped_fetch_jobs, self.index_jobs and self.skipped_index_jobs
"""
self.fetch_jobs = []
self.skipped_fetch_jobs = []
self.index_jobs = []
self.skipped_index_jobs = []
for dm in self.data_managers:
jobs, skipped_jobs = self.get_dm_jobs(dm)
if self.dm_is_fetcher(dm):
self.fetch_jobs.extend(jobs)
self.skipped_fetch_jobs.extend(skipped_jobs)
else:
self.index_jobs.extend(jobs)
self.skipped_index_jobs.extend(skipped_jobs)
def get_dm_jobs(self, dm):
"""Gets the job entries for a single dm. Puts entries that already present in skipped_job_list.
:returns job_list, skipped_job_list"""
job_list = []
skipped_job_list = []
items = self.parse_items(dm.get('items', ['']))
for item in items:
dm_id = dm['id']
params = dm['params']
inputs = dict()
# Iterate over all parameters, replace occurences of {{item}} with the current processing item
# and create the tool_inputs dict for running the data manager job
for param in params:
key, value = list(param.items())[0]
value_template = Template(value)
value = value_template.render(item=item)
inputs.update({key: value})
job = dict(tool_id=dm_id, inputs=inputs)
data_tables = dm.get('data_table_reload', [])
if self.input_entries_exist_in_data_tables(data_tables, inputs):
skipped_job_list.append(job)
else:
job_list.append(job)
return job_list, skipped_job_list
def dm_is_fetcher(self, dm):
"""Checks whether the data manager fetches a sequence instead of indexing.
This is based on the source table.
:returns True if dm is a fetcher. False if it is not."""
data_tables = dm.get('data_table_reload', [])
for data_table in data_tables:
if data_table in self.source_tables:
return True
return False
def data_table_entry_exists(self, data_table_name, entry, column='value'):
"""Checks whether an entry exists in the a specified column in the data_table."""
try:
data_table_content = self.tool_data_client.show_data_table(
data_table_name)
except Exception:
raise Exception('Table "%s" does not exist' % data_table_name)
try:
column_index = data_table_content.get('columns').index(column)
except IndexError:
raise IndexError('Column "%s" does not exist in %s' %
(column, data_table_name))
for field in data_table_content.get('fields'):
if field[column_index] == entry:
return True
return False
def input_entries_exist_in_data_tables(self, data_tables, input_dict):
"""Checks whether name and value entries from the input are already present in the data tables.
If an entry is missing in of the tables, this function returns False"""
value_entry = get_first_valid_entry(input_dict,
self.possible_value_keys)
name_entry = get_first_valid_entry(input_dict, self.possible_name_keys)
# Return False if name and value entries are both None
if not value_entry and not name_entry:
return False
# Check every data table for existence of name and value
# Return False as soon as entry is not present
for data_table in data_tables:
if value_entry:
if not self.data_table_entry_exists(
data_table, value_entry, column='value'):
return False
if name_entry:
if not self.data_table_entry_exists(
data_table, name_entry, column='name'):
return False
# If all checks are passed the entries are present in the database tables.
return True
def parse_items(self, items):
"""
Parses items with jinja2.
:param items: the items to be parsed
:return: the parsed items
"""
if bool(self.genomes):
items_template = Template(json.dumps(items))
rendered_items = items_template.render(
genomes=json.dumps(self.genomes))
# Remove trailing " if present
rendered_items = rendered_items.strip('"')
items = json.loads(rendered_items)
return items
def run(self, log=None, ignore_errors=False, overwrite=False):
"""
Runs the data managers.
:param log: The log to be used.
:param ignore_errors: Ignore erroring data_managers. Continue regardless.
:param overwrite: Overwrite existing entries in data tables
"""
self.initiate_job_lists()
all_succesful_jobs = []
all_failed_jobs = []
all_skipped_jobs = []
if not log:
log = logging.getLogger()
def run_jobs(jobs, skipped_jobs):
job_list = []
for skipped_job in skipped_jobs:
if overwrite:
log.info(
'%s already run for %s. Entry will be overwritten.' %
(skipped_job["tool_id"], skipped_job["inputs"]))
jobs.append(skipped_job)
else:
log.info('%s already run for %s. Skipping.' %
(skipped_job["tool_id"], skipped_job["inputs"]))
all_skipped_jobs.append(skipped_job)
for job in jobs:
started_job = self.tool_client.run_tool(
history_id=None,
tool_id=job["tool_id"],
tool_inputs=job["inputs"])
log.info(
'Dispatched job %i. Running DM: "%s" with parameters: %s' %
(started_job['outputs'][0]['hid'], job["tool_id"],
job["inputs"]))
job_list.append(started_job)
successful_jobs, failed_jobs = wait(self.gi, job_list, log)
if failed_jobs:
if not ignore_errors:
log.error('Not all jobs successful! aborting...')
raise RuntimeError('Not all jobs successful! aborting...')
else:
log.warning('Not all jobs successful! ignoring...')
all_succesful_jobs.extend(successful_jobs)
all_failed_jobs.extend(failed_jobs)
log.info(
"Running data managers that populate the following source data tables: %s"
% self.source_tables)
run_jobs(self.fetch_jobs, self.skipped_fetch_jobs)
log.info("Running data managers that index sequences.")
run_jobs(self.index_jobs, self.skipped_index_jobs)
log.info('Finished running data managers. Results:')
log.info('Successful jobs: %i ' % len(all_succesful_jobs))
log.info('Skipped jobs: %i ' % len(all_skipped_jobs))
log.info('Failed jobs: %i ' % len(all_failed_jobs))
InstallResults = namedtuple(
"InstallResults",
["successful_jobs", "failed_jobs", "skipped_jobs"])
return InstallResults(successful_jobs=all_succesful_jobs,
failed_jobs=all_failed_jobs,
skipped_jobs=all_skipped_jobs)
def runWorkflow(argDictionary, comparisons):
from bioblend.galaxy import GalaxyInstance
from bioblend.galaxy.histories import HistoryClient
from bioblend.galaxy.tools import ToolClient
from bioblend.galaxy.workflows import WorkflowClient
from bioblend.galaxy.libraries import LibraryClient
import time
api_key = ''
galaxy_host = 'http://localhost:8080/'
gi = GalaxyInstance(url=galaxy_host, key=api_key)
history_client = HistoryClient(gi)
tool_client = ToolClient(gi)
workflow_client = WorkflowClient(gi)
library_client = LibraryClient(gi)
history = history_client.create_history(row['accessionNumber'])
# Import the galaxy workflow
workflow = workflow_client.show_workflow('a799d38679e985db')
input_file = tool_client.upload_file(comparisons, history['id'], file_type='txt')
# Run workflow on csv data to create a new history.
params = dict()
for key in workflow['steps'].keys():
params[key] = argDictionary
datamap = {'1' : {'id': input_file['outputs'][0]['id'], 'src': 'hda'}}
workflow_client.invoke_workflow(workflow['id'], inputs = datamap, history_id = history['id'], params = params)
# A diry hack, we want to wait until we have all datasets
while getNumberNotComplete(history['id'], history_client) > 0:
time.sleep(10)
dataset_id = getFoldChangeData(history, history_client)['id']
return_collection = [{'accessionNo':argDictionary['accessionNumber'], 'foldChange': getUrl(dataset_id),
'PCA': getUrl(getMostRecentDatasetByName('PCAplot.png', history, history_client)['id']),'chrDirTable': getUrl(getMostRecentDatasetByName('chrDirTable.tabular', history, history_client)['id'])}]
number_of_comparisons = -1
for line in open(comparisons):
if not line.isspace():
number_of_comparisons += 1
for comparison in range(0, int(number_of_comparisons)):
tool_inputs = {
'foldChangeTable' : {'id': dataset_id, 'src': 'hda'},
'comparisonNumber' : comparison + 1
}
tool_client.run_tool(history['id'], 'cutFoldChangeTable', tool_inputs)
while getNumberNotComplete(history['id'], history_client) > 0:
time.sleep(10)
if argDictionary['species'] in ["Rat","Cow","Horse","Pig","Zebrafish"]:
pathwayAnalysisWorkflow = workflow_client.show_workflow('c9468fdb6dc5c5f1')
params = dict()
for key in pathwayAnalysisWorkflow['steps'].keys():
params[key] = argDictionary
if argDictionary['species'] == "Rat":
network=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="ratStringNetwork")
geneLengths=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="ratGeneLengths")
homology=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="Homology.rat.txt")
if argDictionary['species'] == "Cow":
network=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="cowStringNetwork")
geneLengths=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="cowGeneLengths")
homology=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="Homology.cow.txt")
if argDictionary['species'] == "Horse":
network=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="horseStringNetwork")
geneLengths=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="horseGeneLengths")
homology=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="Homology.horse.txt")
if argDictionary['species'] == "Pig":
network=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="pigStringNetwork.txt")
geneLengths=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="pigGeneLengths.tabular")
homology=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="Homology.pig.txt")
if argDictionary['species'] == "Zebrafish":
network=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="zebrafishStringNetwork")
geneLengths=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="zebrafishGeneLengths")
homology=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="HOM_AllOrganism.rpt")
pathwayDatamap = {'3' : {'id': homology, 'src': 'hda'},'2' : {'id': network, 'src': 'hda'},'1' : {'id': geneLengths, 'src': 'hda'}}
diffExpDataCollection = getDatasetsByName('cutTable.tabular', history, history_client)
for index, diffExpData in enumerate(diffExpDataCollection):
numCompleted = getNumberComplete(history['id'], history_client) + 10
print(numCompleted)
pathwayDatamap["0"] = {'id': diffExpData['id'], 'src': 'hda'}
workflow_client.invoke_workflow(pathwayAnalysisWorkflow['id'],
inputs = pathwayDatamap,
history_id = history['id'],
params = params)
comparisonDict = getRowFromCsv(comparisons, index)
if 'Factor1' in comparisonDict.keys():
comparisonDict['Factor'] = comparisonDict['Factor1'] + "." + comparisonDict['Factor2']
if 'Paired1' in comparisonDict.keys():
comparisonDict['Factor'] = comparisonDict['Paired1']
return_dict = {'accessionNo':argDictionary['accessionNumber'],
'factor':comparisonDict['Factor'],
'comparisonNum':comparisonDict['Numerator'],
'comparisonDenom':comparisonDict['Denominator'],
'foldChange': getUrl(diffExpData['id']),
'interactome': pathwayDatamap['0']['id'],
'exonLength': pathwayDatamap['2']['id']}
while getNumberComplete(history['id'], history_client) < numCompleted:
time.sleep(10)
return_dict['moduleNodes'] = getUrl(getMostRecentDatasetByName('moduleNodes.text',
history, history_client)['id'])
return_dict['modulePlots'] = getUrl(getMostRecentDatasetByName('modulePlots.pdf',
history, history_client)['id'])
return_dict['slimEnrichmentPathways'] = getUrl(getMostRecentDatasetByName('slimEnrichmentPathways.tabular',
history, history_client)['id'])
return_dict['slimEnrichmentPlot'] = getUrl(getMostRecentDatasetByName('slimEnrichmentPlot.png',
history, history_client)['id'])
return_collection.append(return_dict)
# Hard code keys to define the order
keys = ['accessionNo','factor','comparisonNum','comparisonDenom','PCA','chrDirTable','foldChange',
'interactome','exonLength','moduleNodes','modulePlots','enrichmentTable','slimEnrichmentPathways','slimEnrichmentPlot']
with open('output/' + argDictionary['accessionNumber'] + '-workflowOutput.csv', 'wb') as csvFile:
# Get headers from last dictionary in collection as first doesn't contain all keys
csvOutput = csv.DictWriter(csvFile, keys)
csvOutput.writeheader()
csvOutput.writerows(return_collection)
return return_collection
else:
pathwayAnalysisWorkflow = workflow_client.show_workflow('e85a3be143d5905b')
params = dict()
for key in pathwayAnalysisWorkflow['steps'].keys():
params[key] = argDictionary
# MouseGeneLengths.tab has id 457f69dd7016f307 - step 2 of workflow
# Mouse interactome has id 073be90ac6c3bce5 - step 0 of workflow
if argDictionary['species'] == "Mouse":
network=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="mouseStringNetwork")
geneLengths=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="MouseGeneLengths.tab")
homology=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="Homology.mouse.txt")
secretedReference=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="uniprot-secreted-mouse.txt")
pathwayDatamap = {'4' : {'id': secretedReference, 'src': 'hda'},'3' : {'id': homology, 'src': 'hda'},'2' : {'id': network, 'src': 'hda'},'1' : {'id': geneLengths, 'src': 'hda'}}
else:
network=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="humanStringNetwork")
geneLengths=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="geneLengths")
homology=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="Homology.mouse.txt")
secretedReference=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="uniprot-secreted-human.txt")
pathwayDatamap = {'4' : {'id': secretedReference, 'src': 'hda'},'3' : {'id': homology, 'src': 'hda'},'2' : {'id': network, 'src': 'hda'},'1' : {'id': geneLengths, 'src': 'hda'}}
diffExpDataCollection = getDatasetsByName('cutTable.tabular', history, history_client)
for index, diffExpData in enumerate(diffExpDataCollection):
numCompleted = getNumberComplete(history['id'], history_client) + 14
print(numCompleted)
pathwayDatamap["0"] = {'id': diffExpData['id'], 'src': 'hda'}
workflow_client.invoke_workflow(pathwayAnalysisWorkflow['id'],
inputs = pathwayDatamap,
history_id = history['id'],
params = params)
comparisonDict = getRowFromCsv(comparisons, index)
if 'Factor1' in comparisonDict.keys():
comparisonDict['Factor'] = comparisonDict['Factor1'] + "." + comparisonDict['Factor2']
if 'Paired1' in comparisonDict.keys():
comparisonDict['Factor'] = comparisonDict['Paired1']
return_dict = {'accessionNo':argDictionary['accessionNumber'],
'factor':comparisonDict['Factor'],
'comparisonNum':comparisonDict['Numerator'],
'comparisonDenom':comparisonDict['Denominator'],
'foldChange': getUrl(diffExpData['id']),
'interactome': pathwayDatamap['0']['id'],
'exonLength': pathwayDatamap['2']['id']}
while getNumberComplete(history['id'], history_client) < numCompleted:
time.sleep(10)
return_dict['moduleNodes'] = getUrl(getMostRecentDatasetByName('moduleNodes.text',
history, history_client)['id'])
return_dict['modulePlots'] = getUrl(getMostRecentDatasetByName('modulePlots.pdf',
history, history_client)['id'])
return_dict['pathways'] = getUrl(getMostRecentDatasetByName('pathways.tabular',
history, history_client)['id'])
return_dict['enrichPlot'] = getUrl(getMostRecentDatasetByName('enrichmentPlot.png',
history, history_client)['id'])
return_dict['enrichmentTable'] = getUrl(getMostRecentDatasetByName('TF_EnrichmentTable.tabular',
history, history_client)['id'])
return_dict['slimEnrichmentPathways'] = getUrl(getMostRecentDatasetByName('slimEnrichmentPathways.tabular',
history, history_client)['id'])
return_dict['slimEnrichmentPlot'] = getUrl(getMostRecentDatasetByName('slimEnrichmentPlot.png',
history, history_client)['id'])
return_collection.append(return_dict)
# Hard code keys to define the order
keys = ['accessionNo','factor','comparisonNum','comparisonDenom','PCA','chrDirTable','foldChange',
'interactome','exonLength','moduleNodes','modulePlots','pathways','enrichPlot','enrichmentTable','slimEnrichmentPathways','slimEnrichmentPlot']
with open('output/' + argDictionary['accessionNumber'] + '-workflowOutput.csv', 'wb') as csvFile:
# Get headers from last dictionary in collection as first doesn't contain all keys
csvOutput = csv.DictWriter(csvFile, keys)
csvOutput.writeheader()
csvOutput.writerows(return_collection)
return return_collection
def runWorkflow(argDictionary, comparisons,samples):
from bioblend.galaxy import GalaxyInstance
from bioblend.galaxy.histories import HistoryClient
from bioblend.galaxy.tools import ToolClient
from bioblend.galaxy.workflows import WorkflowClient
from bioblend.galaxy.libraries import LibraryClient
import tempfile
import time
api_key = ''
galaxy_host = ''
gi = GalaxyInstance(url=galaxy_host, key=api_key)
history_client = HistoryClient(gi)
tool_client = ToolClient(gi)
workflow_client = WorkflowClient(gi)
library_client = LibraryClient(gi)
history = history_client.create_history(argDictionary['accessionNumber'])
comparisonsTable = tool_client.upload_file(comparisons, history['id'], file_type='txt')
sampleTable = tool_client.upload_file(samples, history['id'], file_type='tabular')
if argDictionary['site'] == "ENA":
#fastqs available on ENA
tool_inputs = {
"accessionNumber":argDictionary["ENA"],"sampleTable":{'id': sampleTable['outputs'][0]['id'], 'src': 'hda'}
}
#run the tool to get the data from ENA
tool_client.run_tool(history['id'],'getRNASeqExpressionData', tool_inputs)
#we want to wait until we have all datasets
while getNumberNotComplete(history['id'], history_client) > 0:
time.sleep(10)
#sleep until all the fastq files are findable
time.sleep(120)
dirpath = tempfile.mkdtemp()
fileList = getDatasetsByApproxName("files.tabular", history,history_client)[0]
fileList = history_client.download_dataset(history["id"],fileList["id"],dirpath)
num_lines = sum(1 for line in open(fileList)) -1
datasets=list()
while len(datasets)!=num_lines:
time.sleep(10)
datasets = getDatasetsByApproxName("fastq",history,history_client )
else: #for SRA
if argDictionary['single'] == "TRUE":
with open(samples) as tsvfile:
reader = csv.DictReader(tsvfile, delimiter='\t')
for sample in reader:
print (sample)
fileNames=str.split(sample["File"],"|")
for fileName in fileNames:
tool_inputs = {
"input|input_select":"accession_number",
"outputformat":"fastqsanger.gz",
"input|accession":fileName
}
#run the tool to get the single data from SRA
tool_client.run_tool(history['id'],'toolshed.g2.bx.psu.edu/repos/iuc/sra_tools/fastq_dump/2.8.1.3', tool_inputs)
else:
with open(samples) as tsvfile:
reader = csv.DictReader(tsvfile, delimiter='\t')
for sample in reader:
tool_inputs = {
"accession_number":sample["File"]
}
#run the tool to get the paired data from SRA
tool_client.run_tool(history['id'],'toolshed.g2.bx.psu.edu/repos/mandorodriguez/fastqdump_paired/fastq_dump_paired/1.1.4', tool_inputs)
while getNumberNotComplete(history['id'], history_client) > 0:
time.sleep(10)
datasets = getDatasetsByApproxName("fastq",history,history_client )
#get the fastQC tool
for fastq in datasets:
try:
tool_inputs = {'input_file' : {'id': fastq['id'], 'src': 'hda'}}
tool_client.run_tool(history['id'],'toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/0.69', tool_inputs)
except Exception:
pass
#wait till complete
while getNumberNotComplete(history['id'], history_client) > 0:
time.sleep(10)
#make dataset collections for quantification using the fastq files
collections=list()
with open(samples) as tsvfile:
reader = csv.DictReader(tsvfile, delimiter='\t')
for row in reader:
datasets=list()
fileNames=str.split(row["File"],"|")
for fileName in fileNames:
datasets= datasets + getDatasetsByApproxName(fileName,history,history_client )
#make list of datasets
collections.append(makeDataSetCollection(datasets,row["Sample"],history,history_client))
#get the correct kallisto index
species = argDictionary['species'].lower()
index = getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name=species +"IndexFile")
index = {'id': index, 'src': 'hda'}
#run kallisto for every dataset collection
for collection in collections:
#set up the tool_inputs
tool_inputs = {'index' : index,'inputs' : {'id': collection['id'], 'src': 'hdca'} ,"single":argDictionary["single"],"stranded":argDictionary["stranded"]}
#often encounter connection broken error - possible problem with Certus server?
#bypass by ignoring the exception
tool_client.run_tool(history['id'],'kallistoQuant', tool_inputs)
# we want to wait until we have all datasets
while getNumberNotComplete(history['id'], history_client) > 0:
time.sleep(10)
# Run multiqc on kallisto logs and fastqc files
datasets = getDatasetsByApproxName("RawData",history,history_client )
kallistoLogs = getDatasetsByApproxName(".log", history, history_client)
tool_inputs = {}
for i, dataset in enumerate(datasets+kallistoLogs):
if not dataset["deleted"]:
if dataset in datasets:
software = 'fastqc'
else:
software = 'kallisto'
params = {'id' : dataset['id'], 'src': 'hda', 'name': dataset['name']}
tool_inputs.update({'results_%s|software_cond|software' % i: software, 'results_%s|input_file' % i: params})
# #summarise with the multiQC tool
tool_client.run_tool(history['id'],'multiqc', tool_inputs)
multiQc = getDatasetsByApproxName("multiqc",history,history_client)[0]
#get all the abundance files to convert to gene level counts matrix
datasets = getDatasetsByApproxName(".abundance",history,history_client )
#make a dataset collection for to make a countsMatrix
collection = makeDataSetCollection(datasets,"abundances",history,history_client)
#set up the tool_inputs
tool_inputs = {'inputs' : {'id': collection['id'], 'src': 'hdca'} ,"species":argDictionary['species']}
#convert abundances to gene level counts matrix
tool_client.run_tool(history['id'],'KallistoAbundancestoGeneCountMatrix', tool_inputs)
# A diry hack, we want to wait until we have all datasets
while getNumberNotComplete(history['id'], history_client) > 0:
time.sleep(10)
txi = getDatasetsByApproxName("txi",history,history_client)
#set up the tool_inputs for PCA
tool_inputs = {'txiData' : {'id': txi[0]['id'], 'src': 'hda'} ,'sampleTable' : {'id': sampleTable['outputs'][0]['id'], 'src': 'hda'} ,"species":argDictionary['species'],'technicalReplicates':argDictionary['technicalReplicates'],'batchCorrect':argDictionary['batchCorrect']}
#run deseq2
tool_client.run_tool(history['id'],'PCARNASeq', tool_inputs)
pca = getDatasetsByApproxName("PCA",history,history_client)[0]
#set up the tool_inputs for DESeq2
tool_inputs = {'txiData' : {'id': txi[0]['id'], 'src': 'hda'} ,'sampleTable' : {'id': sampleTable['outputs'][0]['id'], 'src': 'hda'} ,
'comparisonsTable' : {'id': comparisonsTable['outputs'][0]['id'], 'src': 'hda'} ,"foldChangeOnly":argDictionary['foldChangeOnly'],"species":argDictionary['species'],'technicalReplicates':argDictionary['technicalReplicates'],'batchCorrect':argDictionary['batchCorrect']}
#run deseq2
tool_client.run_tool(history['id'],'DESeq2FoldChange', tool_inputs)
#run chrdir
tool_client.run_tool(history['id'],'characteristicDirectionRNASeq', tool_inputs)
#we want to wait until we have all datasets
while getNumberNotComplete(history['id'], history_client) > 0:
time.sleep(10)
#get the foldchange data, cut and run pathway workflow
dataset_id = getFoldChangeData(history, history_client)['id']
return_collection = [{'accessionNo':argDictionary['accessionNumber'], 'foldChange': getUrl(dataset_id), 'PCA': getUrl(pca["id"]),'chrDirTable': getUrl(getMostRecentDatasetByName('chrDirTable.tabular', history, history_client)['id'])}]
number_of_comparisons = -1
for line in open(comparisons):
if not line.isspace():
number_of_comparisons += 1
for comparison in range(0, int(number_of_comparisons)):
tool_inputs = {
'foldChangeTable' : {'id': dataset_id, 'src': 'hda'},
'comparisonNumber' : comparison + 1
}
tool_client.run_tool(history['id'], 'cutFoldChangeTable', tool_inputs)
while getNumberNotComplete(history['id'], history_client) > 0:
time.sleep(10)
if argDictionary['species'] in ["Rat","Cow","Horse","Pig","Zebrafish"]:
pathwayAnalysisWorkflow = workflow_client.show_workflow('c9468fdb6dc5c5f1')
params = dict()
for key in pathwayAnalysisWorkflow['steps'].keys():
params[key] = argDictionary
if argDictionary['species'] == "Rat":
network=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="ratStringNetwork")
geneLengths=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="ratGeneLengths")
homology=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="HOM_AllOrganism.rpt")
if argDictionary['species'] == "Cow":
network=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="cowStringNetwork")
geneLengths=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="cowGeneLengths")
homology=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="HOM_AllOrganism.rpt")
if argDictionary['species'] == "Horse":
network=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="horseStringNetwork")
geneLengths=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="horseGeneLengths")
homology=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="Homology.horse.txt")
if argDictionary['species'] == "Pig":
network=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="pigStringNetwork.txt")
geneLengths=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="pigGeneLengths.tabular")
homology=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="Homology.pig.txt")
if argDictionary['species'] == "Zebrafish":
network=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="zebrafishStringNetwork")
geneLengths=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="zebrafishGeneLengths")
homology=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="HOM_AllOrganism.rpt")
pathwayDatamap = {'3' : {'id': homology, 'src': 'hda'},'2' : {'id': network, 'src': 'hda'},'1' : {'id': geneLengths, 'src': 'hda'}}
diffExpDataCollection = getDatasetsByName('cutTable.tabular', history, history_client)
for index, diffExpData in enumerate(diffExpDataCollection):
numCompleted = getNumberComplete(history['id'], history_client) + 10
print(numCompleted)
pathwayDatamap["0"] = {'id': diffExpData['id'], 'src': 'hda'}
workflow_client.invoke_workflow(pathwayAnalysisWorkflow['id'],
inputs = pathwayDatamap,
history_id = history['id'],
params = params)
comparisonDict = getRowFromCsv(comparisons, index)
if 'Factor1' in comparisonDict.keys():
comparisonDict['Factor'] = comparisonDict['Factor1'] + "." + comparisonDict['Factor2']
return_dict = {'accessionNo':argDictionary['accessionNumber'],
'factor':comparisonDict['Factor'],
'comparisonNum':comparisonDict['Numerator'],
'comparisonDenom':comparisonDict['Denominator'],
'foldChange': getUrl(diffExpData['id']),
'interactome': pathwayDatamap['0']['id'],
'exonLength': pathwayDatamap['2']['id']}
while getNumberComplete(history['id'], history_client) < numCompleted:
time.sleep(10)
return_dict['moduleNodes'] = getUrl(getMostRecentDatasetByName('moduleNodes.text',
history, history_client)['id'])
return_dict['modulePlots'] = getUrl(getMostRecentDatasetByName('modulePlots.pdf',
history, history_client)['id'])
return_dict['slimEnrichPathways'] = getUrl(getMostRecentDatasetByName('slimEnrichmentPathways.tabular',
history, history_client)['id'])
return_dict['enrichedDrugsReverse'] = getUrl(getMostRecentDatasetByName('enrichedDrugsReverse.tabular',
history, history_client)['id'])
return_dict['enrichedDrugsMimic'] = getUrl(getMostRecentDatasetByName('enrichedDrugsMimic.tabular',
history, history_client)['id'])
return_dict['enrichedTerms'] = getUrl(getMostRecentDatasetByName('enrichedTerms.tabular',
history, history_client)['id'])
return_dict['enrichedTerms.reduced'] = getUrl(getMostRecentDatasetByName('enrichedTerms.reduced.tabular',
history, history_client)['id'])
return_dict['GO.MDS'] = getUrl(getMostRecentDatasetByName('GO.MDS.html',
history, history_client)['id'])
return_collection.append(return_dict)
# Hard code keys to define the order
keys = ['accessionNo','multiQC','factor','PCA','chrDirTable','comparisonNum','comparisonDenom','foldChange',
'interactome','exonLength','moduleNodes','modulePlots',
'slimEnrichPathways','secretedProteins','enrichedDrugsReverse','enrichedDrugsMimic','enrichedTerms','enrichedTerms.reduced','GO.MDS']
outFileName = 'output/' + argDictionary['accessionNumber'] + '-workflowOutput.tsv'
with open(outFileName, 'wb') as csvFile:
# Get headers from last dictionary in collection as first doesn't contain all keys
csvOutput = csv.DictWriter(csvFile, keys, delimiter = "\t")
csvOutput.writeheader()
csvOutput.writerows(return_collection)
#tool_client.upload_file(outFileName, history['id'], file_type='tsv')
return return_collection
else:
pathwayAnalysisWorkflow = workflow_client.show_workflow('e85a3be143d5905b')
params = dict()
for key in pathwayAnalysisWorkflow['steps'].keys():
params[key] = argDictionary
if argDictionary['species'] == "Mouse":
network=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="mouseStringNetwork")
geneLengths=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="MouseGeneLengths.tab")
homology=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="Homology.mouse.txt")
secretedReference=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="uniprot-secreted-mouse.txt")
pathwayDatamap = {'4' : {'id': secretedReference, 'src': 'hda'},'3' : {'id': homology, 'src': 'hda'},'2' : {'id': network, 'src': 'hda'},'1' : {'id': geneLengths, 'src': 'hda'}}
else:
network=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="humanStringNetwork")
geneLengths=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="geneLengths")
homology=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="Homology.mouse.txt")
secretedReference=getLibraryToolDataID(history=history,history_client=history_client,library_client=library_client,name="uniprot-secreted-human.txt")
pathwayDatamap = {'4' : {'id': secretedReference, 'src': 'hda'},'3' : {'id': homology, 'src': 'hda'},'2' : {'id': network, 'src': 'hda'},'1' : {'id': geneLengths, 'src': 'hda'}}
diffExpDataCollection = getDatasetsByName('cutTable.tabular', history, history_client)
for index, diffExpData in enumerate(diffExpDataCollection):
numCompleted = getNumberComplete(history['id'], history_client) + 14
print(numCompleted)
pathwayDatamap["0"] = {'id': diffExpData['id'], 'src': 'hda'}
#pathwayDatamap['1'] = {'id': diffExpData['id'], 'src': 'hda'}
workflow_client.invoke_workflow(pathwayAnalysisWorkflow['id'],
inputs = pathwayDatamap,
history_id = history['id'],
params = params)
comparisonDict = getRowFromCsv(comparisons, index)
if 'Factor1' in comparisonDict.keys():
comparisonDict['Factor'] = comparisonDict['Factor1'] + "." + comparisonDict['Factor2']
return_dict = {'accessionNo':argDictionary['accessionNumber'],
'factor':comparisonDict['Factor'],
'comparisonNum':comparisonDict['Numerator'],
'comparisonDenom':comparisonDict['Denominator'],
'foldChange': getUrl(diffExpData['id']),
'interactome': pathwayDatamap['0']['id'],
'exonLength': pathwayDatamap['2']['id']}
while getNumberComplete(history['id'], history_client) < numCompleted:
time.sleep(10)
return_dict['moduleNodes'] = getUrl(getMostRecentDatasetByName('moduleNodes.text',
history, history_client)['id'])
return_dict['modulePlots'] = getUrl(getMostRecentDatasetByName('modulePlots.pdf',
history, history_client)['id'])
return_dict['pathways'] = getUrl(getMostRecentDatasetByName('pathways.tabular',
history, history_client)['id'])
return_dict['enrichPlot'] = getUrl(getMostRecentDatasetByName('enrichmentPlot.png',
history, history_client)['id'])
return_dict['enrichmentTable'] = getUrl(getMostRecentDatasetByName('TF_EnrichmentTable.tabular',
history, history_client)['id'])
return_dict['slimEnrichPathways'] = getUrl(getMostRecentDatasetByName('slimEnrichmentPathways.tabular',
history, history_client)['id'])
return_dict['secretedProteins'] = getUrl(getMostRecentDatasetByName('secretedProteins.tabular',
history, history_client)['id'])
return_dict['enrichedDrugsReverse'] = getUrl(getMostRecentDatasetByName('enrichedDrugsReverse.tabular',
history, history_client)['id'])
return_dict['enrichedDrugsMimic'] = getUrl(getMostRecentDatasetByName('enrichedDrugsMimic.tabular',
history, history_client)['id'])
return_dict['enrichedTerms'] = getUrl(getMostRecentDatasetByName('enrichedTerms.tabular',
history, history_client)['id'])
return_dict['enrichedTerms.reduced'] = getUrl(getMostRecentDatasetByName('enrichedTerms.reduced.tabular',
history, history_client)['id'])
return_dict['GO.MDS'] = getUrl(getMostRecentDatasetByName('GO.MDS.html',
history, history_client)['id'])
return_collection.append(return_dict)
# Hard code keys to define the order
keys = ['accessionNo','multiQC','factor','PCA','chrDirTable','comparisonNum','comparisonDenom','foldChange',
'interactome','exonLength','moduleNodes','modulePlots','pathways','enrichPlot', 'enrichmentTable',
'slimEnrichPathways','secretedProteins','enrichedDrugsReverse','enrichedDrugsMimic','enrichedTerms','enrichedTerms.reduced','GO.MDS']
outFileName = 'output/' + argDictionary['accessionNumber'] + '-workflowOutput.tsv'
with open(outFileName, 'wb') as csvFile:
# Get headers from last dictionary in collection as first doesn't contain all keys
csvOutput = csv.DictWriter(csvFile, keys, delimiter = "\t")
csvOutput.writeheader()
csvOutput.writerows(return_collection)
return return_collection