def check_contigs_in_bin_cache(self, binnedcontigs_ref, bin_id, token):
ws = Workspace(self.ws_url, token=token)
info = ws.get_object_info3({"objects": [{
"ref": binnedcontigs_ref
}]})['infos'][0]
# base64 encode the string so it is safe for filenames and still unique per contig id
b64key = base64.urlsafe_b64encode(
bin_id.encode("utf-8")).decode('utf-8')
inner_chsum = info[8] + '_' + b64key
index_file = os.path.join(
self.metagenome_index_dir,
inner_chsum + self.CONTIGS_SUFFIX + ".tsv.gz")
if not os.path.isfile(index_file):
if self.debug:
t1 = time.time()
# get the position in the array of the bin
binnedcontigs = ws.get_objects2({
'objects': [{
'ref': binnedcontigs_ref,
'included': ["/bins/[*]/bid"]
}]
})['data'][0]['data']
pos = 0
found = False
for b in binnedcontigs['bins']:
if b['bid'] == bin_id:
found = True
break
pos += 1
if not found:
raise ValueError('No Bin with ID: "' + bin_id + '" found.')
# get the contigs map
selection = ws.get_objects2({
'objects': [{
'ref':
binnedcontigs_ref,
'included': [
'/bins/' + str(pos) + '/contigs',
'/bins/' + str(pos) + '/bid'
]
}]
})['data'][0]['data']
if selection['bins'][0]['bid'] != bin_id:
raise ValueError(
'Something went wrong- bin ids do not match on second ws get_objects2 call'
)
contigs = selection['bins'][0]['contigs']
self.save_contigs_in_bin_tsv(contigs, inner_chsum)
if self.debug:
print(" (time=" + str(time.time() - t1) + ")")
return inner_chsum
def check_assembly_cache(self, ref, token):
ws = Workspace(self.ws_url, token=token)
info = ws.get_object_info3({"objects": [{"ref": ref}]})['infos'][0]
inner_chsum = info[8]
index_file = os.path.join(
self.assembly_index_dir,
inner_chsum + self.ASSEMBLY_SUFFIX + ".tsv.gz")
if not os.path.isfile(index_file):
if self.debug:
print(" Loading WS object...")
t1 = time.time()
if 'KBaseGenomeAnnotations.Assembly' in info[2]:
included = ["/contigs"]
assembly_data = ws.get_objects2(
{'objects': [{
'ref': ref,
'included': included
}]})['data'][0]['data']
contigs = list(assembly_data['contigs'].values())
self.save_assembly_tsv(contigs, inner_chsum)
elif 'KBaseGenomes.ContigSet' in info[2]:
included = [
"/contigs/[*]/id", "/contigs/[*]/length",
"/contigs/[*]/md5", "/contigs/[*]/description"
]
cs_data = ws.get_objects2(
{'objects': [{
'ref': ref,
'included': included
}]})['data'][0]['data']
contigs = []
for c in cs_data['contigs']:
this_contig_data = {'contig_id': ''}
if 'id' in c:
this_contig_data['contig_id'] = c['id']
if 'md5' in c:
this_contig_data['md5'] = c['md5']
if 'length' in c:
this_contig_data['length'] = c['length']
if 'description' in c:
this_contig_data['description'] = c['description']
contigs.append(this_contig_data)
self.save_assembly_tsv(contigs, inner_chsum)
else:
raise ValueError(
'The "ref" is not an Assembly or ContigSet data object. It was a '
+ info[2])
if self.debug:
print(" (time=" + str(time.time() - t1) + ")")
return inner_chsum
def test_build_hisat2_index_from_assembly_ok(self):
manager = Hisat2IndexManager(self.wsURL, self.callback_url,
self.scratch)
ws = Workspace(self.wsURL)
genome_obj_info = ws.get_objects2({
'objects': [{
'ref': self.genome_ref
}],
'no_data': 1
})
# get the list of genome refs from the returned info.
# if there are no refs (or something funky with the return), this will be an empty list.
# this WILL fail if data is an empty list. But it shouldn't be, and we know because
# we have a real genome reference, or get_objects2 would fail.
genome_obj_refs = genome_obj_info.get('data', [{}])[0].get('refs', [])
# see which of those are of an appropriate type (ContigSet or Assembly), if any.
assembly_ref = list()
ref_params = [{'ref': x} for x in genome_obj_refs]
ref_info = ws.get_object_info3({'objects': ref_params})
for idx, info in enumerate(ref_info.get('infos')):
if "KBaseGenomeAnnotations.Assembly" in info[
2] or "KBaseGenomes.ContigSet" in info[2]:
assembly_ref.append(";".join(ref_info.get('paths')[idx]))
assembly_ref = assembly_ref[0]
idx_prefix = manager.get_hisat2_index(assembly_ref)
self.assertIn("kb_hisat2_idx", idx_prefix)
def get_fasta_file(self, genome_ref):
ws = Workspace(self.ws_url)
# test if genome references an assembly type
# do get_objects2 without data. get list of refs
genome_obj_info = ws.get_objects2({
'objects': [{
'ref': genome_ref
}],
'no_data': 1
})
# get the list of genome refs from the returned info.
# if there are no refs (or something funky with the return), this will be an empty list.
# this WILL fail if data is an empty list. But it shouldn't be, and we know because
# we have a real genome reference, or get_objects2 would fail.
genome_obj_refs = genome_obj_info.get('data', [{}])[0].get('refs', [])
# see which of those are of an appropriate type (ContigSet or Assembly), if any.
assembly_ref = list()
ref_params = [{'ref': x} for x in genome_obj_refs]
ref_info = ws.get_object_info3({'objects': ref_params})
for idx, info in enumerate(ref_info.get('infos')):
if "KBaseGenomeAnnotations.Assembly" in info[
2] or "KBaseGenomes.ContigSet" in info[2]:
assembly_ref.append(";".join(ref_info.get('paths')[idx]))
# now just get the file.
au = AssemblyUtil(self.callback_url)
fasta_file = au.get_assembly_as_fasta({'ref': assembly_ref[0]})
return fasta_file["path"]
def fetch_fasta_from_genome(genome_ref, ws_url, callback_url):
"""
Returns an assembly or contigset as FASTA.
"""
if not check_ref_type(genome_ref, ['KBaseGenomes.Genome'], ws_url):
raise ValueError("The given genome_ref {} is not a KBaseGenomes.Genome type!")
# test if genome references an assembly type
# do get_objects2 without data. get list of refs
ws = Workspace(ws_url)
genome_obj_info = ws.get_objects2({
'objects': [{'ref': genome_ref}],
'no_data': 1
})
# get the list of genome refs from the returned info.
# if there are no refs (or something funky with the return), this will be an empty list.
# this WILL fail if data is an empty list. But it shouldn't be, and we know because
# we have a real genome reference, or get_objects2 would fail.
genome_obj_refs = genome_obj_info.get('data', [{}])[0].get('refs', [])
# see which of those are of an appropriate type (ContigSet or Assembly), if any.
assembly_ref = list()
ref_params = [{'ref': genome_ref + ";" + x} for x in genome_obj_refs]
ref_info = ws.get_object_info3({'objects': ref_params})
for idx, info in enumerate(ref_info.get('infos')):
if "KBaseGenomeAnnotations.Assembly" in info[2] or "KBaseGenomes.ContigSet" in info[2]:
assembly_ref.append(";".join(ref_info.get('paths')[idx]))
if len(assembly_ref) == 1:
return fetch_fasta_from_assembly(assembly_ref[0], ws_url, callback_url)
else:
raise ValueError("Multiple assemblies found associated with the given genome ref {}! "
"Unable to continue.")
def check_binnedcontigs_cache(self, ref, token):
ws = Workspace(self.ws_url, token=token)
info = ws.get_object_info3({"objects": [{"ref": ref}]})['infos'][0]
inner_chsum = info[8]
index_file = os.path.join(self.metagenome_index_dir,
inner_chsum + self.BIN_SUFFIX + ".tsv.gz")
if not os.path.isfile(index_file):
if self.debug:
print(" Loading WS object...")
t1 = time.time()
included = [
"/bins/[*]/bid", "/bins/[*]/gc", "/bins/[*]/n_contigs",
"/bins/[*]/sum_contig_len", "/bins/[*]/cov"
]
binnedcontigs = ws.get_objects2(
{'objects': [{
'ref': ref,
'included': included
}]})['data'][0]['data']
self.save_binnedcontigs_tsv(binnedcontigs["bins"], inner_chsum)
if self.debug:
print(" (time=" + str(time.time() - t1) + ")")
return inner_chsum
def get_one_genome(self, params, token=None):
"""Fetch a genome using WSLargeDataIO and return it as a python dict"""
callback_url = os.environ.get('SDK_CALLBACK_URL')
if callback_url:
print('fetching genome object using WsLargeDataIO')
ws_large_data = WsLargeDataIO(callback_url)
res = ws_large_data.get_objects(params)['data'][0]
data = json.load(open(res['data_json_file']))
else:
print('fetching genome object using Workspace')
ws_client = Workspace(self.ws_url, token=token)
data = ws_client.get_objects2(params)["data"][0]["data"]
return data
def search_orthologs_from_pangenome(self, token, ref, query, sort_by,
start, limit, num_found):
search_object = 'orthologs'
info_included = [
'id', 'type', 'function', 'md5', 'protein_translation', 'orthologs'
]
table_indexer = TableIndexer(token, self.ws_url)
ret = table_indexer.run_search(ref, self.pangenome_index_dir,
self.ORTHOLOGS_SUFFIX, search_object,
info_included, query, sort_by, start,
limit, num_found, self.debug)
for orthologs in ret['orthologs']:
orthologs_string = orthologs['orthologs']
if orthologs_string:
orthologs['orthologs'] = list(eval(orthologs_string))
if not isinstance(orthologs['orthologs'][0], list):
orthologs['orthologs'] = [orthologs['orthologs']]
ws = Workspace(self.ws_url, token=token)
genome_feature_function_map = {}
for orthologs in ret['orthologs']:
for orthologs_obj in orthologs['orthologs']:
gene_id = orthologs_obj[0]
if gene_id in genome_feature_function_map:
orthologs_obj.append(
genome_feature_function_map.get(gene_id))
else:
included = ["/features/[*]/function", "/features/[*]/id"]
object_info = ws.get_objects2({
'objects': [{
'ref': orthologs_obj[2],
'included': included
}]
})['data'][0]['data']
map(
lambda feature: genome_feature_function_map.update(
{feature.get('id'): feature.get('function')}),
object_info['features'])
orthologs_obj.append(
genome_feature_function_map.get(gene_id))
return ret
def check_object_cache(self, ref, search_object, info_included,
index_dir, object_suffix, debug):
ws = Workspace(self.ws_url, token=self.token)
info = ws.get_object_info3({"objects": [{"ref": ref}]})['infos'][0]
inner_chsum = info[8]
index_file = os.path.join(index_dir,
inner_chsum + object_suffix + ".tsv.gz")
if not os.path.isfile(index_file):
if debug:
print(" Loading WS object...")
t1 = time.time()
included = self.build_info_included(search_object, info_included)
object = ws.get_objects2({'objects': [{'ref': ref,
'included': included}]})['data'][0]['data']
self.save_object_tsv(object[search_object], inner_chsum, info_included,
index_dir, object_suffix)
if debug:
print(" (time=" + str(time.time() - t1) + ")")
return inner_chsum
class WorkspaceAdminUtil:
def __init__(self, config):
wsurl = config.get('workspace-url')
self.atoken = config.get('ws-admin-token')
self.noadmin = False
if self.atoken is None or self.atoken == '':
self.noadmin = True
self.atoken = config['token']
self.ws = Workspace(wsurl, token=self.atoken)
def list_objects(self, params):
"""
Provide something that acts like a standard listObjects
"""
if self.noadmin:
return self.ws.list_objects(params)
return self.ws.administer({'command': 'listObjects', 'params': params})
def get_objects2(self, params):
"""
Provide something that acts like a standard getObjects
"""
if self.noadmin:
return self.ws.get_objects2(params)
return self.ws.administer({'command': 'getObjects', 'params': params})
def get_workspace_info(self, params):
"""
Provide something that acts like a standard getObjects
"""
if self.noadmin:
return self.ws.get_workspace_info(params)
return self.ws.administer({
'command': 'getWorkspaceInfo',
'params': params
})
class AveExpressionMatrixBuilder:
def _validate_calculate_average_expression_matrix_params(self, params):
"""
_validate_calculate_average_expression_matrix_params:
validates params passed to calculate_average_expression_matrix method
"""
log('start validating calculate_average_expression_matrix params')
# check for required parameters
for p in ['expression_matrix_ref', 'output_suffix', 'workspace_name']:
if p not in params:
raise ValueError('"{}" parameter is required, but missing'.format(p))
def _generate_report(self, expression_matrix_ref, workspace_name):
"""
_generate_report: generate report
"""
objects_created = [{'ref': expression_matrix_ref,
'description': 'Average ExpressionMatrix'}]
report_params = {'message': '',
'workspace_name': workspace_name,
'objects_created': objects_created,
# 'html_links': output_html_files,
# 'direct_html_link_index': 0,
'html_window_height': 366,
'report_object_name': 'kb_ave_expr_matrix_report_' + str(uuid.uuid4())}
kbase_report_client = KBaseReport(self.callback_url, token=self.token)
output = kbase_report_client.create_extended_report(report_params)
report_output = {'report_name': output['name'], 'report_ref': output['ref']}
return report_output
def _save_expression_matrix(self, em_data, em_obj_name, workspace_name):
"""
_save_expression_matrix: saving ExpressionMatrix
"""
try:
log('saving ExpressionMatrix [{}]'.format(em_obj_name))
data_type = 'KBaseFeatureValues.ExpressionMatrix'
obj_info = self.dfu.save_objects({'id': self.dfu.ws_name_to_id(workspace_name),
'objects': [{'type': data_type,
'data': em_data,
'name': em_obj_name}]})[0]
except Exception as e:
log(e)
raise Exception('Failed Saving ExpressionMatrix to Workspace')
expression_matrix_ref = str(obj_info[6]) + '/' + str(obj_info[0]) + '/' + str(obj_info[4])
return expression_matrix_ref
def __init__(self, config):
self.ws_url = config["workspace-url"]
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.shock_url = config['shock-url']
self.ws = Workspace(self.ws_url, token=self.token)
self.dfu = DataFileUtil(self.callback_url)
self.scratch = config['scratch']
def calculate_average_expression_matrix(self, params):
"""
calculate_average_expression_matrix: create an average ExpressionMatrix object
from a ExpressionMatrix object
required params:
expression_matrix_ref: ExpressionMatrix object reference
output_suffix: output average ExpressionMatrix name suffix
workspace_name: the name of the workspace it gets saved to
return:
average_expression_matrix_ref: generated average ExpressionMatrix object reference
report_name: report name generated by KBaseReport
report_ref: report reference generated by KBaseReport
"""
log('--->\nrunning AveExpressionMatrixBuilder.calculate_average_expression_matrix\n' +
'params:\n{}'.format(json.dumps(params, indent=1)))
self._validate_calculate_average_expression_matrix_params(params)
expression_matrix_ref = params.get('expression_matrix_ref')
expression_matrix = self.ws.get_objects2({'objects':
[{'ref':
expression_matrix_ref}]})['data'][0]
expression_matrix_data = expression_matrix['data']
expression_matrix_info = expression_matrix['info']
condition_map = expression_matrix_data['condition_mapping']
ori_data = expression_matrix_data['data']
ori_col_ids = ori_data['col_ids']
ori_row_ids = ori_data['row_ids']
ori_values = ori_data['values']
labels = condition_map.keys()
if set(labels) != set(ori_col_ids):
error_msg = 'available labels: {}\n'.format(ori_col_ids)
error_msg += 'labels in condition_mapping: {}'.format(labels)
raise ValueError(error_msg)
condition_pos = {}
for label, condition in condition_map.iteritems():
if condition not in condition_pos:
condition_pos.update({condition: [ori_col_ids.index(label)]})
else:
condition_list = condition_pos[condition]
condition_list.append(ori_col_ids.index(label))
condition_pos.update({condition: condition_list})
conditions = condition_pos.keys()
ave_values = []
for ori_value in ori_values:
ave_value = [None] * len(conditions)
for condition, poss in condition_pos.iteritems():
ave_pos = conditions.index(condition)
sum_value = 0.0
for pos in poss:
sum_value += round(float(ori_value[pos]), 3)
average = sum_value / len(poss)
ave_value[ave_pos] = round(average, 2)
ave_values.append(ave_value)
average_data = {}
average_data.update({'row_ids': ori_row_ids})
average_data.update({'col_ids': conditions})
average_data.update({'values': ave_values})
em_data = {}
genome_ref = expression_matrix_data.get('genome_ref')
if genome_ref:
em_data.update({'genome_ref': genome_ref})
em_data.update({'scale': expression_matrix_data.get('scale')})
em_data.update({'type': expression_matrix_data.get('type')})
em_data.update({'feature_mapping': expression_matrix_data.get('feature_mapping')})
em_data.update({'condition_mapping': expression_matrix_data.get('condition_mapping')})
em_data.update({'data': average_data})
expression_matrix_name = expression_matrix_info[1]
ave_expression_matrix_name = expression_matrix_name + params.get('output_suffix')
workspace_name = params.get('workspace_name')
ave_expression_matrix_ref = self._save_expression_matrix(em_data,
ave_expression_matrix_name,
workspace_name)
returnVal = {'average_expression_matrix_ref': ave_expression_matrix_ref}
report_output = self._generate_report(ave_expression_matrix_ref,
workspace_name)
returnVal.update(report_output)
return returnVal
class DESeqUtil:
PREPDE_TOOLKIT_PATH = '/kb/deployment/bin/prepDE'
def _validate_run_deseq2_app_params(self, params):
"""
_validate_run_deseq2_app_params:
validates params passed to run_deseq2_app method
"""
log('start validating run_deseq2_app params')
# check for required parameters
for p in [
'expressionset_ref', 'differential_expression_set_suffix',
'workspace_name'
]:
if p not in params:
raise ValueError(
'"{}" parameter is required, but missing'.format(p))
def _mkdir_p(self, path):
"""
_mkdir_p: make directory for given path
"""
if not path:
return
try:
os.makedirs(path)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
def _xor(self, a, b):
return bool(a) != bool(b)
def _run_command(self, command):
"""
_run_command: run command and print result
"""
log('Start executing command:\n{}'.format(command))
pipe = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
output = pipe.communicate()[0]
exitCode = pipe.returncode
if (exitCode == 0):
log('Executed commend:\n{}\n'.format(command) +
'Exit Code: {}\nOutput:\n{}'.format(exitCode, output))
else:
error_msg = 'Error running commend:\n{}\n'.format(command)
error_msg += 'Exit Code: {}\nOutput:\n{}'.format(exitCode, output)
raise ValueError(error_msg)
def _generate_html_report(self, result_directory, diff_expression_obj_ref,
params):
"""
_generate_html_report: generate html summary report
"""
log('start generating html report')
html_report = list()
output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(output_directory)
result_file_path = os.path.join(output_directory, 'report.html')
result_directory
result_dirs = os.listdir(result_directory)
visualization_content = ''
for result_dir in result_dirs:
dispersion_plots_name = result_dir + '_dispersion_plots.png'
dispersion_plots_display_name = '{} {} dispersion plot'.format(
result_dir.split('_')[0],
result_dir.split('_')[1])
shutil.copy2(
os.path.join(result_directory, result_dir,
'deseq2_MAplot.png'),
os.path.join(output_directory, dispersion_plots_name))
visualization_content += '<div class="gallery">'
visualization_content += '<a target="_blank" href="{}">'.format(
dispersion_plots_name)
visualization_content += '<img src="{}" '.format(
dispersion_plots_name)
visualization_content += 'alt="{}" width="600" height="400">'.format(
dispersion_plots_display_name)
visualization_content += '</a><div class="desc">{}</div></div>'.format(
dispersion_plots_display_name)
pca_plots_name = result_dir + '_PCA_MAplot.png'
pca_plots_display_name = '{} {} PCA plot'.format(
result_dir.split('_')[0],
result_dir.split('_')[1])
shutil.copy2(
os.path.join(result_directory, result_dir, 'PCA_MAplot.png'),
os.path.join(output_directory, pca_plots_name))
visualization_content += '<div class="gallery">'
visualization_content += '<a target="_blank" href="{}">'.format(
pca_plots_name)
visualization_content += '<img src="{}" '.format(pca_plots_name)
visualization_content += 'alt="{}" width="600" height="400">'.format(
pca_plots_display_name)
visualization_content += '</a><div class="desc">{}</div></div>'.format(
pca_plots_display_name)
diff_expr_set_data = self.ws.get_objects2(
{'objects': [{
'ref': diff_expression_obj_ref
}]})['data'][0]['data']
items = diff_expr_set_data['items']
# expression_ref = self.expression_set_data['items'][0]['ref']
# expression_object = self.ws.get_objects2({'objects':
# [{'ref': expression_ref}]})['data'][0]
# expression_data = expression_object['data']
# genome_ref = expression_data['genome_id']
# genome_name = self.ws.get_object_info([{"ref": genome_ref}], includeMetadata=None)[0][1]
# feature_num = self.gsu.search({'ref': genome_ref})['num_found']
# genome_features = self.gsu.search({'ref': genome_ref,
# 'limit': feature_num,
# 'sort_by': [['feature_id', True]]})['features']
# feature_ids = []
# for genome_feature in genome_features:
# if not re.match('.*\.\d*', genome_feature.get('feature_id')):
# feature_ids.append(genome_feature.get('feature_id'))
# total_feature_num = len(feature_ids)
overview_content = ''
overview_content += '<br/><table><tr><th>Generated DifferentialExpressionMatrixSet'
overview_content += ' Object</th></tr>'
overview_content += '<tr><td>{} ({})'.format(
params.get('diff_expression_obj_name'), diff_expression_obj_ref)
overview_content += '</td></tr></table>'
overview_content += '<p><br/></p>'
overview_content += '<br/><table><tr><th>Generated DifferentialExpressionMatrix'
overview_content += ' Object</th><th></th><th></th><th></th></tr>'
overview_content += '<tr><th>Differential Expression Matrix Name</th>'
# overview_content += '<th>Reference Genome</th>'
# overview_content += '<th>Reference Genome Feature Count</th>'
overview_content += '<th>Feature Count</th>'
overview_content += '</tr>'
for item in items:
diff_expr_ref = item['ref']
diff_expr_object = self.ws.get_objects2(
{'objects': [{
'ref': diff_expr_ref
}]})['data'][0]
diff_expr_data = diff_expr_object['data']
diff_expr_info = diff_expr_object['info']
diff_expr_name = diff_expr_info[1]
number_features = len(diff_expr_data['data']['row_ids'])
overview_content += '<tr><td>{} ({})</td>'.format(
diff_expr_name, diff_expr_ref)
# overview_content += '<td>{} ({})</td>'.format(genome_name, genome_ref)
# overview_content += '<td>{}</td>'.format(total_feature_num)
overview_content += '<td>{}</td></tr>'.format(number_features)
overview_content += '</table>'
with open(result_file_path, 'w') as result_file:
with open(
os.path.join(os.path.dirname(__file__),
'report_template.html'),
'r') as report_template_file:
report_template = report_template_file.read()
report_template = report_template.replace(
'<p>Overview_Content</p>', overview_content)
report_template = report_template.replace(
'<p>Visualization_Content</p>', visualization_content)
result_file.write(report_template)
report_shock_id = self.dfu.file_to_shock({
'file_path': output_directory,
'pack': 'zip'
})['shock_id']
html_report.append({
'shock_id': report_shock_id,
'name': os.path.basename(result_file_path),
'label': os.path.basename(result_file_path),
'description': 'HTML summary report for DESeq2 App'
})
return html_report
def _generate_output_file_list(self, result_directory):
"""
_generate_output_file_list: zip result files and generate file_links for report
"""
log('start packing result files')
output_files = list()
output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(output_directory)
result_file = os.path.join(output_directory, 'DESeq2_result.zip')
plot_file = os.path.join(output_directory, 'DESeq2_plot.zip')
with zipfile.ZipFile(result_file,
'w',
zipfile.ZIP_DEFLATED,
allowZip64=True) as zip_file:
for root, dirs, files in os.walk(result_directory):
for file in files:
if not (file.endswith('.zip') or file.endswith('.png')
or file.endswith('.DS_Store')):
zip_file.write(
os.path.join(root, file),
os.path.join(os.path.basename(root), file))
output_files.append({
'path': result_file,
'name': os.path.basename(result_file),
'label': os.path.basename(result_file),
'description': 'File(s) generated by DESeq2 App'
})
with zipfile.ZipFile(plot_file,
'w',
zipfile.ZIP_DEFLATED,
allowZip64=True) as zip_file:
for root, dirs, files in os.walk(result_directory):
for file in files:
if file.endswith('.png'):
zip_file.write(
os.path.join(root, file),
os.path.join(os.path.basename(root), file))
output_files.append({
'path': plot_file,
'name': os.path.basename(plot_file),
'label': os.path.basename(plot_file),
'description': 'Visualization plots by DESeq2 App'
})
return output_files
def _generate_report(self, diff_expression_obj_ref, params,
result_directory):
"""
_generate_report: generate summary report
"""
log('creating report')
output_files = self._generate_output_file_list(result_directory)
output_html_files = self._generate_html_report(
result_directory, diff_expression_obj_ref, params)
diff_expr_set_data = self.ws.get_objects2(
{'objects': [{
'ref': diff_expression_obj_ref
}]})['data'][0]['data']
items = diff_expr_set_data['items']
description_set = 'DifferentialExpressionMatrixSet generated by DESeq2'
description_object = 'DifferentialExpressionMatrix generated by DESeq2'
objects_created = []
objects_created.append({
'ref': diff_expression_obj_ref,
'description': description_set
})
for item in items:
diff_expr_ref = item['ref']
objects_created.append({
'ref': diff_expr_ref,
'description': description_object
})
report_params = {
'message': '',
'workspace_name': params.get('workspace_name'),
'objects_created': objects_created,
'file_links': output_files,
'html_links': output_html_files,
'direct_html_link_index': 0,
'html_window_height': 333,
'report_object_name': 'kb_deseq2_report_' + str(uuid.uuid4())
}
kbase_report_client = KBaseReport(self.callback_url)
output = kbase_report_client.create_extended_report(report_params)
report_output = {
'report_name': output['name'],
'report_ref': output['ref']
}
return report_output
def _save_count_matrix_file(self, expressionset_ref, result_directory):
"""
_save_count_matrix_file: download gtf file for each expression
run prepDE.py on them and save reault count matrix file
"""
log('generating count matrix file')
items = self.expression_set_data['items']
gtf_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(gtf_directory)
transcript_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(transcript_directory)
for item in items:
expression_ref = item['ref']
expression_object = self.ws.get_objects2(
{'objects': [{
'ref': expression_ref
}]})['data'][0]
expression_data = expression_object['data']
expression_info = expression_object['info']
handle_id = expression_data.get('file').get('hid')
expression_name = expression_info[1]
tmp_gtf_directory = os.path.join(gtf_directory, expression_name)
self._mkdir_p(tmp_gtf_directory)
self.dfu.shock_to_file({
'handle_id': handle_id,
'file_path': tmp_gtf_directory,
'unpack': 'unpack'
})
tmp_transcript_directory = os.path.join(transcript_directory,
expression_name)
self._mkdir_p(tmp_transcript_directory)
cp_command = 'cp {} {}'.format(
os.path.join(tmp_gtf_directory, 'transcripts.gtf'),
tmp_transcript_directory)
self._run_command(cp_command)
self._run_prepDE(result_directory, transcript_directory)
def _run_prepDE(self, result_directory, input_directory):
"""
_run_prepDE: run prepDE.py script
ref: http://ccb.jhu.edu/software/stringtie/index.shtml?t=manual#deseq
"""
log('generating matrix of read counts')
command = self.PREPDE_TOOLKIT_PATH + '/prepDE.py '
command += '-i {} '.format(input_directory)
command += '-g {} '.format(
os.path.join(result_directory, 'gene_count_matrix.csv'))
command += '-t {} '.format(
os.path.join(result_directory, 'transcript_count_matrix.csv'))
self._run_command(command)
def _generate_diff_expression_csv(self, result_directory, alpha_cutoff,
fold_change_cutoff, condition_string):
"""
_generate_diff_expression_csv: get different expression matrix with DESeq2
"""
result_files = os.listdir(result_directory)
if 'gene_count_matrix.csv' not in result_files:
error_msg = 'Missing gene_count_matrix.csv, available files: {}'.format(
result_files)
raise ValueError(error_msg)
rcmd_list = [
'Rscript',
os.path.join(os.path.dirname(__file__), 'run_DESeq.R')
]
rcmd_list.extend(['--result_directory', result_directory])
rcmd_list.extend(['--alpha_cutoff', alpha_cutoff])
rcmd_list.extend(['--fold_change_cutoff', fold_change_cutoff])
rcmd_list.extend(['--condition_string', condition_string])
rcmd_str = " ".join(str(x) for x in rcmd_list)
self._run_command(rcmd_str)
def _get_condition_string(self, result_directory, condition_labels):
"""
_get_condition_string: get condition string corresponding to givin condition_labels
"""
log('generating condition string')
count_matrix_file = os.path.join(result_directory,
'gene_count_matrix.csv')
tmp_count_matrix_file = os.path.join(result_directory,
'tmp_gene_count_matrix.csv')
with open(count_matrix_file, "rb") as f:
reader = csv.reader(f)
columns = reader.next()[1:]
condition_list = [None] * len(columns)
items = self.expression_set_data.get('items')
expr_name_condition_mapping = {}
for item in items:
expression_ref = item['ref']
expr_object = self.ws.get_objects2(
{'objects': [{
'ref': expression_ref
}]})['data'][0]
expr_data = expr_object['data']
expr_info = expr_object['info']
expr_name = expr_info[1]
expr_condition = expr_data['condition']
expr_name_list = expr_name_condition_mapping.get(expr_condition)
if expr_name_list:
expr_name_list.append(expr_name)
expr_name_condition_mapping.update(
{expr_condition: expr_name_list})
else:
expr_name_condition_mapping.update(
{expr_condition: [expr_name]})
for condition_label in condition_labels:
if condition_label in expr_name_condition_mapping.keys():
expression_names = expr_name_condition_mapping.get(
condition_label)
for expression_name in expression_names:
pos = columns.index(expression_name)
condition_list[pos] = condition_label
else:
error_msg = 'Condition: {} is not availalbe. '.format(
condition_label)
error_msg += 'Available conditions: {}'.format(
expr_name_condition_mapping.keys())
raise ValueError(error_msg)
if None in condition_list:
filtered_pos = [0]
filtered_condition_list = []
for condition in condition_list:
if condition:
pos = [
i + 1 for i, val in enumerate(condition_list)
if val == condition
]
filtered_pos += pos
filtered_condition_list.append(condition)
filtered_pos = list(set(filtered_pos))
with open(count_matrix_file, "rb") as source:
rdr = csv.reader(source)
with open(tmp_count_matrix_file, "wb") as result:
wtr = csv.writer(result)
for r in rdr:
wtr.writerow(tuple(list(numpy.array(r)[filtered_pos])))
os.rename(tmp_count_matrix_file, count_matrix_file)
condition_string = ','.join(filtered_condition_list)
else:
condition_string = ','.join(condition_list)
return condition_string
def _save_diff_expression(self, result_directory, params):
"""
_save_diff_expression: save DifferentialExpression object to workspace
"""
log('start saving KBaseFeatureValues.DifferentialExpressionMatrix object'
)
workspace_name = params.get('workspace_name')
diff_expression_obj_name = params.get('diff_expression_obj_name')
destination_ref = workspace_name + '/' + diff_expression_obj_name
result_dirs = os.listdir(result_directory)
diff_expr_files = list()
for result_dir in result_dirs:
diff_expr_file = dict()
condition_labels = result_dir.split('_')
genes_results_filepath = os.path.join(result_directory, result_dir,
'deseq_results.csv')
with open(genes_results_filepath, "rb") as f:
reader = csv.reader(f)
columns = reader.next()[1:]
columns[columns.index('log2FoldChange')] = 'log2_fold_change'
columns[columns.index('pvalue')] = 'p_value'
columns[columns.index('padj')] = 'q_value'
for line in fileinput.input(genes_results_filepath, inplace=True):
if fileinput.isfirstline():
print 'gene_id,' + ','.join(columns)
else:
print line,
reader = csv.DictReader(open(genes_results_filepath))
diffexpr_filepath = os.path.join(
result_directory, result_dir,
'differential_expression_result.csv')
with open(diffexpr_filepath, 'w') as csvfile:
fieldnames = [
'gene_id', 'log2_fold_change', 'p_value', 'q_value'
]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for row in reader:
writer.writerow({
'gene_id':
row.get('gene_id'),
'log2_fold_change':
row.get('log2_fold_change'),
'p_value':
row.get('p_value'),
'q_value':
row.get('q_value')
})
diff_expr_file.update({
'condition_mapping': {
condition_labels[0]: condition_labels[1]
}
})
diff_expr_file.update({'diffexpr_filepath': diffexpr_filepath})
diff_expr_files.append(diff_expr_file)
expression_ref = self.expression_set_data['items'][0]['ref']
expression_data = self.ws.get_objects2(
{'objects': [{
'ref': expression_ref
}]})['data'][0]['data']
genome_ref = expression_data['genome_id']
upload_diff_expr_params = {
'destination_ref': destination_ref,
'diffexpr_data': diff_expr_files,
'tool_used': 'deseq',
'tool_version': '1.16.1',
'genome_ref': genome_ref
}
deu_upload_return = self.deu.save_differential_expression_matrix_set(
upload_diff_expr_params)
diff_expression_obj_ref = deu_upload_return['diffExprMatrixSet_ref']
return diff_expression_obj_ref
def _generate_deseq_files(self, result_directory, params):
"""
_generate_deseq_files: generate DESeq files
"""
gene_result_file = os.path.join(result_directory,
'gene_count_matrix.csv')
with open(gene_result_file, "rb") as f:
reader = csv.reader(f)
columns = reader.next()[1:]
for line in fileinput.input(gene_result_file, inplace=True):
if fileinput.isfirstline():
print 'gene_id,' + ','.join(columns)
else:
print line,
condition_string = self._get_condition_string(
result_directory, params.get('condition_labels'))
self._generate_diff_expression_csv(result_directory,
params.get('alpha_cutoff'),
params.get('fold_change_cutoff'),
condition_string)
def _get_condition_labels(self):
"""
_get_condition_labels: get all possible condition label pairs
"""
log('getting all possible condition pairs')
condition_label_pairs = []
items = self.expression_set_data.get('items')
condition_replicate_name_mapping = {}
for item in items:
expression_ref = item['ref']
expr_object = self.ws.get_objects2(
{'objects': [{
'ref': expression_ref
}]})['data'][0]
expr_data = expr_object['data']
expr_info = expr_object['info']
expr_name = expr_info[1]
expr_condition = expr_data['condition']
expr_name_list = condition_replicate_name_mapping.get(
expr_condition)
if expr_name_list:
expr_name_list.append(expr_name)
condition_replicate_name_mapping.update(
{expr_condition: expr_name_list})
else:
condition_replicate_name_mapping.update(
{expr_condition: [expr_name]})
condition_labels = condition_replicate_name_mapping.keys()
condition_label_pairs = [
list(pair) for pair in itertools.combinations(condition_labels, 2)
]
log('all pssible conditon pairs:\n{}'.format(condition_label_pairs))
return condition_label_pairs, condition_labels
def _check_input_labels(self, condition_pairs, available_condition_labels):
"""
_check_input_labels: check input condition pairs
"""
checked = True
for condition_pair in condition_pairs:
first_label = condition_pair['condition_label_1'][0].strip()
second_label = condition_pair['condition_label_2'][0].strip()
if first_label not in available_condition_labels:
error_msg = 'Condition: {} is not availalbe. '.format(
first_label)
error_msg += 'Available conditions: {}'.format(
available_condition_labels)
raise ValueError(error_msg)
if second_label not in available_condition_labels:
error_msg = 'Condition: {} is not availalbe. '.format(
second_label)
error_msg += 'Available conditions: {}'.format(
available_condition_labels)
raise ValueError(error_msg)
if first_label == second_label:
raise ValueError('Input conditions are the same')
return checked
def __init__(self, config):
self.ws_url = config["workspace-url"]
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.shock_url = config['shock-url']
self.dfu = DataFileUtil(self.callback_url)
self.rau = ReadsAlignmentUtils(self.callback_url)
self.deu = DifferentialExpressionUtils(self.callback_url,
service_ver='dev')
self.gsu = GenomeSearchUtil(self.callback_url)
self.ws = Workspace(self.ws_url, token=self.token)
self.scratch = config['scratch']
def run_deseq2_app(self, params):
"""
run_deseq2_app: run DESeq2 app
(https://www.bioconductor.org/packages/release/bioc/vignettes/DESeq2/inst/doc/DESeq2.html)
required params:
expressionset_ref: ExpressionSet object reference
differential_expression_set_suffix: DifferentialExpressoinMatrixSet object suffix
workspace_name: the name of the workspace it gets saved to
optional params:
run_all_combinations: run all paired condition combinations
condition_labels: conditions for expression set object
alpha_cutoff: q value cutoff
fold_change_cutoff: fold change cutoff
num_threads: number of threads
fold_scale_type: one of ["linear", "log2+1", "log10+1"]
return:
result_directory: folder path that holds all files generated by run_deseq2_app
diff_expression_obj_ref: generated RNASeqDifferetialExpression object reference
report_name: report name generated by KBaseReport
report_ref: report reference generated by KBaseReport
"""
log('--->\nrunning DESeqUtil.run_deseq2_app\n' +
'params:\n{}'.format(json.dumps(params, indent=1)))
self._validate_run_deseq2_app_params(params)
result_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(result_directory)
expressionset_ref = params.get('expressionset_ref')
expression_set_obj = self.ws.get_objects2(
{'objects': [{
'ref': expressionset_ref
}]})['data'][0]
self.expression_set_data = expression_set_obj['data']
expression_set_name = expression_set_obj['info'][1]
differential_expression_set_suffix = params.get(
'differential_expression_set_suffix')
if re.match('.*_*[Ee]xpression_*[Ss]et', expression_set_name):
diff_expression_obj_name = re.sub(
'_*[Ee]xpression_*[Ss]et', differential_expression_set_suffix,
expression_set_name)
else:
diff_expression_obj_name = expression_set_name + differential_expression_set_suffix
params['diff_expression_obj_name'] = diff_expression_obj_name
available_condition_label_pairs, available_condition_labels = self._get_condition_labels(
)
run_all_combinations = params.get('run_all_combinations')
condition_pairs = params.get('condition_pairs')
if not self._xor(run_all_combinations, condition_pairs):
error_msg = "Invalid input:\nselect 'Run All Paired Condition Combinations' "
error_msg += "or provide partial condition pairs. Don't do both"
raise ValueError(error_msg)
if run_all_combinations:
condition_label_pairs = available_condition_label_pairs
else:
if self._check_input_labels(condition_pairs,
available_condition_labels):
condition_label_pairs = list()
for condition_pair in condition_pairs:
condition_labels = [
condition_pair.get('condition_label_1')[0].strip(),
condition_pair.get('condition_label_2')[0].strip()
]
condition_label_pairs.append(condition_labels)
for condition_label_pair in condition_label_pairs:
params['condition_labels'] = condition_label_pair
dir_suffix = '_'.join(condition_label_pair)
sub_result_directory = os.path.join(result_directory, dir_suffix)
self._mkdir_p(sub_result_directory)
# run prepDE.py and save count matrix file
self._save_count_matrix_file(expressionset_ref,
sub_result_directory)
self._generate_deseq_files(sub_result_directory, params)
diff_expression_obj_ref = self._save_diff_expression(
result_directory, params)
returnVal = {
'result_directory': result_directory,
'diff_expression_obj_ref': diff_expression_obj_ref
}
report_output = self._generate_report(diff_expression_obj_ref, params,
result_directory)
returnVal.update(report_output)
return returnVal
class Service:
def __init__(self, fba_url, ws_url, ctx):
self.ws_client = Workspace(ws_url, token=ctx['token'])
self.fba_client = fba_tools(fba_url)
def get_object(self, objid, wsid, name=None):
"""
Returns an object and it's associated KBase information
Returns an ObjectData (dictionary) like what is returned in the workspace service 'get_objects' function:
/* The data and supplemental info for an object.
UnspecifiedObject data - the object's data or subset data.
object_info info - information about the object.
list<ProvenanceAction> provenance - the object's provenance.
username creator - the user that first saved the object to the
workspace.
timestamp created - the date the object was first saved to the
workspace.
list<obj_ref> - the references contained within the object.
obj_ref copied - the reference of the source object if this object is
a copy and the copy source exists and is accessible.
null otherwise.
boolean copy_source_inaccessible - true if the object was copied from
another object, but that object is no longer accessible to the
user. False otherwise.
mapping<id_type, list<extracted_id>> extracted_ids - any ids extracted
from the object.
string handle_error - if an error occurs while setting ACLs on
embedded handle IDs, it will be reported here.
string handle_stacktrace - the stacktrace for handle_error.
*/
typedef structure {
UnspecifiedObject data;
object_info info;
list<ProvenanceAction> provenance;
username creator;
timestamp created;
list<obj_ref> refs;
obj_ref copied;
boolean copy_source_inaccessible;
mapping<id_type, list<extracted_id>> extracted_ids;
string handle_error;
string handle_stacktrace;
} ObjectData;
:param name: (optional) the name for the object to be retrieved. if included, favored over ID
:param wsid: the workspace to retrieve the object from
:param objid: the id of the object to be retrieved
"""
if name is None:
result = self.ws_client.get_objects2(
{'objects': [{
'objid': objid,
'workspace': wsid
}]})['data'][0]
else:
result = self.ws_client.get_objects2(
{'objects': [{
'name': name,
'workspace': wsid
}]})[0]
return result['data'], result['info']
def get_info(self, wsid, objid=None, name=None):
if name is None:
return self.ws_client.get_object_info_new(
{'objects': [{
'objid': objid,
'workspace': wsid
}]})[0]
else:
return self.ws_client.get_object_info_new(
{'objects': [{
'name': name,
'workspace': wsid
}]})[0]
def save_object(self, data, type, wsid, objid=None, name=None):
"""
Saves an object in KBase
:param data: data representing the object to be saved
:param type: a string representing the KBase type of the object
:param wsid: destination workspace
:param objid: (optional) ID for location of object to be saved (use with care, overwriting/failures are at KBase's
discretion).
:param name: (optional) string name for the pbject to be saved
:return: a list of information about the object as it is stored in KBase
"""
sv = {u'data': data, u'type': type, u'name': name}
if objid is not None:
sv[u'objid'] = objid
if name is not None:
sv[u'name'] = name
info = self.ws_client.save_objects({
u'workspace': wsid,
u'objects': [sv]
})[0]
return info[0], info[7]
def list_objects(self, workspace_id, typestr=None):
"""
returns a list of all the objects within a workspace in tuples (obj_id, ws_id, object_name)
:rtype: list
:param typestr: (optional) if set, lists only objects of this type (filter over default case)
:param workspace_id: the workspace to list the objects from
:return: a list of tuples of objects
"""
objects = self.ws_client.list_objects({'workspaces': [workspace_id]})
result = list()
for obj in objects:
object_type = obj[2]
if typestr is None or typestr in object_type or types(
)[typestr] in object_type: # type filtering of our list
result.append((obj[0], obj[6], obj[1], obj[2]))
return result
def clear_workspace(self, workspace_id):
"""
clear all objects in a workspace (except for a Narrative object if applicable)
:param workspace_id: workspace to clear
:return: None
"""
object_ids = [{
'objid': info[0],
'wsid': workspace_id
} for info in self.ws_client.list_objects({'ids': [workspace_id]})
if not info[2].startswith('KBaseNarrative')]
if len(object_ids) > 0:
self.ws_client.delete_objects(object_ids)
def delete_objects(self, object_tuples):
"""
delete objects
:param object_tuples: list of tuples representing objects to delete of the form (obj_id, ws_id)
:return: None
"""
object_ids = [{
'objid': info[0],
'wsid': info[1]
} for info in object_tuples]
if len(object_ids) > 0:
self.ws_client.delete_objects(object_ids)
def copy_object(self, from_tuple, to_tuple):
"""
Copies an object in the service to another location in the service
:param from_tuple: (objid, wsid) of the object to be copied
:param to_tuple: (name, wsid) of the destination. workspace may differ. NOTE NAME IS A STRING
:return: a tuple with information on the new objectmodel
"""
info = self.ws_client.copy_object({
'from': {
'workspace': from_tuple[1],
'objid': from_tuple[0]
},
'to': {
'workspace': to_tuple[1],
'name': to_tuple[0]
}
})
return info[0], info[7]
def gapfill_model(self, model, media, workspace=None):
"""
:param model: FBAModel to gapfill
:param media: Media to gapfill the model to
:param workspace: destination workspace for new model and gapfill object
:param name: (optional) name for new model. KBase will overwrite original if left unspecified.
:return: the information for a new gap-filled model
"""
if workspace is None:
workspace = model.workspace_id
params = {
u'fbamodel_id': str(model.object_id),
u'fbamodel_workspace': str(model.workspace_id),
u'fbamodel_output_id': str(model.name),
u'workspace': workspace,
u'media_id': media.object_id,
u'media_workspace': media.workspace_id,
u'comprehensive_gapfill': False
}
self.fba_client.gapfill_metabolic_model(params)
return model.object_id, model.workspace_id
def _gapfill_solution(self, fba):
"""
If this FBA was created as a gapfilling solution, then this returns a list of reactions to be added/adjusted
:return: list(tuple) (rxn_id, direction, etc.)
"""
# For now, naively assume first = best = only gap-filling solution
solutions = fba['gapfillingSolutions']
if len(solutions) < 1:
raise ValueError("This is not a gapfilling solution")
gsol = solutions[0]['gapfillingSolutionReactions']
result = []
for r in gsol:
reaction_id = r['reaction_ref'].split('/')[-1] + '_' + \
r['compartment_ref'].split('/')[-1] + str(r['compartmentIndex'])
direction = r['direction']
result.append((reaction_id, direction))
return result
def fba_formulation(self, media):
return {
u'media': str(media.object_id),
u'media_workspace': str(media.workspace_id)
}
def runfba(self, model, media, workspace=None):
"""
runs Flux Balance Analysis on an FBAModel in the fba modeling service
:param model: FBAModel to run flux balance analysis on
:param media: Media to run FBA with
:param workspace: (optional) workspace for the FBA object to be left in, default is model workspace
:return: tuple identity of the FBA stored in the service
"""
if workspace is None:
workspace = model.workspace_id
fba_params = {
u'workspace': workspace,
u'fbamodel_id': model.object_id,
u'fbamodel_workspace': model.workspace_id,
u'media_workspace': str(media.workspace_id),
u'media_id': str(media.object_id),
u'fba_output_id': model.name + '_fba'
}
info = self.fba_client.run_flux_balance_analysis(fba_params)
obj_id = info['new_fba_ref'].split('/')[1]
return obj_id, workspace
def runfva(self, model, media, workspace=None):
"""
runs Flux Balance Analysis on an FBAModel in the fba modeling service
:param model: FBAModel to run flux balance analysis on
:param media: Media to run FBA with
:param workspace: (optional) workspace for the FBA object to be left in, default is model workspace
:return: tuple identity of the FBA stored in the service
"""
if workspace is None:
workspace = model.workspace_id
fba_params = {
u'workspace': workspace,
u'model': model.object_id,
u'model_workspace': model.workspace_id,
u'formulation': self.fba_formulation(media),
u'fva': True
}
info = self.fba_client.runfba(fba_params)
obj_id = info['new_fba_ref'].split('/')[1]
return obj_id, workspace
def translate_model(self, src_model, protcomp, workspace=None):
"""
Uses the service to translate an FBAModel to a close genome relative
:param protcomp: ProteomeComparison with source and target Genome
:param src_model: FBAModel of source
return: tuple identity of the translated model stored in the service
"""
if workspace is None:
workspace = src_model.workspace_id
trans_params = {
u'keep_nogene_rxn': 1,
u'proteincomparison_id': protcomp.object_id,
u'proteincomparison_workspace': protcomp.workspace_id,
u'fbamodel_id': src_model.object_id,
u'fbamodel_output_id': 'translated_' + src_model.name,
u'fbamodel_workspace': src_model.workspace_id,
u'workspace': workspace
}
info = self.fba_client.propagate_model_to_new_genome(trans_params)
obj_id = info['new_fbamodel_ref'].split('/')[1]
return obj_id, workspace
def reconstruct_genome(self, genome, workspace=None):
"""
Reconstructs a genome and returns the identity of a stored draft recon model (FBAModel)
:param workspace: (optional) destination workspace. Default is genome.workspace_id
:param genome: Genome to draft a reconstruction for
:return: tuple identity of the draft model stored in the service (FBAModel)
"""
if workspace is None:
workspace = genome.workspace_id
recon_params = {
u'genome_id': genome.object_id,
u'genome_workspace': genome.workspace_id,
u'fbamodel_output_id': 'recon_' + genome.name,
u'gapfill_model': False, # TODO parameterize as option
u'workspace': workspace
}
info = self.fba_client.build_metabolic_model(recon_params)
# references returned here are sometimes inconsistent from other fba_tools APIs. Fetch obj info from ws service
obj_name = info['new_fbamodel_ref'].split('/')[1]
try:
return int(obj_name), workspace
except ValueError:
ws_object_info = self.ws_client.get_object_info_new(
{'objects': [{
'name': obj_name,
'workspace': workspace
}]})[0]
return ws_object_info[0], workspace
def remove_reactions_in_place(self, model, reactions_to_remove):
"""
Removes reactions from an FBAModel IN PLACE (changes object as it is stored)
Recommended to make a copy first
:param model: FBAModel to remove reactions form
:param reactions_to_remove: reactions to remove (removal_id's)
:return:
"""
model_data, model_info = self.get_object(model.object_id,
model.workspace_id)
rxns_to_remove = set(reactions_to_remove)
prior_ids = set([r['id'] for r in model_data['modelreactions']])
model_data['modelreactions'] = [
r for r in model_data['modelreactions']
if r['id'] not in rxns_to_remove
]
current_ids = set([r['id'] for r in model_data['modelreactions']])
removed = set(
[rxn_id for rxn_id in prior_ids if rxn_id not in current_ids])
if len(reactions_to_remove) != len(removed):
print "WARNING: expected to remove", len(
reactions_to_remove), "reactions but only removed", removed
print "Failed to remove", set(reactions_to_remove) - removed
print "Full arg reactions_to_remove:", ', '.join(
reactions_to_remove)
return self.save_object(model_data,
model_info[2],
model.workspace_id,
name=model.name)
def remove_reaction(self, model, reaction, output_id=None, in_place=False):
"""
:param model: FBAModel to remove the reaction from
:param reaction: removal_id (str) of the reaction to remove
:param output_id: (optional) (str) of the new name for the output model
:param in_place: (optional) set to true if you want to remove the reaction from the model in place instead of making
a new model. Will disregard output_id argument if set to true
:return: info tuple for the new FBAModel in the stored environment
"""
if in_place:
self.remove_reactions_in_place(model, [reaction])
if output_id is None:
i = 0
output_id = model.name + '-' + str(i)
names = set(
[info[3] for info in self.list_objects(model.workspace_id)])
while output_id in names:
i += 1
output_id = model.name + '-' + str(i)
model_data, model_info = self.get_object(model.object_id,
model.workspace_id)
for i, r in enumerate(model_data['modelreactions']):
if reaction == r['id']:
# remove in json and save
del model_data['modelreactions'][i]
return self.save_object(model_data,
model_info[2],
model.workspace_id,
name=output_id)
def add_reactions(self, model, new_reactions, workspace=None, name=None):
"""
adds reactions to an FBAModel, in place or with a copy (set name to a new name)
:param model: FBAModel to add reactions to
:param new_reactions: list of tuples of the form (rxn_id, rxn_comp, direction, gpr) (gpr is optional)
:param workspace: (optional) destination workspace, default is model.workspace_id
:param name: output name for the new model. use to make a new one or modify in place
:return: tuple identity of the model stored in the service (FBAModel)
"""
reactions_to_add = [{
'add_reaction_id':
r[0],
'reaction_compartment_id':
len(r) > 1 and [r[1]] or [],
'add_reaction_name':
r[0],
'add_reaction_direction':
len(r) > 2 and r[2] or '=',
'add_reaction_gpr':
len(r) > 3 and r[3] or '',
} for r in new_reactions]
add_rxn_args = {
'fbamodel_id': model.object_id,
'fbamodel_workspace': model.workspace_id,
'fbamodel_output_id': name or model.name,
'workspace': workspace or model.workspace_id,
'reactions_to_add': reactions_to_add
}
info = self.fba_client.edit_metabolic_model(add_rxn_args)
return self._parse_objid_from_ref(
info['new_fbamodel_ref']), model.workspace_id
def add_reactions_manually(self,
model,
reactions,
workspace=None,
name=None):
"""
Manually fix special reactions within the the object itself (use with caution)
:param name: what to name the model when it is saved
:param workspace: workspace to save the new FBAModel in
:param reactions: (list<ModelReaction>) list of reactions to add manually
:param model: FBAModel to add the reactions to
"""
model.get_object()
if workspace is None:
workspace = model.workspace_id
obj = model.data
cpds = dict([(c['id'], c) for c in obj['modelcompounds']])
for r in reactions:
obj['modelreactions'].append(r.data)
for cpd in r.data['modelReactionReagents']:
c = cpd['modelcompound_ref'].split('/')[-1]
if c not in cpds:
compound = {
'id':
c,
'name':
c,
'aliases': [u'mdlid:' + c.split('_')[0]],
'charge':
0,
'compound_ref':
'489/6/6/compounds/id/cpd00000',
'modelcompartment_ref':
'~/modelcompartments/id/' + c.split('_')[-1],
'formula':
''
}
obj['modelcompounds'].append(compound)
cpds = dict([(c['id'], c) for c in obj['modelcompounds']])
if name is not None:
return self.save_object(obj,
types()['FBAModel'],
workspace,
name=name)
return self.save_object(obj,
types()['FBAModel'],
workspace,
objid=model.object_id)
def adjust_directions_and_gprs(self, model, adjustments):
reactions_to_change = [{
'change_reaction_id': [r[0]],
'change_reaction_direction': str(r[1]),
'change_reaction_gpr': str(r[2])[1:-1],
} for r in adjustments]
change_rxn_args = {
'fbamodel_id': model.object_id,
'fbamodel_workspace': model.workspace_id,
'fbamodel_output_id': model.name,
'workspace': model.workspace_id,
'reactions_to_change': reactions_to_change
}
self.fba_client.edit_metabolic_model(change_rxn_args)
def adjust_directions(self, model, adjustments):
"""
adjusts directions for reactions in an FBAModel
:param model: FBAModel to adjust directions for
:param adjustments: list<tuple> (rxn_id, direction). if rxn_id is not already in the model, it may be added
:return: None
"""
adjust_args = {
'model': model.object_id,
'workspace': model.workspace_id,
'reaction': [r[0] for r in adjustments],
'direction': [str(r[1]) for r in adjustments]
}
self.fba_client.adjust_model_reaction(adjust_args)
def _integrate_gapfill(self, model, solution_fba, workspace=None):
changes = self._gapfill_solution(solution_fba)
reactions = dict([(r.rxn_id(), r) for r in model.get_reactions()])
dirs = []
additions = []
for r in changes:
if r[0] in reactions:
dirs.append((reactions[r[0]].get_removal_id(), r[1]))
else:
temp = r[0].split('_')
rxn_id = temp[0]
rxn_comp = temp[1]
additions.append((rxn_id, rxn_comp, r[1]))
self.adjust_directions(model, dirs)
info = self.add_reactions(model, additions, workspace=workspace)
return info
def model_info(self, model):
comp = self.fba_client.compare_models({
'models': [model.object_id],
'workspaces': [model.workspace_id]
})
return (comp['model_comparisons'],
dict([(r['reaction'], r)
for r in comp['reaction_comparisons']]))
def init_workspace(self, ws=None, name=None):
ws_id = ws
ws_name = name
if ws_name is None:
ws_name = 'MMws'
if ws is None:
ws_conflict = True
while ws_conflict:
create_ws_params = {
'workspace':
ws_name,
'globalread':
'r',
'description':
"A workspace for storing the FBA's and meta data of the algorithm"
}
# Try to create a workspace, catch an error if the name is already in use
try:
new_ws = self.ws_client.create_workspace(create_ws_params)
# new_ws is type workspace_info, a tuple where 0, 1 are id, name
ws_id = new_ws[0]
ws_name = new_ws[1]
ws_conflict = False
except ServerError:
ws_name += str(random.randint(1, 9))
return ws_id, ws_name
def _parse_objid_from_ref(self, ref):
return ref.split('/')[1]
class TreeUtils:
'''
Module Name:
TreeUtils
Module Description:
'''
######## WARNING FOR GEVENT USERS ####### noqa
# Since asynchronous IO can lead to methods - even the same method -
# interrupting each other, you must be *very* careful when using global
# state. A method could easily clobber the state set by another while
# the latter method is running.
######################################### noqa
VERSION = "0.0.1"
GIT_URL = ""
GIT_COMMIT_HASH = "acb216cd302c161d5b4dfb272bd4bbae44cdac28"
#BEGIN_CLASS_HEADER
#END_CLASS_HEADER
# config contains contents of config file in a hash or None if it couldn't
# be found
def __init__(self, config):
#BEGIN_CONSTRUCTOR
self.utils = Utils(config)
self.scratch = config['scratch']
self.dfu = DataFileUtil(os.environ['SDK_CALLBACK_URL'])
self.ws = Workspace(config['workspace-url'])
logging.basicConfig(level=logging.INFO)
#END_CONSTRUCTOR
pass
def get_trees(self, ctx, params):
"""
:param params: instance of type "GetTreesParams" (tree_refs -
(required) list of WS references included_fields - (optional)
subset of tree fields to include) -> structure: parameter
"tree_refs" of list of String, parameter "included_fields" of list
of String
:returns: instance of list of type "TreeData" -> structure: parameter
"data" of type "Tree" (Data type for phylogenetic trees. @optional
name description type tree_attributes @optional
default_node_labels ws_refs kb_refs leaf_list) -> structure:
parameter "name" of String, parameter "description" of String,
parameter "type" of String, parameter "tree" of type "newick_tree"
(Trees are represented in KBase by default in newick format
(http://en.wikipedia.org/wiki/Newick_format) and are returned to
you in this format by default.) -> type "tree" (A string
representation of a phylogenetic tree. The format/syntax of the
string is specified by using one of the available typedefs
declaring a particular format, such as 'newick_tree',
'phylo_xml_tree' or 'json_tree'. When a format is not explictily
specified, it is possible to return trees in different formats
depending on addtional parameters. Regardless of format, all leaf
nodes in trees built from MSAs are indexed to a specific MSA row.
You can use the appropriate functionality of the API to replace
these IDs with other KBase Ids instead. Internal nodes may or may
not be named. Nodes, depending on the format, may also be
annotated with structured data such as bootstrap values and
distances.), parameter "tree_attributes" of mapping from String to
String, parameter "default_node_labels" of mapping from type
"node_id" to type "label", parameter "ws_refs" of mapping from
type "node_id" to mapping from type "ref_type" (An enumeration of
reference types for a node. Either the one letter abreviation or
full name can be given. For large trees, it is strongly advised
you use the one letter abreviations. Supported types are: g |
genome => genome typed object or CDS data p | protein => protein
sequence object or CDS data, often given as the MD5 of the
sequence n | dna => dna sequence object or CDS data, often
given as the MD5 of the sequence f | feature => feature object or
CDS data) to list of type "ws_obj_id" (@id ws), parameter
"kb_refs" of mapping from type "node_id" to mapping from type
"ref_type" (An enumeration of reference types for a node. Either
the one letter abreviation or full name can be given. For large
trees, it is strongly advised you use the one letter abreviations.
Supported types are: g | genome => genome typed object or CDS
data p | protein => protein sequence object or CDS data, often
given as the MD5 of the sequence n | dna => dna sequence
object or CDS data, often given as the MD5 of the sequence f |
feature => feature object or CDS data) to list of type "kbase_id"
(A KBase ID is a string starting with the characters "kb|". KBase
IDs are typed. The types are designated using a short string. For
instance," g" denotes a genome, "tree" denotes a Tree, and "aln"
denotes a sequence alignment. KBase IDs may be hierarchical. For
example, if a KBase genome identifier is "kb|g.1234", a protein
encoding gene within that genome may be represented as
"kb|g.1234.peg.771". @id kb), parameter "leaf_list" of list of
type "node_id", parameter "info" of type "object_info"
(Information about an object, including user provided metadata.
obj_id objid - the numerical id of the object. obj_name name - the
name of the object. type_string type - the type of the object.
timestamp save_date - the save date of the object. obj_ver ver -
the version of the object. username saved_by - the user that saved
or copied the object. ws_id wsid - the workspace containing the
object. ws_name workspace - the workspace containing the object.
string chsum - the md5 checksum of the object. int size - the size
of the object in bytes. usermeta meta - arbitrary user-supplied
metadata about the object.) -> tuple of size 11: parameter "objid"
of type "obj_id" (The unique, permanent numerical ID of an
object.), parameter "name" of type "obj_name" (A string used as a
name for an object. Any string consisting of alphanumeric
characters and the characters |._- that is not an integer is
acceptable.), parameter "type" of type "type_string" (A type
string. Specifies the type and its version in a single string in
the format [module].[typename]-[major].[minor]: module - a string.
The module name of the typespec containing the type. typename - a
string. The name of the type as assigned by the typedef statement.
major - an integer. The major version of the type. A change in the
major version implies the type has changed in a non-backwards
compatible way. minor - an integer. The minor version of the type.
A change in the minor version implies that the type has changed in
a way that is backwards compatible with previous type definitions.
In many cases, the major and minor versions are optional, and if
not provided the most recent version will be used. Example:
MyModule.MyType-3.1), parameter "save_date" of type "timestamp" (A
time in the format YYYY-MM-DDThh:mm:ssZ, where Z is either the
character Z (representing the UTC timezone) or the difference in
time to UTC in the format +/-HHMM, eg: 2012-12-17T23:24:06-0500
(EST time) 2013-04-03T08:56:32+0000 (UTC time)
2013-04-03T08:56:32Z (UTC time)), parameter "version" of Long,
parameter "saved_by" of type "username" (Login name of a KBase
user account.), parameter "wsid" of type "ws_id" (The unique,
permanent numerical ID of a workspace.), parameter "workspace" of
type "ws_name" (A string used as a name for a workspace. Any
string consisting of alphanumeric characters and "_", ".", or "-"
that is not an integer is acceptable. The name may optionally be
prefixed with the workspace owner's user name and a colon, e.g.
kbasetest:my_workspace.), parameter "chsum" of String, parameter
"size" of Long, parameter "meta" of type "usermeta" (User provided
metadata about an object. Arbitrary key-value pairs provided by
the user.) -> mapping from String to String
"""
# ctx is the context object
# return variables are: result
#BEGIN get_trees
logging.info("Starting 'get_trees' with params:{}".format(params))
self.utils.validate_params(params, ("tree_refs",), ("included_fields",))
ws_objs = [{'ref': r, 'included': params.get('included_fields', None)}
for r in params['tree_refs']]
result = self.ws.get_objects2({'objects': ws_objs})['data']
#END get_trees
# At some point might do deeper type checking...
if not isinstance(result, list):
raise ValueError('Method get_trees return value ' +
'result is not type list as required.')
# return the results
return [result]
def save_trees(self, ctx, params):
"""
:param params: instance of type "SaveTreesParams" -> structure:
parameter "ws_id" of type "ws_id" (The unique, permanent numerical
ID of a workspace.), parameter "trees" of list of type
"ObjectSaveData" (An object and associated data required for
saving. Required arguments: type_string type - the type of the
object. Omit the version information to use the latest version.
UnspecifiedObject data - the object data. Optional arguments: One
of an object name or id. If no name or id is provided the name
will be set to 'auto' with the object id appended as a string,
possibly with -\d+ appended if that object id already exists as a
name. obj_name name - the name of the object. obj_id objid - the
id of the object to save over. usermeta meta - arbitrary
user-supplied metadata for the object, not to exceed 16kb; if the
object type specifies automatic metadata extraction with the 'meta
ws' annotation, and your metadata name conflicts, then your
metadata will be silently overwritten. list<ProvenanceAction>
provenance - provenance data for the object. boolean hidden - true
if this object should not be listed when listing workspace
objects.) -> structure: parameter "type" of type "type_string" (A
type string. Specifies the type and its version in a single string
in the format [module].[typename]-[major].[minor]: module - a
string. The module name of the typespec containing the type.
typename - a string. The name of the type as assigned by the
typedef statement. major - an integer. The major version of the
type. A change in the major version implies the type has changed
in a non-backwards compatible way. minor - an integer. The minor
version of the type. A change in the minor version implies that
the type has changed in a way that is backwards compatible with
previous type definitions. In many cases, the major and minor
versions are optional, and if not provided the most recent version
will be used. Example: MyModule.MyType-3.1), parameter "data" of
unspecified object, parameter "name" of type "obj_name" (A string
used as a name for an object. Any string consisting of
alphanumeric characters and the characters |._- that is not an
integer is acceptable.), parameter "objid" of type "obj_id" (The
unique, permanent numerical ID of an object.), parameter "meta" of
type "usermeta" (User provided metadata about an object. Arbitrary
key-value pairs provided by the user.) -> mapping from String to
String, parameter "provenance" of list of type "ProvenanceAction"
(A provenance action. A provenance action (PA) is an action taken
while transforming one data object to another. There may be
several PAs taken in series. A PA is typically running a script,
running an api command, etc. All of the following fields are
optional, but more information provided equates to better data
provenance. resolved_ws_objects should never be set by the user;
it is set by the workspace service when returning data. On input,
only one of the time or epoch may be supplied. Both are supplied
on output. The maximum size of the entire provenance object,
including all actions, is 1MB. timestamp time - the time the
action was started epoch epoch - the time the action was started.
string caller - the name or id of the invoker of this provenance
action. In most cases, this will be the same for all PAs. string
service - the name of the service that performed this action.
string service_ver - the version of the service that performed
this action. string method - the method of the service that
performed this action. list<UnspecifiedObject> method_params - the
parameters of the method that performed this action. If an object
in the parameters is a workspace object, also put the object
reference in the input_ws_object list. string script - the name of
the script that performed this action. string script_ver - the
version of the script that performed this action. string
script_command_line - the command line provided to the script that
performed this action. If workspace objects were provided in the
command line, also put the object reference in the input_ws_object
list. list<obj_ref> input_ws_objects - the workspace objects that
were used as input to this action; typically these will also be
present as parts of the method_params or the script_command_line
arguments. list<obj_ref> resolved_ws_objects - the workspace
objects ids from input_ws_objects resolved to permanent workspace
object references by the workspace service. list<string>
intermediate_incoming - if the previous action produced output
that 1) was not stored in a referrable way, and 2) is used as
input for this action, provide it with an arbitrary and unique ID
here, in the order of the input arguments to this action. These
IDs can be used in the method_params argument. list<string>
intermediate_outgoing - if this action produced output that 1) was
not stored in a referrable way, and 2) is used as input for the
next action, provide it with an arbitrary and unique ID here, in
the order of the output values from this action. These IDs can be
used in the intermediate_incoming argument in the next action.
list<ExternalDataUnit> external_data - data external to the
workspace that was either imported to the workspace or used to
create a workspace object. list<SubAction> subactions - the
subactions taken as a part of this action. mapping<string, string>
custom - user definable custom provenance fields and their values.
string description - a free text description of this action.) ->
structure: parameter "time" of type "timestamp" (A time in the
format YYYY-MM-DDThh:mm:ssZ, where Z is either the character Z
(representing the UTC timezone) or the difference in time to UTC
in the format +/-HHMM, eg: 2012-12-17T23:24:06-0500 (EST time)
2013-04-03T08:56:32+0000 (UTC time) 2013-04-03T08:56:32Z (UTC
time)), parameter "epoch" of type "epoch" (A Unix epoch (the time
since 00:00:00 1/1/1970 UTC) in milliseconds.), parameter "caller"
of String, parameter "service" of String, parameter "service_ver"
of String, parameter "method" of String, parameter "method_params"
of list of unspecified object, parameter "script" of String,
parameter "script_ver" of String, parameter "script_command_line"
of String, parameter "input_ws_objects" of list of type "obj_ref"
(A string that uniquely identifies an object in the workspace
service. There are two ways to uniquely identify an object in one
string: "[ws_name or id]/[obj_name or id]/[obj_ver]" - for
example, "MyFirstWorkspace/MyFirstObject/3" would identify the
third version of an object called MyFirstObject in the workspace
called MyFirstWorkspace. 42/Panic/1 would identify the first
version of the object name Panic in workspace with id 42.
Towel/1/6 would identify the 6th version of the object with id 1
in the Towel workspace. "kb|ws.[ws_id].obj.[obj_id].ver.[obj_ver]"
- for example, "kb|ws.23.obj.567.ver.2" would identify the second
version of an object with id 567 in a workspace with id 23. In all
cases, if the version number is omitted, the latest version of the
object is assumed.), parameter "resolved_ws_objects" of list of
type "obj_ref" (A string that uniquely identifies an object in the
workspace service. There are two ways to uniquely identify an
object in one string: "[ws_name or id]/[obj_name or id]/[obj_ver]"
- for example, "MyFirstWorkspace/MyFirstObject/3" would identify
the third version of an object called MyFirstObject in the
workspace called MyFirstWorkspace. 42/Panic/1 would identify the
first version of the object name Panic in workspace with id 42.
Towel/1/6 would identify the 6th version of the object with id 1
in the Towel workspace. "kb|ws.[ws_id].obj.[obj_id].ver.[obj_ver]"
- for example, "kb|ws.23.obj.567.ver.2" would identify the second
version of an object with id 567 in a workspace with id 23. In all
cases, if the version number is omitted, the latest version of the
object is assumed.), parameter "intermediate_incoming" of list of
String, parameter "intermediate_outgoing" of list of String,
parameter "external_data" of list of type "ExternalDataUnit" (An
external data unit. A piece of data from a source outside the
Workspace. On input, only one of the resource_release_date or
resource_release_epoch may be supplied. Both are supplied on
output. string resource_name - the name of the resource, for
example JGI. string resource_url - the url of the resource, for
example http://genome.jgi.doe.gov string resource_version -
version of the resource timestamp resource_release_date - the
release date of the resource epoch resource_release_epoch - the
release date of the resource string data_url - the url of the
data, for example
http://genome.jgi.doe.gov/pages/dynamicOrganismDownload.jsf?
organism=BlaspURHD0036 string data_id - the id of the data, for
example 7625.2.79179.AGTTCC.adnq.fastq.gz string description - a
free text description of the data.) -> structure: parameter
"resource_name" of String, parameter "resource_url" of String,
parameter "resource_version" of String, parameter
"resource_release_date" of type "timestamp" (A time in the format
YYYY-MM-DDThh:mm:ssZ, where Z is either the character Z
(representing the UTC timezone) or the difference in time to UTC
in the format +/-HHMM, eg: 2012-12-17T23:24:06-0500 (EST time)
2013-04-03T08:56:32+0000 (UTC time) 2013-04-03T08:56:32Z (UTC
time)), parameter "resource_release_epoch" of type "epoch" (A Unix
epoch (the time since 00:00:00 1/1/1970 UTC) in milliseconds.),
parameter "data_url" of String, parameter "data_id" of String,
parameter "description" of String, parameter "subactions" of list
of type "SubAction" (Information about a subaction that is invoked
by a provenance action. A provenance action (PA) may invoke
subactions (SA), e.g. calling a separate piece of code, a service,
or a script. In most cases these calls are the same from PA to PA
and so do not need to be listed in the provenance since providing
information about the PA alone provides reproducibility. In some
cases, however, SAs may change over time, such that invoking the
same PA with the same parameters may produce different results.
For example, if a PA calls a remote server, that server may be
updated between a PA invoked on day T and another PA invoked on
day T+1. The SubAction structure allows for specifying information
about SAs that may dynamically change from PA invocation to PA
invocation. string name - the name of the SA. string ver - the
version of SA. string code_url - a url pointing to the SA's
codebase. string commit - a version control commit ID for the SA.
string endpoint_url - a url pointing to the access point for the
SA - a server url, for instance.) -> structure: parameter "name"
of String, parameter "ver" of String, parameter "code_url" of
String, parameter "commit" of String, parameter "endpoint_url" of
String, parameter "custom" of mapping from String to String,
parameter "description" of String, parameter "hidden" of type
"boolean" (A boolean. 0 = false, other = true.)
:returns: instance of list of type "object_info" (Information about
an object, including user provided metadata. obj_id objid - the
numerical id of the object. obj_name name - the name of the
object. type_string type - the type of the object. timestamp
save_date - the save date of the object. obj_ver ver - the version
of the object. username saved_by - the user that saved or copied
the object. ws_id wsid - the workspace containing the object.
ws_name workspace - the workspace containing the object. string
chsum - the md5 checksum of the object. int size - the size of the
object in bytes. usermeta meta - arbitrary user-supplied metadata
about the object.) -> tuple of size 11: parameter "objid" of type
"obj_id" (The unique, permanent numerical ID of an object.),
parameter "name" of type "obj_name" (A string used as a name for
an object. Any string consisting of alphanumeric characters and
the characters |._- that is not an integer is acceptable.),
parameter "type" of type "type_string" (A type string. Specifies
the type and its version in a single string in the format
[module].[typename]-[major].[minor]: module - a string. The module
name of the typespec containing the type. typename - a string. The
name of the type as assigned by the typedef statement. major - an
integer. The major version of the type. A change in the major
version implies the type has changed in a non-backwards compatible
way. minor - an integer. The minor version of the type. A change
in the minor version implies that the type has changed in a way
that is backwards compatible with previous type definitions. In
many cases, the major and minor versions are optional, and if not
provided the most recent version will be used. Example:
MyModule.MyType-3.1), parameter "save_date" of type "timestamp" (A
time in the format YYYY-MM-DDThh:mm:ssZ, where Z is either the
character Z (representing the UTC timezone) or the difference in
time to UTC in the format +/-HHMM, eg: 2012-12-17T23:24:06-0500
(EST time) 2013-04-03T08:56:32+0000 (UTC time)
2013-04-03T08:56:32Z (UTC time)), parameter "version" of Long,
parameter "saved_by" of type "username" (Login name of a KBase
user account.), parameter "wsid" of type "ws_id" (The unique,
permanent numerical ID of a workspace.), parameter "workspace" of
type "ws_name" (A string used as a name for a workspace. Any
string consisting of alphanumeric characters and "_", ".", or "-"
that is not an integer is acceptable. The name may optionally be
prefixed with the workspace owner's user name and a colon, e.g.
kbasetest:my_workspace.), parameter "chsum" of String, parameter
"size" of Long, parameter "meta" of type "usermeta" (User provided
metadata about an object. Arbitrary key-value pairs provided by
the user.) -> mapping from String to String
"""
# ctx is the context object
# return variables are: result
#BEGIN save_trees
logging.info("Starting 'save_trees'")
self.utils.validate_params(params, ("ws_id", "trees"), ('type',))
trees = []
for i, t in enumerate(params['trees']):
self.utils.validate_params(t, ("data",), ("name", "hidden", "meta", "type"))
if 'type' in t and t['type'] != 'KBaseTrees.Tree':
raise ValueError("This method only saves KBaseTrees.Tree objects")
if "tree" not in t['data']:
raise ValueError("Object {} missing 'tree' attribute containing newick tree"
.format(i))
if not Utils.validate_newick(t['data']['tree']):
raise ValueError("Object {} has an invalid newick tree: {}"
.format(i, t['data']['tree']))
t['type'] = 'KBaseTrees.Tree'
trees.append(t)
result = self.dfu.save_objects({"id": params["ws_id"], "objects": trees})
#END save_trees
# At some point might do deeper type checking...
if not isinstance(result, list):
raise ValueError('Method save_trees return value ' +
'result is not type list as required.')
# return the results
return [result]
def tree_to_newick_file(self, ctx, params):
"""
:param params: instance of type "TreeToNewickFileParams" ->
structure: parameter "input_ref" of type "Tree_id" (@id kb
KBaseTrees.Tree), parameter "destination_dir" of String
:returns: instance of type "TreeToNewickFileOutput" -> structure:
parameter "file_path" of String
"""
# ctx is the context object
# return variables are: result
#BEGIN tree_to_newick_file
logging.info("Starting 'tree_to_newick' with params: {}".format(params))
self.utils.validate_params(params, ("destination_dir", "input_ref"))
_, result = self.utils.to_newick(params)
#END tree_to_newick_file
# At some point might do deeper type checking...
if not isinstance(result, dict):
raise ValueError('Method tree_to_newick_file return value ' +
'result is not type dict as required.')
# return the results
return [result]
def export_tree_newick(self, ctx, params):
"""
:param params: instance of type "ExportTreeParams" -> structure:
parameter "input_ref" of type "Tree_id" (@id kb KBaseTrees.Tree)
:returns: instance of type "ExportTreeOutput" -> structure: parameter
"shock_id" of String
"""
# ctx is the context object
# return variables are: result
#BEGIN export_tree_newick
logging.info("Starting 'export_tree_newick' with params:{}".format(params))
self.utils.validate_params(params, ("input_ref",))
params['destination_dir'] = self.scratch
cs_id, files = self.utils.to_newick(params)
result = self.utils.export(files['file_path'], cs_id, params['input_ref'])
#END export_tree_newick
# At some point might do deeper type checking...
if not isinstance(result, dict):
raise ValueError('Method export_tree_newick return value ' +
'result is not type dict as required.')
# return the results
return [result]
def status(self, ctx):
#BEGIN_STATUS
returnVal = {'state': "OK",
'message': "",
'version': self.VERSION,
'git_url': self.GIT_URL,
'git_commit_hash': self.GIT_COMMIT_HASH}
#END_STATUS
return [returnVal]
class DiffExprMatrixUtils:
"""
Constains a set of functions for expression levels calculations.
"""
PARAM_IN_WS_NAME = 'workspace_name'
PARAM_IN_OBJ_NAME = 'output_obj_name'
PARAM_IN_DIFFEXPMATSET_REF = 'diffExprMatrixSet_ref'
def __init__(self, config, logger=None):
self.config = config
self.logger = logger
self.scratch = os.path.join(config['scratch'], 'DEM_' + str(uuid.uuid4()))
self.ws_url = config['workspace-url']
self._mkdir_p(self.scratch)
pass
def _mkdir_p(self, path):
"""
_mkdir_p: make directory for given path
"""
if not path:
return
try:
os.makedirs(path)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
def process_params(self, params):
"""
validates params passed to gen expression matrix method
"""
for p in [self.PARAM_IN_DIFFEXPMATSET_REF]:
if p not in params:
raise ValueError('"{}" parameter is required, but missing'.format(p))
def get_expressionset_data(self, expressionset_ref):
expr_set_obj = self.ws_client.get_objects2(
{'objects': [{'ref': expressionset_ref}]})['data'][0]
expr_set_obj_type = expr_set_obj.get('info')[2]
expr_set_data = dict()
expr_set_data['ws_name'] = expr_set_obj.get('info')[7]
expr_set_data['obj_name'] = expr_set_obj.get('info')[1]
if re.match('KBaseRNASeq.RNASeqExpressionSet-\d.\d', expr_set_obj_type):
expr_set_data['genome_ref'] = expr_set_obj['data']['genome_id']
expr_obj_refs = list()
for expr_obj in expr_set_obj['data']['mapped_expression_ids']:
expr_obj_refs.append(expr_obj.values()[0])
expr_set_data['expr_obj_refs'] = expr_obj_refs
elif re.match('KBaseSets.ExpressionSet-\d.\d', expr_set_obj_type):
items = expr_set_obj.get('data').get('items')
expr_obj_refs = list()
for item in items:
expr_obj_refs.append(item['ref'])
expr_obj = self.ws_client.get_objects2(
{'objects': [{'ref': expr_obj_refs[0]}]})['data'][0]
expr_set_data['genome_ref'] = expr_obj['data']['genome_id']
expr_set_data['expr_obj_refs'] = expr_obj_refs
else:
raise TypeError(self.PARAM_IN_EXPSET_REF + ' should be of type ' +
'KBaseRNASeq.RNASeqExpressionSet ' +
'or KBaseSets.ExpressionSet')
return expr_set_data
def get_diffexpr_matrixset(self, params, token):
self.ws_client = Workspace(self.ws_url, token=token)
col_names = {'gene_id': 'gene',
'log2_fold_change': 'log2fc_f',
'p_value': 'p_value_f',
'q_value': 'q_value'}
json_fields = ['log2fc_f', 'p_value_f', 'q_value']
self.process_params(params)
diffexprmatset_list = list()
diffexprmatset_ref = params.get(self.PARAM_IN_DIFFEXPMATSET_REF)
diffexprmatset_obj = self.ws_client.get_objects2(
{'objects': [{'ref': diffexprmatset_ref}]})['data'][0]
items = diffexprmatset_obj.get('data').get('items')
diffexprmat_refs = list()
for item in items:
diffexprmat_refs.append(item['ref'])
self.logger.info('DiffExprMatrix ref: ' + item['ref'])
for diffexprmat_ref in diffexprmat_refs:
diffexprmat_dict = dict()
diffexprmat_obj = self.ws_client.get_objects2(
{'objects': [{'ref': diffexprmat_ref}]})['data'][0]
diffexprmat = diffexprmat_obj.get('data')
diffexprmat_dict['condition_1'] = diffexprmat.get('condition_mapping').keys()[0]
diffexprmat_dict['condition_2'] = diffexprmat.get('condition_mapping').values()[0]
voldata = list()
data = diffexprmat.get('data')
for row_index, row_id in enumerate(data.get('row_ids')):
row_data = dict()
row_data['gene'] = row_id
values = data.get('values')[row_index]
for col_index in range(len(values)):
row_data[json_fields[col_index]] = values[col_index]
voldata.append(row_data)
diffexprmat_dict['voldata'] = voldata
diffexprmatset_list.append(diffexprmat_dict)
return diffexprmatset_list
class BallgownUtil:
def __init__(self, config):
self.ws_url = config["workspace-url"]
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.shock_url = config['shock-url']
self.dfu = DataFileUtil(self.callback_url)
self.rau = ReadsAlignmentUtils(self.callback_url)
self.fv = KBaseFeatureValues(self.callback_url)
self.deu = DifferentialExpressionUtils(self.callback_url, service_ver='dev')
self.ws = Workspace(self.ws_url, token=self.token)
self.scratch = config['scratch']
self.config = config
def _xor(self, a, b):
return bool(a) != bool(b)
def _validate_run_ballgown_app_params(self, params):
"""
_validate_run_ballgown_app_params:
validates params passed to run_ballgown_app method
"""
log('start validating run_ballgown_app params')
# check for required parameters
for p in ['expressionset_ref', 'diff_expression_matrix_set_suffix',
'workspace_name']:
if p not in params:
raise ValueError('"{}" parameter is required, but missing'.format(p))
run_all_combinations = params.get('run_all_combinations')
condition_pair_subset = params.get('condition_pair_subset')
if not self._xor(run_all_combinations, condition_pair_subset):
error_msg = "Invalid input:\nselect 'Run All Paired Condition Combinations' "
error_msg += "or provide subset of condition pairs. Don't provide both, or neither."
raise ValueError(error_msg)
def _mkdir_p(self, path):
"""
_mkdir_p: make directory for given path
"""
if not path:
return
try:
os.makedirs(path)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
def _generate_html_report(self, result_directory, params, diff_expression_matrix_set_ref):
"""
_generate_html_report: generate html summary report
"""
log('start generating html report')
html_report = list()
output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(output_directory)
result_file_path = os.path.join(output_directory, 'report.html')
for file in glob.glob(os.path.join(result_directory, '*.tsv')):
shutil.copy(file, output_directory)
# volcano_plot exists only if there are two condition groups
for file in glob.glob(os.path.join(result_directory, '*.png')):
shutil.copy(file, output_directory)
diff_expr_set = self.ws.get_objects2({'objects':
[{'ref':
diff_expression_matrix_set_ref[
'diffExprMatrixSet_ref']}]})['data'][0]
diff_expr_set_data = diff_expr_set['data']
diff_expr_set_info = diff_expr_set['info']
diff_expr_set_name = diff_expr_set_info[1]
overview_content = ''
overview_content += '<br/><table><tr><th>Generated DifferentialExpressionMatrixSet'
overview_content += ' Object</th></tr>'
overview_content += '<tr><td>{} ({})'.format(diff_expr_set_name,
diff_expression_matrix_set_ref[
'diffExprMatrixSet_ref'])
overview_content += '</td></tr></table>'
overview_content += '<p><br/></p>'
overview_content += '<br/><table><tr><th>Generated DifferentialExpressionMatrix'
overview_content += ' Object</th><th></th><th></th><th></th></tr>'
overview_content += '<tr><th>Differential Expression Matrix Name</th>'
overview_content += '<th>Condition 1</th>'
overview_content += '<th>Condition 2</th>'
overview_content += '</tr>'
for item in diff_expr_set_data['items']:
item_diffexprmatrix_object = self.ws.get_objects2({'objects':
[{'ref': item['ref']}]})[
'data'][0]
item_diffexprmatrix_info = item_diffexprmatrix_object['info']
item_diffexprmatrix_data = item_diffexprmatrix_object['data']
diffexprmatrix_name = item_diffexprmatrix_info[1]
overview_content += '<tr><td>{} ({})</td>'.format(diffexprmatrix_name,
item['ref'])
overview_content += '<td>{}</td>'.format(item_diffexprmatrix_data.
get('condition_mapping').keys()[0])
overview_content += '<td>{}</td>'.format(item_diffexprmatrix_data.
get('condition_mapping').values()[0])
overview_content += '</tr>'
overview_content += '</table>'
# visualization
image_content = ''
for image in glob.glob(output_directory + "/*.png"):
image = image.replace(output_directory + '/', '')
caption = image.replace(output_directory + '/', '').replace('.png', '')
image_content += '<p style="text-align:center"><img align="center" src="{}" ' \
'width="600" height="400"></a><a target="_blank"><br>' \
'<p align="center">{}</p></p>'.format(
image, caption)
with open(result_file_path, 'w') as result_file:
with open(os.path.join(os.path.dirname(__file__), 'report_template.html'),
'r') as report_template_file:
report_template = report_template_file.read()
report_template = report_template.replace('<p>Overview_Content</p>',
overview_content)
report_template = report_template.replace('<p>Image Gallery</p>',
image_content)
result_file.write(report_template)
report_shock_id = self.dfu.file_to_shock({'file_path': output_directory,
'pack': 'zip'})['shock_id']
html_report.append({'shock_id': report_shock_id,
'name': os.path.basename(result_file_path),
'label': os.path.basename(result_file_path),
'description': 'HTML summary report for Ballgown App'})
return html_report
def _generate_output_file_list(self, result_directory):
"""
_generate_output_file_list: zip result files and generate file_links for report
"""
log('Start packing result files')
output_files = list()
output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(output_directory)
result_file = os.path.join(output_directory, 'ballgown_result.zip')
with zipfile.ZipFile(result_file, 'w',
zipfile.ZIP_DEFLATED,
allowZip64=True) as zip_file:
for root, dirs, files in os.walk(result_directory):
for file in files:
if not (file.endswith('.zip') or
file.endswith('.png') or
file.endswith('.DS_Store')):
zip_file.write(os.path.join(root, file), file)
output_files.append({'path': result_file,
'name': os.path.basename(result_file),
'label': os.path.basename(result_file),
'description': 'File(s) generated by Ballgown App'})
return output_files
def _generate_report(self, params, result_directory, diff_expression_matrix_set_ref):
"""
_generate_report: generate summary report
"""
log('creating report')
output_files = self._generate_output_file_list(result_directory)
output_html_files = self._generate_html_report(
result_directory, params, diff_expression_matrix_set_ref)
report_params = {
'message': '',
'workspace_name': params.get('workspace_name'),
'file_links': output_files,
'html_links': output_html_files,
'direct_html_link_index': 0,
'html_window_height': 333,
'report_object_name': 'kb_ballgown_report_' + str(uuid.uuid4())}
kbase_report_client = KBaseReport(self.callback_url)
output = kbase_report_client.create_extended_report(report_params)
report_output = {'report_name': output['name'], 'report_ref': output['ref']}
return report_output
def get_sample_dir_group_file(self, mapped_expression_ids, condition_labels):
ngroups = 0
group_name_indices = {}
group_counts = {}
for group in condition_labels:
if not group in group_name_indices:
group_name_indices[group] = ngroups
ngroups = ngroups + 1
if not group in group_counts:
group_counts[group] = 1
else:
group_counts[group] = group_counts[group] + 1
# checks for proper ballgown execution:
if ngroups < 2:
raise Exception("At least two condition groups are needed for this analysis. ")
for group in condition_labels:
if group_counts[group] < 2:
raise Exception(
"Condition group {0} has less than 2 members; ballgown will not run. "
"At least two condition groups are needed for this analysis. ".format(group))
group_file_dir = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(group_file_dir)
try:
condition_labels_uniqued = list(set(condition_labels))
sgf_name = os.path.join(group_file_dir, 'sample_dir_group_file_' +
condition_labels_uniqued[0] + '_' +
condition_labels_uniqued[1])
sgf = open(sgf_name, "w")
except Exception:
raise Exception(
"Can't open file {0} for writing {1}".format(
sgf_name, traceback.format_exc()))
index = 0 # condition label index
for ii in mapped_expression_ids:
for alignment_id, expression_id in ii.items():
expression_object = self.ws.get_objects2(
{'objects':
[{'ref': expression_id}]})['data'][0]
handle_id = expression_object['data']['file']['hid']
expression_name = expression_object['info'][1]
expression_dir = os.path.join(group_file_dir, expression_name)
self._mkdir_p(expression_dir)
print('expression_name: ' + str(expression_dir) + ' ' +
str(group_name_indices[condition_labels[index]]))
sgf.write("{0} {1}\n".format(expression_dir,
group_name_indices[condition_labels[index]]))
self.dfu.shock_to_file({'handle_id': handle_id,
'file_path': expression_dir,
'unpack': 'unpack'})
required_files = [
'e2t.ctab',
'e_data.ctab',
'i2t.ctab',
'i_data.ctab',
't_data.ctab']
for file in glob.glob(expression_dir + '/*'):
if not os.path.basename(file) in required_files:
os.remove(file)
index += 1
return sgf_name
def _cleanup(self, directory=None):
"""
Clean up after the job. At the moment this just means removing the working
directory, but later could mean other things.
"""
try:
# it would not delete if fold is not empty
shutil.rmtree(directory, ignore_errors=True)
# need to iterate each entry
except IOError as e:
log("Unable to remove working directory {0}".format(directory))
raise
def _setupWorkingDir(self, directory=None):
"""
Clean up an existing workingdir and create a new one
"""
try:
if os.path.exists(directory):
self._cleanup(directory)
os.mkdir(directory)
except IOError:
log("Unable to setup working dir {0}".format(directory))
raise
def _check_intron_measurements(self, sample_dir_group_table_file):
"""
Check if intron measurements files are non-empty
:param sample_dir_group_table_file:
:return:
"""
log('checking for intron level measurements... ')
file = open(sample_dir_group_table_file, 'r')
textFileLines = file.readlines()
for line in textFileLines:
expr_dir = line.split()[0]
log(expr_dir)
i2t_file = open(os.path.join(expr_dir, 'i2t.ctab'), 'r')
if len(i2t_file.readlines()) <= 1: # only header line exists
raise Exception("No intron measurements found! Input expressions are possibly "
"from a prokaryote. Ballgown functions only on eukaryotic data."
" Consider using DeSeq2 or CuffDiff instead of BallGown.")
idata_file = open(os.path.join(expr_dir, 'i_data.ctab'), 'r')
if len(idata_file.readlines()) <= 1: # only header line exists
raise Exception("No intron measurements found! Input expressions are possibly "
"from a prokaryote. Ballgown functions only on eukaryotic data."
" Consider using DeSeq2 or CuffDiff instead of BallGown")
def run_ballgown_diff_exp(self,
rscripts_dir,
sample_dir_group_table_file,
ballgown_output_dir,
output_csv,
volcano_plot_file
):
""" Make R call to execute the system
:param rscripts_dir:
:param sample_dir_group_table_file:
:param ballgown_output_dir:
sample_group_table is a listing of output Stringtie subdirectories,
(full path specification) paired with group label (0 or 1), ie
/path/WT_rep1_stringtie 0
/path/WT_rep2_stringtie 0
/path/EXP_rep1_stringtie 1
/path/EXP_rep2_stringtie 1
(order doesn't matter, but the directory-group correspondance does)
:param output_csv:
:param volcano_plot_file:
:return:
"""
# check if intron-level expression measurements are present
self._check_intron_measurements(sample_dir_group_table_file)
rcmd_list = ['Rscript', os.path.join(rscripts_dir, 'ballgown_fpkmgenematrix.R'),
'--sample_dir_group_table', sample_dir_group_table_file,
'--output_dir', ballgown_output_dir,
'--output_csvfile', output_csv,
'--volcano_plot_file', volcano_plot_file
]
rcmd_str = " ".join(str(x) for x in rcmd_list)
log("rcmd_string is {0}".format(rcmd_str))
openedprocess = subprocess.Popen(rcmd_str, shell=True)
openedprocess.wait()
# Make sure the openedprocess.returncode is zero (0)
if openedprocess.returncode != 0:
log("R script did not return normally, return code - "
+ str(openedprocess.returncode))
raise Exception("Rscript failure")
def load_diff_expr_matrix(self, ballgown_output_dir, output_csv):
"""
Reads csv diff expr matrix file from Ballgown and returns as a
dictionary of rows with the gene as key. Each key gives a row of
length three corresponding to fold_change, pval and qval in string form
- can include 'NA'
:param ballgown_output_dir
:param output_csv:
:return:
"""
diff_matrix_file = os.path.join(ballgown_output_dir, output_csv)
if not os.path.isfile(diff_matrix_file):
raise Exception("differential expression matrix csvfile {0} doesn't exist!".format(
diff_matrix_file))
n = 0
dm = {}
with open(diff_matrix_file, "r") as csv_file:
csv_rows = csv.reader(csv_file, delimiter="\t", quotechar='"')
for row in csv_rows:
n = n + 1
if (n == 1):
if (row != ['id', 'fc', 'pval', 'qval']):
raise Exception(
"did not get expected column heading from {0}".format(
diff_matrix_file))
else:
if (len(row) != 4):
raise Exception(
"did not get 4 elements in row {0} of csv file {1} ".format(
n, diff_matrix_file))
key = row[0]
# put in checks for NA or numeric for row[1] through 4
if (key in dm):
raise Exception(
"duplicate key {0} in row {1} of csv file {2} ".format(
key, n, diff_matrix_file))
dm[key] = row[1:5]
return dm
def _transform_expression_set_data(self, expression_set_data):
"""
The stitch to connect KBaseSets.ExpressionSet-2.0 type data to
the older KBaseRNASeq.RNASeqExpressionSet-3.0 that the implementation
depends on. This is done by doing a dive into the nested alignment
object ref and getting the required data
:param expression_set_data:
:return: transformed expression_set_data
"""
transform = dict()
# get genome id
expression_ref = expression_set_data['items'][0]['ref']
wsid, objid, ver = expression_ref.split('/')
expression_obj = self.ws.get_objects([{'objid': objid, 'wsid': wsid}])
transform['genome_id'] = expression_obj[0]['data']['genome_id']
# get sampleset_id
#alignment_ref = expression_obj[0]['data']['mapped_rnaseq_alignment'].values()[0]
#wsid, objid, ver = alignment_ref.split('/')
#alignment_obj = self.ws.get_objects([{'objid': objid, 'wsid': wsid}])
#transform['sampleset_id'] = alignment_obj[0]['data']['sampleset_id']
# build mapped_expression_ids
mapped_expression_ids = list()
for item in expression_set_data['items']:
expression_ref = item['ref']
wsid, objid, ver = expression_ref.split('/')
expression_obj = self.ws.get_objects([{'objid': objid, 'wsid': wsid}])
alignment_ref = expression_obj[0]['data']['mapped_rnaseq_alignment'].values()[0]
mapped_expression_ids.append({alignment_ref: expression_ref})
transform['mapped_expression_ids'] = mapped_expression_ids
return transform
def _build_condition_label_list(self, mapped_expression_ids):
"""
Extracts the condition labels from each expression in the specified expression set data
and builds a list of condition labels
:param expression_set_data: expression set data
:return: list of condition labels whose order resembles the expression order in
the expression data
"""
condition_labels = list()
for ii in mapped_expression_ids:
for alignment_id, expression_id in ii.items():
expression_object = self.ws.get_objects2(
{'objects':
[{'ref': expression_id}]})['data'][0]
condition_labels.append(expression_object['data']['condition'])
return condition_labels
def _update_output_file_header(self, output_file):
"""
Modify header of output file (required by DifferentialExpressionUtils)
:param output_file:
:return:
"""
f = open(output_file, 'r')
filedata = f.read()
f.close()
modified_output = filedata.replace(
'"id"\t"fc"\t"pval"\t"qval"',
'gene_id\tlog2_fold_change\tp_value\tq_value')
f = open(output_file, 'w')
f.write(modified_output)
f.close()
def _check_input_labels(self, condition_pair_subset, available_condition_labels):
"""
_check_input_labels: check input condition pairs
"""
checked = True
# example struct: [{u'condition': u'hy5'}, {u'condition': u'WT'}]
condition_values = set()
for condition in condition_pair_subset:
condition_values.add(condition['condition'])
if len(condition_values) < 2:
error_msg = 'At least two unique conditions must be specified. '
raise ValueError(error_msg)
for condition in condition_pair_subset:
label = condition['condition'].strip()
if label not in available_condition_labels:
error_msg = 'Condition label "{}" is not a valid condition. '.format(label)
error_msg += 'Must be one of "{}"'.format(available_condition_labels)
raise ValueError(error_msg)
return checked
def run_ballgown_app(self, params):
"""
run_ballgown_app: run Ballgown app
(https://www.bioconductor.org/packages/release/bioc/html/ballgown.html)
required params:
expressionset_ref: ExpressionSet object reference
diff_expression_matrix_set_suffix: suffix to KBaseSets.DifferetialExpressionMatrixSet
name
condition_labels: conditions for expression set object
alpha_cutoff: q value cutoff
fold_change_cutoff: fold change cutoff
workspace_name: the name of the workspace it gets saved to
optional params:
fold_scale_type: one of ["linear", "log2+1", "log10+1"]
return:
result_directory: folder path that holds all files generated by run_deseq2_app
diff_expression_matrix_set_ref: generated KBaseSets.DifferetialExpressionMatrixSet
object reference
report_name: report name generated by KBaseReport
report_ref: report reference generated by KBaseReport
"""
log('--->\nrunning BallgownUtil.run_ballgown_app\n' +
'params:\n{}'.format(json.dumps(params, indent=1)))
self._validate_run_ballgown_app_params(params)
expressionset_ref = params.get('expressionset_ref')
expression_set_info = self.ws.get_object_info3({
"objects": [{"ref": expressionset_ref}]})['infos'][0]
expression_object_type = expression_set_info[2]
# set output object name
differential_expression_suffix = params['diff_expression_matrix_set_suffix']
expression_name = expression_set_info[1]
if re.match('.*_[Ee]xpression$', expression_name):
params['diff_expression_matrix_set_name'] = re.sub(
'_[Ee]xpression$', differential_expression_suffix, expression_name)
if re.match('.*_[Ee]xpression_[Ss]et$', expression_name):
params['diff_expression_matrix_set_name'] = re.sub(
'_[Ee]xpression_[Ss]et$', differential_expression_suffix, expression_name)
else:
params['diff_expression_matrix_set_name'] = expression_name + \
differential_expression_suffix
log('--->\nexpression object type: \n' +
'{}'.format(expression_object_type))
if re.match('KBaseRNASeq.RNASeqExpressionSet-\d.\d', expression_object_type):
expression_set_data = self.ws.get_objects2(
{'objects':
[{'ref': expressionset_ref}]})['data'][0]['data']
elif re.match('KBaseSets.ExpressionSet-\d.\d', expression_object_type):
expression_set_data = self.ws.get_objects2(
{'objects':
[{'ref': expressionset_ref}]})['data'][0]['data']
expression_set_data = self._transform_expression_set_data(expression_set_data)
mgroup = MultiGroup(self.ws)
pairwise_mapped_expression_ids = mgroup.build_pairwise_groups(
expression_set_data['mapped_expression_ids'])
ballgown_output_dir = os.path.join(self.scratch, "ballgown_out")
log("ballgown output dir is {0}".format(ballgown_output_dir))
self._setupWorkingDir(ballgown_output_dir)
# get set of all condition labels
available_condition_labels = \
self._build_condition_label_list(expression_set_data['mapped_expression_ids'])
if params.get('run_all_combinations'):
requested_condition_labels = available_condition_labels
else:
# get set of user specified condition labels
condition_pair_subset = params.get('condition_pair_subset')
if self._check_input_labels(condition_pair_subset, available_condition_labels):
requested_condition_labels = list()
# example: [{u'condition': u'hy5'}, {u'condition': u'WT'}]
for condition in condition_pair_subset:
if condition.get('condition').strip() not in requested_condition_labels:
requested_condition_labels.append(condition.get('condition').strip())
log("User requested pairwise combinations from condition label list : " +
str(requested_condition_labels))
diff_expr_files = list()
for mapped_expression_ids in pairwise_mapped_expression_ids:
print('processing pairwise combination: ')
pprint(mapped_expression_ids)
print('with condtion labels: ')
condition_labels = self._build_condition_label_list(mapped_expression_ids)
pprint(condition_labels)
# skip if condition labels in this pairwise combination don't exist in
# set of user requested condition labels
skip = False
for condition in condition_labels:
if condition not in requested_condition_labels:
log("skipping " + str(condition_labels))
skip = True
if skip:
continue
sample_dir_group_file = self.get_sample_dir_group_file(mapped_expression_ids,
condition_labels)
log("about to run_ballgown_diff_exp")
rscripts_dir = '/kb/module/rscripts'
condition_labels_uniqued = list()
for condition in condition_labels:
if condition not in condition_labels_uniqued:
condition_labels_uniqued.append(condition)
output_csv = 'ballgown_diffexp_' + \
condition_labels_uniqued[0] + '_' + condition_labels_uniqued[1] + '.tsv'
volcano_plot_file = 'volcano_plot_' + \
condition_labels_uniqued[0] + '_' + condition_labels_uniqued[1] + '.png'
self.run_ballgown_diff_exp(rscripts_dir,
sample_dir_group_file,
ballgown_output_dir,
output_csv,
volcano_plot_file)
log("back from run_ballgown_diff_exp, about to load diff exp matrix file")
# diff_expr_matrix = self.load_diff_expr_matrix(ballgown_output_dir,
# output_csv) # read file before its zipped
self._update_output_file_header(os.path.join(ballgown_output_dir, output_csv))
diff_expr_file = dict()
diff_expr_file.update({'condition_mapping':
{condition_labels_uniqued[0]: condition_labels_uniqued[1]}})
diff_expr_file.update(
{'diffexpr_filepath': os.path.join(ballgown_output_dir, output_csv)})
diff_expr_files.append(diff_expr_file)
deu_param = {
'destination_ref': params['workspace_name'] + '/' +
params['diff_expression_matrix_set_name'],
'diffexpr_data': diff_expr_files,
'tool_used': TOOL_NAME,
'tool_version': TOOL_VERSION,
'genome_ref': expression_set_data.get('genome_id'),
}
diff_expression_matrix_set_ref = self.deu.save_differential_expression_matrix_set(
deu_param)
returnVal = {'result_directory': ballgown_output_dir,
'diff_expression_matrix_set_ref':
diff_expression_matrix_set_ref['diffExprMatrixSet_ref']}
report_output = self._generate_report(params,
ballgown_output_dir, diff_expression_matrix_set_ref)
returnVal.update(report_output)
return returnVal
class STARUtils:
STAR_VERSION = 'STAR 2.5.3a'
STAR_BIN = '/kb/deployment/bin/STAR'
STAR_IDX_DIR = 'STAR_Genome_index'
STAR_OUT_DIR = 'STAR_Output'
PARAM_IN_WS = 'output_workspace'
PARAM_IN_FASTA_FILES = 'genomeFastaFiles'
PARAM_IN_OUTFILE_PREFIX = 'outFileNamePrefix'
PARAM_IN_STARMODE = 'runMode'
PARAM_IN_THREADN = 'runThreadN'
PARAM_IN_READS_FILES = 'readFilesIn'
PARAM_IN_READS = 'readsset_ref'
PARAM_IN_GENOME = 'genome_ref'
SET_READS = 'set_reads_refs'
def __init__(self, scratch_dir, workspace_url, callback_url, srv_wiz_url, provenance):
self.workspace_url = workspace_url
self.callback_url = callback_url
self.srv_wiz_url = srv_wiz_url
self.au = AssemblyUtil(self.callback_url)
self.dfu = DataFileUtil(self.callback_url, service_ver='beta')
self.scratch = scratch_dir
self.working_dir = scratch_dir
self.prog_runner = Program_Runner(self.STAR_BIN, self.scratch)
self.provenance = provenance
self.ws_client = Workspace(self.workspace_url)
self.parallel_runner = KBParallel(self.callback_url)
self.qualimap = kb_QualiMap(self.callback_url, service_ver='dev')
self.set_api_client = SetAPI(self.srv_wiz_url, service_ver='dev')
self.eu = ExpressionUtils(self.callback_url, service_ver='beta')
def process_params(self, params):
"""
process_params: checks params passed to run_star method and set default values
"""
log('Start validating run_star parameters')
# check for required parameters
if params.get(self.PARAM_IN_WS, None) is None:
raise ValueError(self.PARAM_IN_WS + ' parameter is required')
if params.get(self.PARAM_IN_STARMODE, None) is None:
params[self.PARAM_IN_STARMODE] = 'alignReads'
if params.get(self.PARAM_IN_GENOME, None) is None:
raise ValueError(self.PARAM_IN_GENOME +
' parameter is required for generating genome index')
if (params.get(self.PARAM_IN_STARMODE, None) is not None and
params[self.PARAM_IN_STARMODE] != "genomeGenerate"):
if params.get(self.PARAM_IN_READS, None) is None:
raise ValueError(self.PARAM_IN_READS +
' parameter is required for reads mapping')
if not valid_string(params[self.PARAM_IN_READS], is_ref=True):
raise ValueError("Parameter readsset_ref must be a valid Workspace object reference, "
"not {}".format(params.get(self.PARAM_IN_READS, None)))
if params.get(self.PARAM_IN_THREADN, None) is not None:
if not isinstance(params[self.PARAM_IN_THREADN], int):
raise ValueError(self.PARAM_IN_HASH_THREADN + ' must be of type int')
else:
params[self.PARAM_IN_THREADN] = 2
if "alignment_suffix" not in params or not valid_string(params["alignment_suffix"]):
raise ValueError("Parameter alignment_suffix must be a valid Workspace object string, "
"not {}".format(params.get("alignment_suffix", None)))
if params.get(self.PARAM_IN_OUTFILE_PREFIX, None) is not None:
if params[self.PARAM_IN_OUTFILE_PREFIX].find('/') != -1:
raise ValueError(self.PARAM_IN_OUTFILE_PREFIX + ' cannot contain subfolder(s).')
else:
params[self.PARAM_IN_OUTFILE_PREFIX] = 'star_'
if params.get('create_report', None) is None:
params['create_report'] = 0
return self._setDefaultParameters(params)
def _setDefaultParameters(self, params_in):
"""set default for this group of parameters
"""
params = copy.deepcopy(params_in)
if params.get('outFilterType', None) is None:
params['outFilterType'] = "\"BySJout\""
if params.get('outFilterMultimapNmax', None) is None:
params['outFilterMultimapNmax'] = 20
if params.get('outSAMtype', None) is None:
params['outSAMtype'] = 'BAM'
if params.get('outSAMattrIHstart', None) is None:
params['outSAMattrIHstart'] = 0
if params.get('outSAMstrandField', None) is None:
params['outSAMstrandField'] = 'intronMotif'
if params.get('outFilterIntronMotifs', None) is None:
params['outFilterIntronMotifs'] = 'RemoveNoncanonical'
if params.get(self.SET_READS, None) is None:
params[self.SET_READS] = self._get_reads_refs_from_setref(params)
return params
def _get_genome_gtf_file(self, gnm_ref, gtf_file_dir):
"""
Get data from genome object ref and return the GTF filename (with path)
for STAR indexing and mapping.
STAR uses the reference annotation to guide assembly and for creating alignment
"""
log("Converting genome {0} to GFF file in folder {1}".format(gnm_ref, gtf_file_dir))
gfu = GenomeFileUtil(self.callback_url)
try:
gfu_ret = gfu.genome_to_gff({self.PARAM_IN_GENOME: gnm_ref,
'is_gtf': 1,
'target_dir': gtf_file_dir
})
except ValueError as egfu:
log('GFU getting GTF file raised error:\n')
pprint(egfu)
return None
else:#no exception raised
return gfu_ret.get('file_path')
def _construct_indexing_cmd(self, params):
# STEP 1: construct the command for running `STAR --runMode genomeGenerate...`
idx_cmd = [self.STAR_BIN]
idx_cmd.append('--genomeDir')
idx_cmd.append(params[self.STAR_IDX_DIR])
idx_cmd.append('--' + self.PARAM_IN_STARMODE)
idx_cmd.append('genomeGenerate')
idx_cmd.append('--' + self.PARAM_IN_THREADN)
idx_cmd.append(str(params[self.PARAM_IN_THREADN]))
if params.get(self.PARAM_IN_FASTA_FILES, None) is not None:
idx_cmd.append('--' + self.PARAM_IN_FASTA_FILES)
for fasta_file in params[self.PARAM_IN_FASTA_FILES]:
idx_cmd.append(fasta_file)
# STEP 2: append the standard optional inputs
if params.get('sjdbGTFfile', None) is not None:
idx_cmd.append('--sjdbGTFfile')
idx_cmd.append(params['sjdbGTFfile'])
if (params.get('sjdbOverhang', None) is not None
and params['sjdbOverhang'] > 0):
idx_cmd.append('--sjdbOverhang')
idx_cmd.append(str(params['sjdbOverhang']))
#print ('STAR indexing CMD:')
#print ' '.join(idx_cmd)
return idx_cmd
def _construct_mapping_cmd(self, params):
if params.get(self.PARAM_IN_STARMODE, None) is None:
params[self.PARAM_IN_STARMODE] = 'alignReads'
# STEP 1: set the working folder housing the STAR output results as well as the reads info
star_out_dir = ''
if params.get('align_output', None) is None:
star_out_dir = self.scratch
else:
star_out_dir = params['align_output']
# STEP 2: construct the command for running STAR mapping
mp_cmd = [self.STAR_BIN]
mp_cmd.append('--genomeDir')
mp_cmd.append(params[self.STAR_IDX_DIR])
mp_cmd.append('--' + self.PARAM_IN_STARMODE)
mp_cmd.append(params[self.PARAM_IN_STARMODE])
mp_cmd.append('--' + self.PARAM_IN_THREADN)
mp_cmd.append(str(params[self.PARAM_IN_THREADN]))
if params.get(self.PARAM_IN_READS_FILES, None) is not None:
#print('Input reads files:\n' + pformat(params[self.PARAM_IN_READS_FILES]))
mp_cmd.append('--' + self.PARAM_IN_READS_FILES)
for reads_file in params[self.PARAM_IN_READS_FILES]:
mp_cmd.append(reads_file)
readName, readsExtension = os.path.splitext(reads_file)
#print ('Reads file name-- {}/extension-- {}:'.format(readName, readsExtension))
if readsExtension == '.gz':
mp_cmd.append('--readFilesCommand')
mp_cmd.append('gunzip')
mp_cmd.append('-c')
if readsExtension == '.bz2':
mp_cmd.append('--readFilesCommand')
mp_cmd.append('bunzip2')
mp_cmd.append('-c')
# STEP 3: appending the advanced optional inputs
mp_cmd.append('--' + self.PARAM_IN_OUTFILE_PREFIX)
mp_cmd.append(os.path.join(star_out_dir, params[self.PARAM_IN_OUTFILE_PREFIX]))
if params.get('sjdbGTFfile', None) is not None:
mp_cmd.append('--sjdbGTFfile')
mp_cmd.append(params['sjdbGTFfile'])
if (params.get('sjdbOverhang', None) is not None
and params['sjdbOverhang'] > 0):
mp_cmd.append('--sjdbOverhang')
mp_cmd.append(str(params['sjdbOverhang']))
if (params.get('outFilterType', None) is not None
and isinstance(params['outFilterType'], str)):
mp_cmd.append('--outFilterType')
mp_cmd.append(params['outFilterType'])
if (params.get('outFilterMultimapNmax', None) is not None
and isinstance(params['outFilterMultimapNmax'], int)
and params['outFilterMultimapNmax'] >= 0):
mp_cmd.append('--outFilterMultimapNmax')
mp_cmd.append(str(params['outFilterMultimapNmax']))
#output sorted file:Aligned.sortedByCoord.out.bam
#allowed values of --outSAMtype are BAM Unsorted or SortedByCoordinate or both
if params.get('outSAMtype', None) is not None:
mp_cmd.append('--outSAMtype')
mp_cmd.append(params['outSAMtype'])
if params.get('outSAMtype', None) == 'BAM':
mp_cmd.append('SortedByCoordinate')
# 'It is recommended to remove the non-canonical junctions for Cnks runs using
# --outFilterIntronMotifs RemoveNoncanonical'
if params.get('outFilterIntronMotifs', None) is not None:
mp_cmd.append('--outFilterIntronMotifs')
mp_cmd.append('RemoveNoncanonical')
if (params.get('outSAMattrIHstart', None) is not None
and isinstance(params['outSAMattrIHstart'], int)
and params['outSAMattrIHstart'] >= 0):
mp_cmd.append('--outSAMattrIHstart')
mp_cmd.append(str(params['outSAMattrIHstart']))
if (params.get('outSAMstrandField', None) is not None
and isinstance(params['outSAMstrandField'], str)):
mp_cmd.append('--outSAMstrandField')
mp_cmd.append(params['outSAMstrandField'])
quant_modes = ["TranscriptomeSAM", "GeneCounts", "Both"]
if (params.get('quantMode', None) is not None
and params.get('quantMode', None) in quant_modes):
mp_cmd.append('--quantMode')
if params['quantMode'] == "Both":
mp_cmd.append("TranscriptomeSAM")
mp_cmd.append("GeneCounts")
else:
mp_cmd.append(params['quantMode'])
if (params.get('alignSJoverhangMin', None) is not None
and isinstance(params['alignSJoverhangMin'], int)
and params['alignSJoverhangMin'] > 0):
mp_cmd.append('--alignSJoverhangMin')
mp_cmd.append(str(params['alignSJoverhangMin']))
if (params.get('alignSJDBoverhangMin', None) is not None
and isinstance(params['alignSJDBoverhangMin'], int)
and params['alignSJDBoverhangMin'] > 0):
mp_cmd.append('--alignSJDBoverhangMin')
mp_cmd.append(str(params['alignSJDBoverhangMin']))
if (params.get('outFilterMismatchNmax', None) is not None
and isinstance(params['outFilterMismatchNmax'], int)
and params['outFilterMismatchNmax'] > 0):
mp_cmd.append('--outFilterMismatchNmax')
mp_cmd.append(str(params['outFilterMismatchNmax']))
if (params.get('alignIntronMin', None) is not None
and isinstance(params['alignIntronMin'], int)
and params['alignIntronMin'] > 0):
mp_cmd.append('--alignIntronMin')
mp_cmd.append(str(params['alignIntronMin']))
if (params.get('alignIntronMax', None) is not None
and isinstance(params['alignIntronMax'], int)
and params['alignIntronMax'] >= 0):
mp_cmd.append('--alignIntronMax')
mp_cmd.append(str(params['alignIntronMax']))
if (params.get('alignMatesGapMax', None) is not None
and isinstance(params['alignMatesGapMax'], int)
and params['alignMatesGapMax'] >= 0):
mp_cmd.append('--alignMatesGapMax')
mp_cmd.append(str(params['alignMatesGapMax']))
#print ('STAR mapping CMD:')
#print ' '.join(mp_cmd)
return mp_cmd
def _exec_indexing(self, params):
log('Running STAR index generating with params:\n' + pformat(params))
idx_cmd = self._construct_indexing_cmd(params)
exitCode = self.prog_runner.run(idx_cmd, self.scratch)
return exitCode
def _exec_mapping(self, params):
log('Running STAR mapping with params:\n' + pformat(params))
mp_cmd = self._construct_mapping_cmd(params)
exitCode = self.prog_runner.run(mp_cmd, self.scratch)
return exitCode
def _exec_star_pipeline(self, params, rds_files, rds_name, idx_dir, out_dir):
# build the parameters
params_idx = self._get_indexing_params(params, idx_dir)
params_mp = self._get_mapping_params(params, rds_files, rds_name, idx_dir, out_dir)
# execute indexing and then mapping
retVal = {}
try:
if params[self.PARAM_IN_STARMODE]=='genomeGenerate':
ret = self._exec_indexing(params_idx)
else:
ret = 0
while( ret != 0 ):
time.sleep(1)
except ValueError as eidx:
log('STAR genome indexing raised error:\n')
pprint(eidx)
else:#no exception raised by genome indexing and STAR returns 0, then run mapping
params_mp[self.PARAM_IN_STARMODE] = 'alignReads'
try:
ret = self._exec_mapping(params_mp)
while( ret != 0 ):
time.sleep(1)
except ValueError as emp:
log('STAR mapping raised error:\n')
pprint(emp)
else:#no exception raised by STAR mapping and STAR returns 0, then move to saving and reporting
ret = {'star_idx': star_idx, 'star_output': params_mp.get('align_output')}
return ret
def upload_STARalignment(self, input_params, reads_ref, reads_info, output_bam_file):
"""
Uploads the alignment file + metadata.
Returns the STAR alignment reference.
"""
aligner_opts = dict()
for k in input_params:
aligner_opts[k] = str(input_params[k])
pprint(reads_info)
alignment_name = reads_ref['alignment_output_name']
align_upload_params = {
"destination_ref": "{}/{}".format(input_params[self.PARAM_IN_WS], alignment_name),
"file_path": output_bam_file,
"assembly_or_genome_ref": input_params[self.PARAM_IN_GENOME],
"read_library_ref": reads_info['object_ref'],
"library_type": reads_info['style'],
"condition": reads_info['condition'],
"aligned_using": 'STAR',
"aligner_version":self.STAR_VERSION,
"aligner_opts": aligner_opts
}
pprint(align_upload_params)
ra_util = ReadsAlignmentUtils(self.callback_url, service_ver='beta')
rau_upload_ret = ra_util.upload_alignment(align_upload_params)
alignment_ref = rau_upload_ret["obj_ref"]
print("STAR alignment uploaded as object {}".format(alignment_ref))
return rau_upload_ret
def generate_report_for_single_run(self, run_output_info, params):
input_ref = run_output_info['upload_results']['obj_ref']
index_dir = run_output_info['index_dir']
output_dir = run_output_info['output_dir']
output_files = self._generate_output_file_list(index_dir, output_dir)
# first run qualimap
qualimap_report = self.qualimap.run_bamqc({'input_ref': input_ref})
qc_result_zip_info = qualimap_report['qc_result_zip_info']
# create report
report_text = 'Ran on a single reads library.\n\n'
alignment_info = self.get_obj_infos(input_ref)[0]
report_text = 'Created ReadsAlignment: ' + str(alignment_info[1]) + '\n'
report_text += ' ' + input_ref + '\n'
kbr = KBaseReport(self.callback_url)
report_info = kbr.create_extended_report({'message': report_text,
'file_links': output_files,
'objects_created': [{'ref': input_ref,
'description': 'ReadsAlignment'}],
'report_object_name': 'kb_STAR_report_' + str(uuid.uuid4()),
'direct_html_link_index': 0,
'html_links': [{'shock_id': qc_result_zip_info['shock_id'],
'name': qc_result_zip_info['index_html_file_name'],
'label': qc_result_zip_info['name']}],
'html_window_height': 366,
'workspace_name': params['output_workspace']
})
return report_info #{'report_name': report_info['name'], 'report_ref': report_info['ref']}
def _get_reads_info(self, reads, readsSet_ref):
'''
_get_reads_info:fetches the detailed info for each reads with ref in list reads_refs
return an object of the following structure:
{
"style": "paired", "single", or "interleaved",
"file_fwd": path_to_file,
"name": name of the reads,
"file_rev": path_to_file, only if paired end,
"object_ref": reads reference for downstream convenience,
"condition": the condition for the reads.
}
'''
try:
print("Fetching FASTA file from reads reference {}".format(reads['ref']))
ret_reads_info = fetch_reads_from_reference(reads['ref'], self.callback_url)
except ValueError:
print("Incorrect object type for fetching a FASTA file!")
raise
if ret_reads_info.get("file_fwd", None) is None:
raise RuntimeError("FASTA file fetched from reads {} doesn't seem to exist!".format(reads['ref']))
else:
if reads.get('condition', None) is not None:
ret_reads_info['condition'] = reads['condition']
else:
ret_reads_info['condition'] = 'unspecified'
if reads.get('object_ref', None) != readsSet_ref:
ret_reads_info[self.PARAM_IN_READS] = readsSet_ref
return ret_reads_info
def _get_genome_fasta(self, gnm_ref):
genome_fasta_files = list()
if gnm_ref is not None:
try:
print("Fetching FASTA file from object {}".format(gnm_ref))
genome_fasta_file = fetch_fasta_from_object(gnm_ref, self.workspace_url, self.callback_url)
print("Done fetching FASTA file! Path = {}".format(genome_fasta_file.get("path", None)))
except ValueError:
print("Incorrect object type for fetching a FASTA file!")
raise
if genome_fasta_file.get("path", None) is None:
raise RuntimeError("FASTA file fetched from object {} doesn't seem exist!".format(gnm_ref))
else:
genome_fasta_files.append(genome_fasta_file["path"])
return genome_fasta_files
def convert_params(self, validated_params):
"""
Convert input parameters with KBase ref format into STAR parameters,
and add the advanced options.
"""
params = copy.deepcopy(validated_params)
params['runMode'] = 'genomeGenerate'
if validated_params.get('create_report', None) is not None:
params['create_report'] = validated_params['create_report']
if validated_params.get('concurrent_local_tasks', None) is not None:
params['concurrent_local_tasks'] = validated_params['concurrent_local_tasks']
if validated_params.get('concurrent_njsw_tasks', None) is not None:
params['concurrent_njsw_tasks'] = validated_params['concurrent_njsw_tasks']
if validated_params.get('alignmentset_suffix', None) is not None:
params['alignmentset_suffix'] = validated_params['alignmentset_suffix']
# Add advanced options from validated_params to params
sjdbGTFfile = validated_params.get("sjdbGTFfile", None)
if sjdbGTFfile is not None:
params['sjdbGTFfile'] = sjdbGTFfile
else:
params['sjdbGTFfile'] = self._get_genome_gtf_file(
params[self.PARAM_IN_GENOME],
os.path.join(self.scratch, self.STAR_IDX_DIR))
if validated_params.get('sjdbOverhang', None) is not None :
params['sjdbOverhang'] = validated_params['sjdbOverhang']
else:
params['sjdbOverhang'] = 100
quant_modes = ["TranscriptomeSAM", "GeneCounts", "Both"]
if (validated_params.get('quantMode', None) is not None
and validated_params.get('quantMode', None) in quant_modes):
params['quantMode'] = validated_params['quantMode']
else:
params['quantMode'] = 'Both'
return params
def _get_indexing_params(self, params, star_idx_dir):
params_idx = {
'runMode': 'genomeGenerate',
'runThreadN': params[self.PARAM_IN_THREADN],
self.STAR_IDX_DIR: star_idx_dir,
'genomeFastaFiles': params[self.PARAM_IN_FASTA_FILES]
}
if params.get('sjdbGTFfile', None) is not None:
params_idx['sjdbGTFfile'] = params['sjdbGTFfile']
if params.get('sjdbOverhang', None) is not None :
params_idx['sjdbOverhang'] = params['sjdbOverhang']
return params_idx
def _get_mapping_params(self, params, rds_files, rds_name, idx_dir, out_dir):
''' build the mapping parameters'''
aligndir = out_dir
if rds_name:
aligndir = os.path.join(out_dir, rds_name)
self._mkdir_p(aligndir)
#print '**********STAR output directory created:{}'.format(aligndir)
params_mp = copy.deepcopy(params)
params_mp['runMode'] = 'alignReads'
params_mp['readFilesIn'] = rds_files
params_mp[self.STAR_IDX_DIR] = idx_dir
params_mp['align_output'] = aligndir
return params_mp
def determine_input_info(self, validated_params):
''' get info on the readsset_ref object and determine if we run once or run on a set
input info provides information on the input and tells us if we should
run as a single_library or as a set:
input_info = {'run_mode': '', 'info': [..], 'ref': '55/1/2'}
'''
info = self.get_obj_infos(validated_params[self.PARAM_IN_READS])[0]
obj_type = self.get_type_from_obj_info(info)
if obj_type in ['KBaseAssembly.PairedEndLibrary', 'KBaseAssembly.SingleEndLibrary',
'KBaseFile.PairedEndLibrary', 'KBaseFile.SingleEndLibrary']:
return {'run_mode': 'single_library', 'info': info, 'ref': validated_params[self.PARAM_IN_READS]}
if obj_type == 'KBaseRNASeq.RNASeqSampleSet':
return {'run_mode': 'sample_set', 'info': info, 'ref': validated_params[self.PARAM_IN_READS]}
if obj_type == 'KBaseSets.ReadsSet':
return {'run_mode': 'sample_set', 'info': info, 'ref': validated_params[self.PARAM_IN_READS]}
raise ValueError('Object type of readsset_ref is not valid, was: ' + str(obj_type))
def determine_unique_reads_names(self, validated_params):
infos = self.get_obj_infos(validated_params[self.PARAM_IN_READS])
return get_unique_names(infos)
def get_type_from_obj_info(self, info):
return info[2].split('-')[0]
def get_name_from_obj_info(self, info):
return info[1]
def get_obj_infos(self, ref):
return self.ws_client.get_object_info3({'objects': [{'ref': ref}]})['infos']
def get_object_names(self, ref_list):
"""
From a list of workspace references, returns a mapping from ref -> name of the object.
"""
obj_ids = list()
for ref in ref_list:
obj_ids.append({"ref": ref})
info = self.ws_client.get_object_info3({"objects": obj_ids})
name_map = dict()
# we already have the refs as passed previously, so use those for mapping, as they're in
# the same order as what's returned.
for i in range(len(info["infos"])):
name_map[ref_list[i]] = info["infos"][i][1]
return name_map
def _mkdir_p(self, dir):
"""
_mkdir_p: make directory for given path
"""
log('Creating a new dir: ' + dir)
if not dir:
return
if not os.path.exists(dir):
os.makedirs(dir)
else:
log('{} has existed, so skip creating.'.format(dir))
def create_star_dirs(self, star_home):
'''creating the directories for STAR'''
# the index directory
idxdir = os.path.join(star_home, self.STAR_IDX_DIR)
self._mkdir_p(idxdir)
# the output directory
outdir = os.path.join(star_home, self.STAR_OUT_DIR)
self._mkdir_p(outdir)
return (idxdir, outdir)
def _get_reads_refs_from_setref(self, params):
readsSet_ref = params[self.PARAM_IN_READS]
reads_refs = list()
try:
#print("Fetching reads ref(s) from sample/reads set ref {}".format(readsSet_ref))
reads_refs = fetch_reads_refs_from_sampleset(
readsSet_ref,
self.workspace_url,
self.callback_url,
params)
#print("\nDone fetching reads ref(s) from readsSet {}--\nDetails:\n".format(readsSet_ref))
#pprint(reads_refs)
except ValueError:
print("Incorrect object type for fetching reads ref(s)!")
raise
return reads_refs
def _generate_output_file_list(self, idx_dir, out_dir):
"""
_generate_output_file_list: zip result files and generate file_links for report
"""
log('start packing result files')
output_files = list()
output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(output_directory)
star_index = os.path.join(output_directory, 'star_index.zip')
star_output = os.path.join(output_directory, 'star_output.zip')
self.zip_folder(idx_dir, star_index)
self.zip_folder(out_dir, star_output)
#star_index = self.zip_folder_withDFU(idx_dir, 'star_index')
#star_output = self.zip_folder_withDFU(out_dir, 'star_output')
output_files.append({'path': star_index,
'name': os.path.basename(star_index),
'label': os.path.basename(star_index),
'description': 'Index file(s) generated by STAR'})
output_files.append({'path': star_output,
'name': os.path.basename(star_output),
'label': os.path.basename(star_output),
'description': 'Output file(s) generated by STAR'})
return output_files
def zip_folder_withDFU(self, folder_path, output_name):
"""Zip the contents of an entire folder (with that folder included
in the archive). Empty subfolders will be included in the archive
as well.
"""
output_path = self.dfu.pack_file(
{'file_path': folder_path + '/' + output_name,
'pack': 'zip'})['file_path']
print "{} created successfully.".format(output_path)
#with zipfile.ZipFile(output_path, "r") as f:
#print 'Checking the zipped file......\n'
#for info in f.infolist():
# print info.filename, info.date_time, info.file_size, info.compress_size
#for fn in f.namelist():
#print fn
return output_path
def zip_folder(self, folder_path, output_path):
"""Zip the contents of an entire folder (with that folder included in the archive).
Empty subfolders could be included in the archive as well if the commented portion is used.
"""
with zipfile.ZipFile(output_path, 'w',
zipfile.ZIP_DEFLATED,
allowZip64=True) as ziph:
for root, folders, files in os.walk(folder_path):
# Include all subfolders, including empty ones.
#for folder_name in folders:
# absolute_path = os.path.join(root, folder_name)
# relative_path = os.path.join(os.path.basename(root), folder_name)
# print "Adding {} to archive.".format(absolute_path)
# ziph.write(absolute_path, relative_path)
for f in files:
absolute_path = os.path.join(root, f)
relative_path = os.path.join(os.path.basename(root), f)
#print "Adding {} to archive.".format(absolute_path)
ziph.write(absolute_path, relative_path)
print "{} created successfully.".format(output_path)
#with zipfile.ZipFile(output_path, "r") as f:
# print 'Checking the zipped file......\n'
# for info in f.infolist():
# print info.filename, info.date_time, info.file_size, info.compress_size
def _generate_html_report(self, out_dir, obj_ref):
"""
_generate_html_report: generate html summary report
"""
log('start generating html report')
html_report = list()
output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(output_directory)
result_file_path = os.path.join(output_directory, 'report.html')
star_obj = self.ws_client.get_objects2({'objects':
[{'ref': obj_ref}]})['data'][0]
star_obj_info = star_obj['info']
star_obj_data = star_obj['data']
star_obj_type = star_obj_info[2]
Overview_Content = ''
if re.match('KBaseRNASeq.RNASeqAlignment-\d.\d', star_obj_type):
Overview_Content += '<br/><table><tr><th>Generated Alignment Object</th>'
Overview_Content += '<th></th></tr>'
Overview_Content += '<tr><th>Alignment Name</th><th>Condition</th></tr>'
Overview_Content += '<tr><td>{} ({})</td>'.format(star_obj_info[1],obj_ref)
Overview_Content += '<td>{}</td></tr>'.format(star_obj_data['condition'])
Overview_Content += '</table>'
elif (re.match('KBaseRNASeq.RNASeqAlignmentSet-\d.\d', star_obj_type)
or re.match('KBaseSets.ReadsAlignmentSet-\d.\d', star_obj_type)
or re.match('KBaseSet.RNASeqAlignmentSet-\d.\d', star_obj_type)):
Overview_Content += '<br/><table><tr><th>Generated AlignmentSet Object</th></tr>'
Overview_Content += '<tr><td>{} ({})'.format(star_obj_info[1],obj_ref)
Overview_Content += '</td></tr></table>'
Overview_Content += '<p><br/></p>'
Overview_Content += '<table><tr><th>Generated Alignment Objects</th>'
Overview_Content += '<th></th></tr>'
Overview_Content += self._fill_html_trs('Alignment Name', star_obj_data)
Overview_Content += '</table>'
elif re.match('KBaseRNASeq.RNASeqExpression-\d.\d', star_obj_type):
Overview_Content += '<br/><table><tr><th>Generated Expression Object</th>'
Overview_Content += '<th></th></tr>'
Overview_Content += '<tr><th>Expression Name</th><th>Condition</th></tr>'
Overview_Content += '<tr><td>{} ({})</td>'.format(star_obj_info[1], obj_ref)
Overview_Content += '<td>{}</td></tr>'.format(star_obj_data['condition'])
Overview_Content += '</table>'
elif re.match('KBaseSets.ExpressionSet-\d.\d', star_obj_type):
Overview_Content += '<br/><table><tr><th>Generated ExpressionSet Object</th></tr>'
Overview_Content += '<tr><td>{} ({})'.format(star_obj_info[1], obj_ref)
Overview_Content += '</td></tr></table>'
Overview_Content += '<p><br/></p>'
Overview_Content += '<table><tr><th>Generated Expression Objects</th>'
Overview_Content += '<th></th></tr>'
Overview_Content += self._fill_html_trs('Expression Name', star_obj_data)
Overview_Content += '</table>'
with open(result_file_path, 'w') as result_file:
with open(os.path.join(os.path.dirname(__file__), 'report_template.html'),
'r') as report_template_file:
report_template = report_template_file.read()
report_template = report_template.replace('<p>Overview_Content</p>',
Overview_Content)
result_file.write(report_template)
html_report.append({'path': result_file_path,
'name': os.path.basename(result_file_path),
'label': os.path.basename(result_file_path),
'description': 'HTML summary report for STAR'})
return html_report
def _fill_html_trs(self, col_caption, obj_data):
'''
_fill_html_trs: simple creates an html string that has rows (tr) of td for a table
'''
tr_html_str = '<tr><th>{}</th><th>Condition</th></tr>'.format(col_caption)
for item in obj_data['items']:
item_obj = self.ws_client.get_objects2({'objects':[{'ref': item['ref']}]})['data'][0]
item_obj_info = item_obj['info']
item_obj_data = item_obj['data']
obj_name = item_obj_info[1]
tr_html_str += '<tr><td>{} ({})</td>'.format(obj_name, item['ref'])
tr_html_str += '<td>{}</td></tr>'.format(item_obj_data['condition'])
return tr_html_str
def _generate_star_report(self, obj_ref, report_text, html_links, workspace_name, index_dir, output_dir):
"""
_generate_star_report: generate summary report
"""
log('creating STAR report')
output_files = self._generate_output_file_list(index_dir, output_dir)
output_html_files = self._generate_html_report(output_dir, obj_ref)
output_html_files += html_links
star_obj = self.ws_client.get_objects2({'objects':[{'ref': obj_ref}]})['data'][0]
star_obj_info = star_obj['info']
star_obj_data = star_obj['data']
star_obj_type = star_obj_info[2]
if re.match('KBaseRNASeq.RNASeqAlignment-\d+.\d+', star_obj_type):
objects_created = [{'ref': obj_ref,
'description': 'RNASeqAlignment generated by STAR'}]
elif (re.match('KBaseRNASeq.RNASeqAlignmentSet-\d+.\d+', star_obj_type)
or re.match('KBaseSets.ReadsAlignmentSet-\d+.\d+', star_obj_type)
or re.match('KBaseSet.RNASeqAlignmentSet-\d+.\d+', star_obj_type)):
objects_created = [{'ref': obj_ref,
'description': '{} generated by STAR'.format(re.sub(r"-\d+.\d+", "",star_obj_type))}]
items = star_obj_data['items']
for item in items:
objects_created.append({'ref': item['ref'],
'description': 'Alignment generated by STAR'})
elif re.match('KBaseRNASeq.RNASeqExpression-\d+.\d+', star_obj_type):
objects_created = [{'ref': obj_ref,
'description': 'Expression generated by STAR'}]
elif re.match('KBaseSets.ExpressionSet-\d+.\d+', star_obj_type):
objects_created = [{'ref': obj_ref,
'description': 'ExpressionSet generated by STAR'}]
items = star_obj_data['items']
for item in items:
objects_created.append({'ref': item['ref'],
'description': 'Expression generated by STAR'})
report_params = {'message': report_text,
'workspace_name': workspace_name,
'file_links': output_files,
'objects_created': objects_created,
'html_links': output_html_files,
'direct_html_link_index': 0,
'html_window_height': 366,
'report_object_name': 'kb_STAR_report_' + str(uuid.uuid4())}
kbase_report_client = KBaseReport(self.callback_url)
report_output = kbase_report_client.create_extended_report(report_params)
return report_output
def upload_alignment_set(self, alignment_items, alignmentset_name, ws_name):
"""
Compiles and saves a set of alignment references (+ other stuff) into a
KBaseRNASeq.RNASeqAlignmentSet.
Returns the reference to the new alignment set.
alignment_items: [{
"ref": alignment_ref,
"label": condition label.
}]
"""
print("Uploading completed alignment set")
alignment_set = {
"description": "Alignments using STAR, v.{}".format(self.STAR_VERSION),
"items": alignment_items
}
set_info = self.set_api_client.save_reads_alignment_set_v1({
"workspace": ws_name,
"output_object_name": alignmentset_name,
"data": alignment_set
})
return set_info
class MutualInfoUtil:
def __init__(self, config):
self.ws_url = config["workspace-url"]
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.shock_url = config['shock-url']
self.dfu = DataFileUtil(self.callback_url)
self.ws = Workspace(self.ws_url, token=self.token)
self.scratch = config['scratch']
def _mkdir_p(self, path):
"""
_mkdir_p: make directory for given path
"""
if not path:
return
try:
os.makedirs(path)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
def test_dfu(self):
output_directory = self.scratch
#output_directory = "/kb/module/test1/"
#os.mkdir(output_directory)
#self._mkdir_p(output_directory)
test_file = os.path.join(output_directory, 'index.html')
with open(test_file, 'w') as file:
file.write("test!")
print("OUTPUT DIR")
print(output_directory)
print(os.listdir(output_directory))
print("file_to_shock")
report_shock_id = self.dfu.file_to_shock({
'file_path': output_directory,
'pack': 'targz'
#'pack': 'zip'
})
print(report_shock_id)
return
def _validate_run_flux_mutual_information_analysis_params(self, params):
"""
_validate_run_flux_mutual_information_analysis_params:
validates params passed to run_flux_mutual_information_analysis method
"""
log('start validating validate_run_flux_mutual_information_analysis params'
)
# check for required parameters
for p in ['fbamodel_id', 'compounds', 'media_id', 'workspace_name']:
if p not in params:
raise ValueError(
'"{}" parameter is required, but missing'.format(p))
def _get_file_from_ws(self, ref):
try:
file_path = self.ws.get_objects2({'objects': [{'ref': ref}]})
file_path = file_path['data'][0]
except Exception as e:
raise ValueError('Unable to get object from workspace: (' + ref +
')' + str(e))
return file_path
def _make_media_files(self, ws_name, base, compounds):
"""
Build and store media objects for each combination of compound added to the base media.
:param base: The base media file
:param compounds: the set of compound to test
:return: A list of media ids and a matrix with each media combination defined
"""
ref = ws_name + "/" + base
if base.find("/") != -1:
ref = base
output = self._get_file_from_ws(ref)
base_media = output['data']
base = output['info'][1]
myuuid = str(uuid.uuid4())
media_ids = [base]
new_media_list = []
media_matrix = [[""] + compounds]
media_matrix.append([[base] + [0] * len(compounds)])
for n_comp in range(1, len(compounds) + 1):
for combo in combinations(compounds, n_comp):
new_media_id = base + '_v%s' % len(media_matrix)
media_ids.append(new_media_id)
media_matrix.append(
[new_media_id] +
[1 if comp in combo else 0 for comp in compounds])
new_media = deepcopy(base_media)
new_media['id'] = new_media_id
new_media['name'] = new_media_id
for new_comp in combo:
new_media['mediacompounds'].append({
'compound_ref':
'48/1/1/compounds/id/%s' % new_comp.split('_')[0],
'concentration':
1.0,
'maxFlux':
1000,
'minFlux':
-1000
})
new_media_list.append(new_media)
print("Made %s Media Files" % (len(media_ids) - 1))
info = self.ws.save_objects({
'workspace':
ws_name,
"objects": [{
"hidden": 1,
"type": "KBaseBiochem.Media",
"data": media,
"name": myuuid + "-" + media['name']
} for media in new_media_list]
})
#print(info)
return media_ids, media_matrix, myuuid
def _run_fba(self, workspace_name, media_id_list, fbamodel_id, myuuid,
base_media):
print('running fba')
fba_tool_obj = fba_tools(self.callback_url, service_ver='dev')
new_media_list = []
for media in media_id_list:
if media == base_media:
new_media_list.append(workspace_name + "/" + media)
else:
new_media_list.append(workspace_name + "/" + myuuid + "-" +
media)
fba_tool_obj.run_flux_balance_analysis({
"max_c_uptake":
60, #"max_c_uptake": 6, // previously default is 6 later set to 60
"workspace": workspace_name,
"fbamodel_id": fbamodel_id,
"fba_output_id": fbamodel_id + ".mifba",
"fbamodel_workspace": workspace_name,
"media_id_list": new_media_list,
"target_reaction": "bio1",
"minimize_flux": 1
})
output = self.ws.get_objects2({
'objects': [{
'ref': workspace_name + "/" + fbamodel_id + '.mifba'
}]
})
#json.dump(output, open(self.scratch+'/fba.json', 'w'))
fba = output['data'][0]['data']
biomass_data = "FBAs,Biomass\n"
secretion_file = "," + ','.join(media_id_list) + "\n"
full_secretion_file = "," + ','.join(media_id_list) + "\n"
full_flux_file = "," + ','.join(media_id_list) + "\n"
flux_file = "," + ','.join(media_id_list) + "\n"
objectives = fba['other_objectives']
for i in range(0, len(objectives)):
biomass_data = biomass_data + media_id_list[i] + "," + str(
objectives[i]) + "\n"
flux_vars = fba['FBAReactionVariables']
for var in flux_vars:
id = var['modelreaction_ref'].split("/").pop()
flux_file = flux_file + id
full_flux_file = full_flux_file + id
fluxes = var['other_values']
for i in range(0, len(objectives)):
if objectives[i] == 0:
full_flux_file = full_flux_file + ",0"
flux_file = flux_file + ",0"
else:
full_flux_file = full_flux_file + "," + str(fluxes[i])
if abs(fluxes[i]) < 1e-7:
flux_file = flux_file + ",0"
else:
flux_file = flux_file + ",1"
flux_file = flux_file + "\n"
full_flux_file = full_flux_file + "\n"
secretion_vars = fba['FBACompoundVariables']
for var in secretion_vars:
id = var['modelcompound_ref'].split("/").pop()
secretion_file = secretion_file + id
full_secretion_file = full_secretion_file + id
fluxes = var['other_values']
for i in range(0, len(objectives)):
if objectives[i] == 0:
full_secretion_file = full_secretion_file + ",0"
secretion_file = secretion_file + ",0"
else:
full_secretion_file = full_secretion_file + "," + str(
fluxes[i])
if abs(fluxes[i]) < 1e-7:
secretion_file = secretion_file + ",0"
elif fluxes[i] < 0:
secretion_file = secretion_file + ",-1"
else:
secretion_file = secretion_file + ",1"
secretion_file = secretion_file + "\n"
full_secretion_file = full_secretion_file + "\n"
output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(output_directory)
biomass_path = os.path.join(output_directory, 'biomass.csv')
secretion_path = os.path.join(output_directory, 'secretion.csv')
flux_path = os.path.join(output_directory, 'flux.csv')
full_secretion_path = os.path.join(output_directory,
'full_secretion.csv')
full_flux_path = os.path.join(output_directory, 'full_flux.csv')
with open(biomass_path, 'w') as biomass_f:
biomass_f.write(biomass_data)
with open(secretion_path, 'w') as secretion_f:
secretion_f.write(secretion_file)
with open(flux_path, 'w') as flux_f:
flux_f.write(flux_file)
with open(full_secretion_path, 'w') as full_secretion_f:
full_secretion_f.write(full_secretion_file)
with open(full_flux_path, 'w') as full_flux_f:
full_flux_f.write(full_flux_file)
return [
biomass_path, secretion_path, flux_path, full_secretion_path,
full_flux_path
]
def _make_index_html(self, result_file_path, mutual_info_dict):
overview_content = ''
overview_content += '<table><tr><th>Mutual Information for various chemical compound combinations'
overview_content += ' Object</th></td>'
overview_content += '<tr><th>Input Chemical Compound Combination</th>'
overview_content += '<th>Mutual Information (in Bits)</th>'
overview_content += '</tr>'
for k, v in mutual_info_dict.items():
overview_content += '<tr><td>{}</td><td>{}</td></tr>'.format(k, v)
overview_content += '</table>'
with open(result_file_path, 'w') as result_file:
with open(
os.path.join(os.path.dirname(__file__),
'report_template.html'),
'r') as report_template_file:
report_template = report_template_file.read()
report_template = report_template.replace(
'<p>Overview_Content</p>', overview_content)
result_file.write(report_template)
return
def _generate_html_report(self, result_directory, mutual_info_dict):
"""
_generate_html_report: generate html summary report
"""
#scratch, uui, datafileutil, file_to_shock, shockId, extended report
log('start generating html report')
html_report = list()
output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(output_directory)
result_file_path = os.path.join(output_directory,
'mutual_information_report.html')
shutil.copy(os.path.join(result_directory, 'MI_plot.png'),
os.path.join(output_directory, 'MI_plot.png'))
overview_content = ''
overview_content += '<table><tr><th>Mutual Information for various chemical compound combinations'
overview_content += ' Object</th></td>'
overview_content += '<tr><th>Input Chemical Compound Combination</th>'
overview_content += '<th>Mutual Information (in Bits)</th>'
overview_content += '</tr>'
for k, v in mutual_info_dict.items():
overview_content += '<tr><td>{}</td><td>{}</td></tr>'.format(k, v)
overview_content += '</table>'
with open(result_file_path, 'w') as result_file:
with open(
os.path.join(os.path.dirname(__file__),
'report_template.html'),
'r') as report_template_file:
report_template = report_template_file.read()
report_template = report_template.replace(
'<p>Overview_Content</p>', overview_content)
result_file.write(report_template)
report_shock_id = self.dfu.file_to_shock({
'file_path': output_directory,
'pack': 'targz'
})['shock_id']
#report_shock_id = self.dfu.file_to_shock({'file_path': output_directory,
# 'pack': 'zip'})['shock_id']
html_report.append({
'shock_id':
report_shock_id,
'name':
os.path.basename(result_file_path),
'label':
os.path.basename(result_file_path),
'description':
'HTML summary report for Mutual Information App'
})
return html_report
def _generate_report(self, result_directory, mutual_info_dict,
workspace_name):
"""
_generate_report: generate summary report
"""
print('-->I am here *************')
uuidStr = str(uuid.uuid4())
output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(output_directory)
test_file = os.path.join(output_directory, "index.html")
self._make_index_html(test_file, mutual_info_dict[1])
#shutil.copy2(os.path.join(os.path.dirname(__file__), 'data', 'index.html'), output_directory)
# shutil.copy('/kb/module/data/index.html', result_directory + '/' + uuidStr + '/index.html')
json.dump(mutual_info_dict[0],
open(os.path.join(output_directory, 'pdata.json'), 'w'))
#shutil.copy('pdata.json', result_directory + '/' + uuidStr + '/pdata.json')
# DataFileUtils to shock
print(output_directory)
print(os.listdir(output_directory))
report_shock_result = self.dfu.file_to_shock({
'file_path': output_directory,
'pack': 'targz'
})
#report_shock_result = self.dfu.file_to_shock({'file_path': output_directory,
# 'pack': 'zip'})
report_shock_id = report_shock_result['shock_id']
print(report_shock_result)
report_file = {
'name': 'index.html',
'description': 'the report',
'shock_id': report_shock_id
}
log('creating report')
#output_html_files = self._generate_html_report(result_directory,
# mutual_info_dict)
report_params = {
'message': '',
'workspace_name': workspace_name,
'html_links': [report_file],
'file_links': [],
'direct_html_link_index': 0,
'html_window_height': 333,
'report_object_name': 'MutualInfomation_report_' + uuidStr
}
kbase_report_client = KBaseReport(self.callback_url)
output = kbase_report_client.create_extended_report(report_params)
report_output = {
'report_name': output['name'],
'report_ref': output['ref']
}
return report_output
######### @@@@@@@ALL THREE MUTUAL INFORMATION CALCULATION START FROM HERE@@@@@@@#############
def _generate_mutual_info(self, media_matrix, fba_file, mi_options):
#print('this is fba_file')
#print(fba_file)
df1 = pd.read_csv(fba_file[0])
df1.values
#df1.as_matrix()
#print('-->printing df1')# **** rm
#print(df1.to_string())# **** rm
#print(type(df1)) # **** rm
#print('-->printing media_matrix')
#print(media_matrix)
df3 = pd.DataFrame(columns=media_matrix[0][1:])
for i in range(1, len(media_matrix)):
if i == 1:
df3.loc[media_matrix[i][0][0]] = media_matrix[i][0][1:]
else:
df3.loc[media_matrix[i][0]] = media_matrix[i][1:]
#print('-->*************OK')
#print(df3)
#----Input validation of Media/FBAs with Binary Matrix FBAs------
# 1.0 Number of rows in Media.csv file = (Number of columns -1)
# 1.0. If they are different: Through an ERROR saying missed match number of FBAs in media and binary matrix.
# 1.1 Check whether the elements in Media.csv file contains only binary values (i.e. 0 and 1)
# 1.1. If the elements are different: Through an ERROR saying not approapriate input values
# 1.2 Check whether the compounds in Media.csv file match with number of FBAs
# 1.2. If the compounds are different from number of FBAs: Through an ERROR saying not appropriate input values
media_matrix = df3
s_df1 = df1.shape
s_df2 = media_matrix.shape
#print(media_matrix,type(media_matrix))
Temp_df2 = np.array(media_matrix.values)
#print('-->******')
#print(Temp_df2)
# Create matrix with only the elements remove first column and all the rows
Temp_df2 = Temp_df2[0:, 1:]
Bin_val_check = np.array_equal(Temp_df2, Temp_df2.astype(bool))
#num_compounds = (s_df2[1])-1
num_compounds = s_df2[1]
if ((s_df1[1] - 1) != s_df2[0]) or (Bin_val_check != True) or (int(
math.log(s_df2[0], 2)) != num_compounds):
print('invalid input values')
#-----All possible combination of the chemical compounds----------------------
# 2.0 Sperating m0 from rest of the lables
Temp1_df2 = media_matrix
#print('-->*************OK')
#print(Temp1_df2)
cols = Temp1_df2.columns
for i in range(0, len(cols)):
Temp1_df2.loc[Temp1_df2[cols[i]] == 1, cols[i]] = cols[i]
#print('-->*************OK')
#print (Temp1_df2)
# 2.1 Creating a disctionary for all FBAs except m0
#print(len(Temp1_df2))
#print('--->*********')
#print(Temp1_df2)
mydict = {}
for x in range(0, len(Temp1_df2)):
for i in range(0, s_df2[1]):
currentvalue = Temp1_df2.iloc[x, i]
currentid = Temp1_df2.index[x]
mydict.setdefault(currentid, [])
if currentvalue != 0:
mydict[currentid].append(currentvalue)
# Add the first key as m0
media_0_name = Temp1_df2.index[0]
mydict[media_0_name] = ["0"]
# Sort the keys
mydict = collections.OrderedDict(natsort.natsorted(mydict.items()))
#print ('--> ********')
compoundslist = Temp1_df2.columns.get_values()
compoundslist.tolist()
#print(compoundslist)
#print('all possible combination')
#print(len(compoundslist))
# List of Compounds combination in the list
my_combi_list = []
for L in range(0, len(compoundslist) + 1):
for subset in itertools.combinations(compoundslist, L):
my_combi_list.append(list(subset))
my_combi_list[0] = [0]
# print(my_combi_list)
'''
for k, v in mydict.iteritems():
#print('--> ********')
print(k, v)
'''
# Created a dictionary where the keys:
# list of compounds combination
# values are corresponding FBAs list in df2
result_dict = {}
for element in my_combi_list[1:]:
for k, v in mydict.iteritems():
if set(v).issubset(set(map(lambda x: str(x), element))):
key = ','.join(map(lambda x: str(x), element))
if result_dict.get(key):
media_list = result_dict[key]
media_list.append(k)
media_list = list(set(media_list))
result_dict.update({key: media_list})
else:
result_dict.update({key: [media_0_name, k]})
# Sort the keys
result_dict['0'] = [media_0_name]
result_dict = collections.OrderedDict(
natsort.natsorted(result_dict.items()))
# print(result_dict)
#print('-->I am here **** OK')
#print(result_dict)
#print (df1)
# Created a dictionary where the keys are:
# list of compounds combination
# values are compounds combination FBAs with df1 vaules
All_Comp_Combi_dic = {}
for column, value in result_dict.items():
All_Comp_Combi_dic.update({column: df1.get(value)})
# print('-->All_Comp_Combi_dic******')
# print (All_Comp_Combi_dic)
# print(result_dict)
# To print an item from the All_Comp_Combi_dic
df = (pd.DataFrame(All_Comp_Combi_dic.items()))
#print('--> printing df')
#print(df[0].to_string())
#print(df[1][7])
######### INTRACELLULAR FLUX MUTUAL INFORMATION CALCULATION #############
if mi_options == "flux":
print('Intracellular flux')
MI_dict = {}
for k in range(0, len(df[0])):
drop_rows_df = df[1][k].drop_duplicates(keep="first")
drop_columns_df = drop_rows_df.T.drop_duplicates(
keep="first").T
remove = []
removed = {}
count_values = {}
cols = df[1][k].columns
for i in range(len(cols) - 1):
duplicated = []
v = df[1][k][cols[i]].values
for j in range(i + 1, len(cols)):
if np.array_equal(v, df[1][k][cols[j]].values):
remove.append(cols[j])
duplicated.append(cols[j])
if duplicated and cols[i] not in remove:
removed.update({cols[i]: duplicated})
count = {}
for key, value in removed.items():
count.update({key: len(value)})
#print v
# print drop_columns_df
count_values = count.values()
# print count_values
count_values = map(lambda x: x + 1, count_values)
# print count_values
d = {x: count_values.count(x) for x in count_values}
#print('-->count_values')
#print(count_values)
#-------Mutual Inforamtion (MI) calculation-------------
FBAs = len(df[1][k].columns)
pure_entropy = math.log(FBAs, 2)
#print (pure_entropy) (-->ok rm)
# If No duplicates exist and list "value" is empty
if not count_values:
#print("List is empty")
No_duplicate_FBAs = len(drop_columns_df.columns)
conditional_entropy = -1 * (No_duplicate_FBAs * (
(1 / No_duplicate_FBAs) *
((1 / 1) * math.log(1.0 / 1.0, 2))))
Mutual_Info = pure_entropy - conditional_entropy
#print('Mutaul Info:', Mutual_Info)
if count_values:
# If duplicates exist and list "value" is not empty
conditional_entropy = 0
for key in d:
#print key, d[key]
Temp = -1 * d[key] * (key / float(FBAs)) * key * (
1.0 / key) * math.log(1.0 / key, 2)
conditional_entropy = Temp + conditional_entropy
#print "%3f" %Temp
Mutual_Info = pure_entropy - conditional_entropy
MI_dict[df[0][k]] = Mutual_Info
MI_dict['0'] = 0.0
#Sorted MI_dict
MI_dict = sorted(MI_dict.items(), key=lambda x: (-len(x[0]), x[0]))
MI_dict = OrderedDict(MI_dict)
#print(MI_dict)
#print('-->rest')
#print(compoundslist)
#print(num_compounds)
x_groups = [[] for x in range(num_compounds)]
y_groups = [[] for x in range(num_compounds)]
names = [[] for x in range(num_compounds)]
Comp_Mapping = [[] for x in range(num_compounds)]
for key, val in MI_dict.iteritems():
del_count = key.count(',')
x_groups[del_count].append(key)
y_groups[del_count].append(val)
# for x, y in zip(x_groups, y_groups):
# data.append(go.Bar(x=x, y=y, name='test'))
pdata = []
for i in range(0, len(x_groups)):
names[i] = str(i + 1) + ' Compound Combination'
Comp_Mapping = str(i + 1) + '-' + compoundslist[i]
record = {}
record["x"] = []
for e in x_groups[i]:
record["x"].append("c" + e)
record["y"] = y_groups[i]
record["names"] = names[i]
record["Comp_Mapping"] = Comp_Mapping
pdata.append(record)
#print (pdata)
#json.dump(pdata, open(self.scratch+'/pdata.json', 'w'))
return [pdata, MI_dict]
#return MI_dict
######### INPUT COMPONENTS AND BIOMASS FLUX MUTUAL INFORMATION CALCULATION #############
if mi_options == "biomass":
# Load the file contain the information of FBAs(media) along with corresponding Biomass (growth)
print('biomass flux')
df2 = pd.read_csv(fba_file[1])
df2.values
#print(df)
MI_dict_biomass = {}
for r in range(0, len(df[0])):
reaction_states = df[1][r].head(1000)
def get_groups(flux_df):
groups = collections.defaultdict(list)
unique = flux_df.aggregate(lambda x: hash(str(x.values)))
for k, v in unique[0:].iteritems():
groups[v].append(k)
return dict([(i, g)
for i, g in enumerate(groups.values())])
n_group = collections.defaultdict(int)
groups = get_groups(reaction_states)
for group in groups.values():
n_group[len(group)] += 1
groups_count = {}
for key, values in groups.items():
groups_count[key] = len(values)
# print groups_count
# Take first FBA label of every group
group_id = {}
for k, v in groups.items():
group_id.update({k: groups.values()[k][0]})
# Obtain the Biomass of each Group
cols_df = group_id.values()
cols_df2 = df2.columns
#print (cols_df)
# Dictionary of first FBA label of every group and its corresponding number of members
groups_label_count = {}
for k, v in groups_count.items():
groups_label_count.update({cols_df[k]: v})
#print('groups_label_count')
#print(groups_label_count)
def get_cond_count(re_group):
media_cond = 0
for media in re_group['FBAs']:
media_cond += groups_label_count[media]
return media_cond
# Extract FBA Groups biomass inside df2
Groups_Biomass = df2[df2['FBAs'].isin(cols_df)]
#print('-->I am here')
#print(Groups_Biomass)
# Regroup based on the biomass values
re_group = Groups_Biomass.groupby('Biomass')
biomass_FBAs_groups = re_group.aggregate(get_cond_count)
biomass_FBAs_label_groups = Groups_Biomass.groupby(
"Biomass", sort=True).sum()
#print(biomass_FBAs_label_groups)
#print (biomass_FBAs_label_groups)
Summery = pd.merge(left=biomass_FBAs_label_groups,
left_index=True,
right=biomass_FBAs_groups,
right_index=True,
how='inner')
Data_4_CondMI = Summery.groupby('FBAs_y').count()
Data_4_CondMI = Data_4_CondMI.to_dict(orient='dict')
for k, v in Data_4_CondMI.items():
Data_4_CondMI = v
Num_of_FBAs = Data_4_CondMI.keys()
Count_Num_of_FBAs = Data_4_CondMI.values()
# -------Mutual Inforamtion (MI) calculation Stage II (input compounds respect to BIOMASS-------------
# Pure Entropy
FBAs = len(df[1][r].columns)
pure_entropy = math.log(FBAs, 2)
conditional_entropy = 0.0
for l in range(0, len(Count_Num_of_FBAs)):
temp = -1 * Count_Num_of_FBAs[l] * (
Num_of_FBAs[l] / float(FBAs)) * Num_of_FBAs[l] * (
1.0 / float(Num_of_FBAs[l]) *
(math.log(1.0 / float(Num_of_FBAs[l]), 2)))
conditional_entropy += temp
Mutual_Info_Biomass = pure_entropy - conditional_entropy
# print('Mutaul Info:', Mutual_Info_Biomass)
#print(Mutual_Info_Biomass)
MI_dict_biomass.update({df[0][r]: Mutual_Info_Biomass})
#print(MI_dict_biomass)
# Sorted MI_dict_biomass
MI_dict_biomass = sorted(MI_dict_biomass.items(),
key=lambda x: (-len(x[0]), x[0]))
MI_dict_biomass = OrderedDict(MI_dict_biomass)
#print(MI_dict_biomass)
x_groups = [[] for x in range(num_compounds)]
y_groups = [[] for x in range(num_compounds)]
names = [[] for x in range(num_compounds)]
Comp_Mapping = [[] for x in range(num_compounds)]
for key, val in MI_dict_biomass.iteritems():
del_count = key.count(',')
x_groups[del_count].append(key)
y_groups[del_count].append(val)
pdata = []
for i in range(0, len(x_groups)):
names[i] = str(i + 1) + ' Compound Combination'
Comp_Mapping = str(i + 1) + '-' + compoundslist[i]
record = {}
record["x"] = []
for e in x_groups[i]:
record["x"].append("c" + e)
record["y"] = y_groups[i]
record["names"] = names[i]
record["Comp_Mapping"] = Comp_Mapping
pdata.append(record)
return [pdata, MI_dict_biomass]
######### INPUT COMPONENTS AND BIOMASS, SECRETION FLUX MUTUAL INFORMATION CALCULATION #############
if mi_options == "secretion":
#Load the file contain the information of FBAs(media) along with corresponding Biomass (growth)
print('secretion flux')
df4 = pd.read_csv(fba_file[2], header=0, index_col=0)
df4.index.name = 'FBAs'
df4 = df4.T
dfbiomass = pd.read_csv(fba_file[1])
aa = dfbiomass['Biomass'].values.tolist()
# print(len(aa))
df4['Biomass'] = aa
# print(df4.shape)
compoundslist_b_u_s = list(df4.columns.values)
#print(compoundslist_b_u_s)
MI_dict_b_u_s = {}
for r in range(0, len(df[0])):
reaction_states = df[1][r].head(1000)
def get_groups(flux_df):
groups = collections.defaultdict(list)
unique = flux_df.aggregate(lambda x: hash(str(x.values)))
for k, v in unique[0:].iteritems():
groups[v].append(k)
return dict([(i, g)
for i, g in enumerate(groups.values())])
n_group = collections.defaultdict(int)
groups = get_groups(reaction_states)
for group in groups.values():
n_group[len(group)] += 1
#print(n_group)
#print(groups)
groups_count = {}
for key, values in groups.items():
groups_count[key] = len(values)
# print(groups_count)
# Take first FBA label of every group
group_id = {}
for k, v in groups.items():
group_id.update({k: groups.values()[k][0]})
# Obtain the Biomass of each Group
cols_df = group_id.values()
cols_df4 = df4.columns
# Dictionary of first FBA label of every group and its corresponding number of members
groups_label_count = {}
for k, v in groups_count.items():
groups_label_count.update({cols_df[k]: v})
#print(groups_label_count)
# Extract FBA Groups biomass inside df4
df5 = df4.reset_index()
Groups_Biomass = df5[df5['index'].isin(cols_df)]
#print(Groups_Biomass)
# Regroup based on the biomass values
re_group = Groups_Biomass.groupby(compoundslist_b_u_s)
#print(re_group)
my_list = []
for index, values in re_group:
my_list.append(values['index'].values)
#print(my_list)
B_U_S_dict = {}
for media in my_list:
if len(media) > 1:
media_cond = 0
for i in (0, len(media) - 1):
media_cond += groups_label_count[media[i]]
B_U_S_dict.update({str(media)[1:-1]: media_cond})
#final_my_dict.update({tuple(media.tolist()):media_cond})
else:
B_U_S_dict.update({
str(media)[1:-1]:
groups_label_count[str(tuple(
media.tolist()))[1:-1][:-1][1:-1]]
})
B_U_S_dict = {k: v for k, v in B_U_S_dict.iteritems()}
#print(B_U_S_dict)
Summery = pd.DataFrame(B_U_S_dict.items(),
columns=['index_x', 'index_y'])
Data_4_CondMI = Summery.groupby('index_y').count()
Data_4_CondMI = Data_4_CondMI.to_dict(orient='dict')
#print(Data_4_CondMI)
for k, v in Data_4_CondMI.items():
Data_4_CondMI = v
Num_of_FBAs = Data_4_CondMI.keys()
Count_Num_of_FBAs = Data_4_CondMI.values()
#print(Num_of_FBAs)
#print(Count_Num_of_FBAs)
#print('-->***<---')
# -------Mutual Inforamtion (MI) calculation Stage II (input compounds respect to Biomass, Uptake and Secretion-------------
# Pure Entropy
FBAs = len(df[1][r].columns)
pure_entropy = math.log(FBAs, 2)
conditional_entropy = 0.0
for l in range(0, len(Count_Num_of_FBAs)):
temp = -1 * Count_Num_of_FBAs[l] * (
Num_of_FBAs[l] / float(FBAs)) * Num_of_FBAs[l] * (
1.0 / float(Num_of_FBAs[l]) *
(math.log(1.0 / float(Num_of_FBAs[l]), 2)))
conditional_entropy += temp
Mutual_Info_B_U_S = pure_entropy - conditional_entropy
# print('Mutaul Info:', Mutual_Info_B_U_S)
MI_dict_b_u_s.update({df[0][r]: Mutual_Info_B_U_S})
# Sorted MI_dict_biomass
MI_dict_b_u_s = sorted(MI_dict_b_u_s.items(),
key=lambda x: (-len(x[0]), x[0]))
MI_dict_b_u_s = OrderedDict(MI_dict_b_u_s)
#print(MI_dict_b_u_s)
x_groups = [[] for x in range(num_compounds)]
y_groups = [[] for x in range(num_compounds)]
names = [[] for x in range(num_compounds)]
Comp_Mapping = [[] for x in range(num_compounds)]
for key, val in MI_dict_b_u_s.iteritems():
del_count = key.count(',')
x_groups[del_count].append(key)
y_groups[del_count].append(val)
# for x, y in zip(x_groups, y_groups):
# data.append(go.Bar(x=x, y=y, name='test'))
pdata = []
for i in range(0, len(x_groups)):
names[i] = str(i + 1) + ' Compound Combination'
Comp_Mapping = str(i + 1) + '-' + compoundslist[i]
record = {}
record["x"] = []
for e in x_groups[i]:
record["x"].append("c" + e)
record["y"] = y_groups[i]
record["names"] = names[i]
record["Comp_Mapping"] = Comp_Mapping
pdata.append(record)
return [pdata, MI_dict_b_u_s]
class TaxonAPI:
'''
Module Name:
TaxonAPI
Module Description:
A KBase module: TaxonAPI
'''
######## WARNING FOR GEVENT USERS ####### noqa
# Since asynchronous IO can lead to methods - even the same method -
# interrupting each other, you must be *very* careful when using global
# state. A method could easily clobber the state set by another while
# the latter method is running.
######################################### noqa
VERSION = "1.0.2"
GIT_URL = ""
GIT_COMMIT_HASH = "5b8cdf63a676a609ea4f180891cf75006640f148"
#BEGIN_CLASS_HEADER
_GENOME_TYPES = ['KBaseGenomes.Genome',
'KBaseGenomeAnnotations.GenomeAnnotation']
_TAXON_TYPES = ['KBaseGenomeAnnotations.Taxon']
@functools32.lru_cache(maxsize=1000)
def get_object(self, ref):
res = self.ws.get_objects2({'objects': [{'ref': ref}]})['data'][0]
return res
def get_data(self, ref):
obj = self.get_object(ref)
return obj['data']
@functools32.lru_cache(maxsize=1000)
def translate_to_MD5_types(self, ktype):
return self.ws.translate_to_MD5_types([ktype]).values()[0]
def get_referrers(self, ref):
referrers = self.ws.list_referencing_objects(
[{"ref": ref}])[0]
object_refs_by_type = dict()
tlist = []
for x in referrers:
tlist.append(x[2])
typemap = self.ws.translate_to_MD5_types(tlist)
for x in referrers:
typestring = typemap[x[2]]
if typestring not in object_refs_by_type:
object_refs_by_type[typestring] = list()
upa = '%d/%d/%d' % (x[6], x[0], x[4])
object_refs_by_type[typestring].append(upa)
return object_refs_by_type
def get_reffers_type(self, ref, types):
referrers = self.get_referrers(ref)
children = list()
for object_type in referrers:
if object_type.split('-')[0] in types:
children.extend(referrers[object_type])
return children
def make_hash(self, i):
omd = i[10]
if i[10] == {}:
omd = None
return {
'type_string': i[2],
'workspace_id': i[6],
'object_checksum': i[8],
'object_reference': '%d/%d' % (i[6], i[0]),
'object_size': i[9],
'saved_by': i[5],
'object_id': i[0],
'save_date': i[3],
'object_metadata': omd,
'object_name': i[1],
'version': i[4],
'workspace_name': i[7],
'object_reference_versioned': '%d/%d/%d' % (i[6], i[0], i[4])
}
#END_CLASS_HEADER
# config contains contents of config file in a hash or None if it couldn't
# be found
def __init__(self, config):
#BEGIN_CONSTRUCTOR
self.workspaceURL = config['workspace-url']
self.ws = Workspace(self.workspaceURL)
self.shockURL = config['shock-url']
self.logger = logging.getLogger()
log_handler = logging.StreamHandler()
log_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
self.logger.addHandler(log_handler)
#END_CONSTRUCTOR
pass
def get_parent(self, ctx, ref):
"""
Retrieve parent Taxon.
@return Reference to parent Taxon.
:param ref: instance of type "ObjectReference"
:returns: instance of type "ObjectReference"
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_parent
data = self.get_data(ref)
try:
returnVal = data['parent_taxon_ref']
# returnVal=taxon_api.get_parent(ref_only=True)
except:
returnVal = ''
#END get_parent
# At some point might do deeper type checking...
if not isinstance(returnVal, basestring):
raise ValueError('Method get_parent return value ' +
'returnVal is not type basestring as required.')
# return the results
return [returnVal]
def get_children(self, ctx, ref):
"""
Retrieve children Taxon.
@return List of references to child Taxons.
:param ref: instance of type "ObjectReference"
:returns: instance of list of type "ObjectReference"
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_children
returnVal = self.get_reffers_type(ref, self._TAXON_TYPES)
#END get_children
# At some point might do deeper type checking...
if not isinstance(returnVal, list):
raise ValueError('Method get_children return value ' +
'returnVal is not type list as required.')
# return the results
return [returnVal]
def get_genome_annotations(self, ctx, ref):
"""
funcdef GenomeAnnotation(s) that refer to this Taxon.
If this is accessing a KBaseGenomes.Genome object, it will
return an empty list (this information is not available).
@return List of references to GenomeAnnotation objects.
:param ref: instance of type "ObjectReference"
:returns: instance of list of type "ObjectReference"
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_genome_annotations
returnVal = self.get_reffers_type(ref, self._GENOME_TYPES)
#END get_genome_annotations
# At some point might do deeper type checking...
if not isinstance(returnVal, list):
raise ValueError('Method get_genome_annotations return value ' +
'returnVal is not type list as required.')
# return the results
return [returnVal]
def get_scientific_lineage(self, ctx, ref):
"""
Retrieve the scientific lineage.
@return Strings for each 'unit' of the lineage, ordered in
the usual way from Domain to Kingdom to Phylum, etc.
:param ref: instance of type "ObjectReference"
:returns: instance of list of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_scientific_lineage
o = self.ws.get_objects2({'objects': [{'ref': ref}]})['data'][0]['data']
returnVal = [x.strip() for x in o['scientific_lineage'].split(";")]
#END get_scientific_lineage
# At some point might do deeper type checking...
if not isinstance(returnVal, list):
raise ValueError('Method get_scientific_lineage return value ' +
'returnVal is not type list as required.')
# return the results
return [returnVal]
def get_scientific_name(self, ctx, ref):
"""
Retrieve the scientific name.
@return The scientific name, e.g., "Escherichia Coli K12 str. MG1655"
:param ref: instance of type "ObjectReference"
:returns: instance of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_scientific_name
obj = self.ws.get_objects2({'objects': [{'ref': ref}]})['data'][0]['data']
returnVal = obj['scientific_name']
#END get_scientific_name
# At some point might do deeper type checking...
if not isinstance(returnVal, basestring):
raise ValueError('Method get_scientific_name return value ' +
'returnVal is not type basestring as required.')
# return the results
return [returnVal]
def get_taxonomic_id(self, ctx, ref):
"""
Retrieve the NCBI taxonomic ID of this Taxon.
For type KBaseGenomes.Genome, the ``source_id`` will be returned.
@return Integer taxonomic ID.
:param ref: instance of type "ObjectReference"
:returns: instance of Long
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_taxonomic_id
obj = self.ws.get_objects2({'objects': [{'ref': ref}]})['data'][0]['data']
returnVal = obj['taxonomy_id']
#END get_taxonomic_id
# At some point might do deeper type checking...
if not isinstance(returnVal, int):
raise ValueError('Method get_taxonomic_id return value ' +
'returnVal is not type int as required.')
# return the results
return [returnVal]
def get_kingdom(self, ctx, ref):
"""
Retrieve the kingdom.
:param ref: instance of type "ObjectReference"
:returns: instance of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_kingdom
obj = self.ws.get_objects2({'objects': [{'ref': ref}]})['data'][0]['data']
returnVal = obj['kingdom']
#END get_kingdom
# At some point might do deeper type checking...
if not isinstance(returnVal, basestring):
raise ValueError('Method get_kingdom return value ' +
'returnVal is not type basestring as required.')
# return the results
return [returnVal]
def get_domain(self, ctx, ref):
"""
Retrieve the domain.
:param ref: instance of type "ObjectReference"
:returns: instance of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_domain
obj = self.ws.get_objects2({'objects': [{'ref': ref}]})['data'][0]['data']
returnVal = obj['domain']
#END get_domain
# At some point might do deeper type checking...
if not isinstance(returnVal, basestring):
raise ValueError('Method get_domain return value ' +
'returnVal is not type basestring as required.')
# return the results
return [returnVal]
def get_genetic_code(self, ctx, ref):
"""
Retrieve the genetic code.
:param ref: instance of type "ObjectReference"
:returns: instance of Long
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_genetic_code
obj = self.ws.get_objects2({'objects': [{'ref': ref}]})['data'][0]['data']
returnVal = obj['genetic_code']
#END get_genetic_code
# At some point might do deeper type checking...
if not isinstance(returnVal, int):
raise ValueError('Method get_genetic_code return value ' +
'returnVal is not type int as required.')
# return the results
return [returnVal]
def get_aliases(self, ctx, ref):
"""
Retrieve the aliases.
:param ref: instance of type "ObjectReference"
:returns: instance of list of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_aliases
obj = self.ws.get_objects2({'objects': [{'ref': ref}]})['data'][0]['data']
if 'aliases' in obj:
returnVal = obj['aliases']
else:
returnVal = list()
#END get_aliases
# At some point might do deeper type checking...
if not isinstance(returnVal, list):
raise ValueError('Method get_aliases return value ' +
'returnVal is not type list as required.')
# return the results
return [returnVal]
def get_info(self, ctx, ref):
"""
Retrieve object info.
@skip documentation
:param ref: instance of type "ObjectReference"
:returns: instance of type "ObjectInfo" (* @skip documentation) ->
structure: parameter "object_id" of Long, parameter "object_name"
of String, parameter "object_reference" of String, parameter
"object_reference_versioned" of String, parameter "type_string" of
String, parameter "save_date" of String, parameter "version" of
Long, parameter "saved_by" of String, parameter "workspace_id" of
Long, parameter "workspace_name" of String, parameter
"object_checksum" of String, parameter "object_size" of Long,
parameter "object_metadata" of mapping from String to String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_info
# returnVal = self.ws.get_objects2({'objects': [{'ref': ref}]})['data'][0]['info']
i = self.get_object(ref)['info']
#md5_typestr = self.ws.translate_to_MD5_types([i[2]]).values()[0]
returnVal = self.make_hash(i)
#END get_info
# At some point might do deeper type checking...
if not isinstance(returnVal, dict):
raise ValueError('Method get_info return value ' +
'returnVal is not type dict as required.')
# return the results
return [returnVal]
def get_history(self, ctx, ref):
"""
Retrieve object history.
@skip documentation
:param ref: instance of type "ObjectReference"
:returns: instance of type "ObjectHistory" (* @skip documentation) ->
list of type "ObjectInfo" (* @skip documentation) -> structure:
parameter "object_id" of Long, parameter "object_name" of String,
parameter "object_reference" of String, parameter
"object_reference_versioned" of String, parameter "type_string" of
String, parameter "save_date" of String, parameter "version" of
Long, parameter "saved_by" of String, parameter "workspace_id" of
Long, parameter "workspace_name" of String, parameter
"object_checksum" of String, parameter "object_size" of Long,
parameter "object_metadata" of mapping from String to String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_history
# returnVal = self.ws.get_object_history({'ref': ref})
returnVal = []
for i in self.ws.get_object_history({'ref': ref}):
returnVal.append(self.make_hash(i))
#END get_history
# At some point might do deeper type checking...
if not isinstance(returnVal, list):
raise ValueError('Method get_history return value ' +
'returnVal is not type list as required.')
# return the results
return [returnVal]
def get_provenance(self, ctx, ref):
"""
Retrieve object provenance.
@skip documentation
:param ref: instance of type "ObjectReference"
:returns: instance of type "ObjectProvenance" (* @skip documentation)
-> list of type "ObjectProvenanceAction" (* @skip documentation)
-> structure: parameter "time" of String, parameter "service_name"
of String, parameter "service_version" of String, parameter
"service_method" of String, parameter "method_parameters" of list
of String, parameter "script_name" of String, parameter
"script_version" of String, parameter "script_command_line" of
String, parameter "input_object_references" of list of String,
parameter "validated_object_references" of list of String,
parameter "intermediate_input_ids" of list of String, parameter
"intermediate_output_ids" of list of String, parameter
"external_data" of list of type "ExternalDataUnit" (* @skip
documentation) -> structure: parameter "resource_name" of String,
parameter "resource_url" of String, parameter "resource_version"
of String, parameter "resource_release_date" of String, parameter
"data_url" of String, parameter "data_id" of String, parameter
"description" of String, parameter "description" of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_provenance
prov = self.ws.get_object_provenance([{"ref": ref}])[0]['provenance']
returnVal = []
copy_keys = {"time": "time",
"service": "service_name",
"service_ver": "service_version",
"method": "service_method",
"method_params": "method_parameters",
"script": "script_name",
"script_ver": "script_version",
"script_command_line": "script_command_line",
"input_ws_objects": "input_object_references",
"resolved_ws_objects": "validated_object_references",
"intermediate_incoming": "intermediate_input_ids",
"intermediate_outgoing": "intermediate_output_ids",
"external_data": "external_data",
"description": "description"
}
for object_provenance in prov:
action = dict()
for k in copy_keys:
if k in object_provenance:
if isinstance(object_provenance[k], list) and len(object_provenance[k]) == 0:
continue
action[copy_keys[k]] = object_provenance[k]
returnVal.append(action)
#END get_provenance
# At some point might do deeper type checking...
if not isinstance(returnVal, list):
raise ValueError('Method get_provenance return value ' +
'returnVal is not type list as required.')
# return the results
return [returnVal]
def get_id(self, ctx, ref):
"""
Retrieve object identifier.
@skip documentation
:param ref: instance of type "ObjectReference"
:returns: instance of Long
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_id
returnVal = self.get_object(ref)['info'][0]
#END get_id
# At some point might do deeper type checking...
if not isinstance(returnVal, int):
raise ValueError('Method get_id return value ' +
'returnVal is not type int as required.')
# return the results
return [returnVal]
def get_name(self, ctx, ref):
"""
Retrieve object name.
@skip documentation
:param ref: instance of type "ObjectReference"
:returns: instance of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_name
returnVal = self.get_object(ref)['info'][1]
#END get_name
# At some point might do deeper type checking...
if not isinstance(returnVal, basestring):
raise ValueError('Method get_name return value ' +
'returnVal is not type basestring as required.')
# return the results
return [returnVal]
def get_version(self, ctx, ref):
"""
Retrieve object version.
@skip documentation
:param ref: instance of type "ObjectReference"
:returns: instance of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_version
returnVal = str(self.get_object(ref)['info'][4])
#END get_version
# At some point might do deeper type checking...
if not isinstance(returnVal, basestring):
raise ValueError('Method get_version return value ' +
'returnVal is not type basestring as required.')
# return the results
return [returnVal]
def get_all_data(self, ctx, params):
"""
:param params: instance of type "GetAllDataParams" -> structure:
parameter "ref" of type "ObjectReference", parameter
"include_decorated_scientific_lineage" of type "boolean" (A
boolean. 0 = false, other = true.), parameter
"include_decorated_children" of type "boolean" (A boolean. 0 =
false, other = true.), parameter "exclude_children" of type
"boolean" (A boolean. 0 = false, other = true.)
:returns: instance of type "TaxonData" -> structure: parameter
"parent" of type "ObjectReference", parameter "children" of list
of type "ObjectReference", parameter "decorated_children" of list
of type "TaxonInfo" -> structure: parameter "ref" of type
"ObjectReference", parameter "scientific_name" of String,
parameter "scientific_lineage" of list of String, parameter
"decorated_scientific_lineage" of list of type "TaxonInfo" ->
structure: parameter "ref" of type "ObjectReference", parameter
"scientific_name" of String, parameter "scientific_name" of
String, parameter "taxonomic_id" of Long, parameter "kingdom" of
String, parameter "domain" of String, parameter "genetic_code" of
Long, parameter "aliases" of list of String, parameter "obj_info"
of type "ObjectInfo" (* @skip documentation) -> structure:
parameter "object_id" of Long, parameter "object_name" of String,
parameter "object_reference" of String, parameter
"object_reference_versioned" of String, parameter "type_string" of
String, parameter "save_date" of String, parameter "version" of
Long, parameter "saved_by" of String, parameter "workspace_id" of
Long, parameter "workspace_name" of String, parameter
"object_checksum" of String, parameter "object_size" of Long,
parameter "object_metadata" of mapping from String to String
"""
# ctx is the context object
# return variables are: d
#BEGIN get_all_data
d = {}
ref = params['ref']
obj = self.get_object(ref)
data = obj['data']
try:
d['parent'] = data['parent_taxon_ref']
except KeyError:
print('Error getting parent for ' + ref)
# +':\n'+ str(traceback.format_exc()))
d['parent'] = None
if 'exclude_children' in params and params['exclude_children'] == 1:
pass
else:
d['children'] = self.get_reffers_type(ref, self._TAXON_TYPES)
d['scientific_lineage'] = data['scientific_lineage']
d['scientific_name'] = data['scientific_name']
d['taxonomic_id'] = data['taxonomy_id']
try:
d['kingdom'] = data['kingdom']
# throws error if not found, so catch and log it
except KeyError:
print('Error getting kingdom for ' + ref)
# +':\n'+ str(traceback.format_exc()))
d['kingdom'] = None
d['domain'] = data['domain']
d['genetic_code'] = data['genetic_code']
d['aliases'] = None
if 'aliases' in data:
d['aliases'] = data['aliases']
d['info'] = self.make_hash(obj['info'])
key = 'include_decorated_scientific_lineage'
if key in params and params[key] == 1:
l = self.get_decorated_scientific_lineage(ctx, {'ref': ref})[0]
d['decorated_scientific_lineage'] = l['decorated_scientific_lineage']
key = 'include_decorated_children'
if key in params and params[key] == 1:
l = self.get_decorated_children(ctx, {'ref': ref})[0]
d['decorated_children'] = l['decorated_children']
#END get_all_data
# At some point might do deeper type checking...
if not isinstance(d, dict):
raise ValueError('Method get_all_data return value ' +
'd is not type dict as required.')
# return the results
return [d]
def get_decorated_scientific_lineage(self, ctx, params):
"""
:param params: instance of type "GetDecoratedScientificLineageParams"
-> structure: parameter "ref" of type "ObjectReference"
:returns: instance of type "DecoratedScientificLineage" (list starts
at the root, and goes on down to this) -> structure: parameter
"decorated_scientific_lineage" of list of type "TaxonInfo" ->
structure: parameter "ref" of type "ObjectReference", parameter
"scientific_name" of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_decorated_scientific_lineage
lineageList = []
ref = params['ref']
while True:
parent_data = None
try:
# note: doesn't look like there is a way to get a reference
# of a Taxon directly (without constructing it from
# object_info), so first get reference, then instantiate
# another API object
parent_ref = self.get_data(ref)['parent_taxon_ref']
if parent_ref is not None:
data = self.get_data(ref)
scientific_name = data['scientific_name']
if scientific_name != 'root':
parent_data = {
'ref': parent_ref,
'scientific_name': scientific_name
}
ref = parent_ref
except KeyError:
# case where parent is not found
pass
if parent_data is not None:
lineageList.append(parent_data)
else:
break
lineageList.reverse() # reverse list to match scientific_lineage style
returnVal = {'decorated_scientific_lineage': lineageList[:-1]}
#END get_decorated_scientific_lineage
# At some point might do deeper type checking...
if not isinstance(returnVal, dict):
raise ValueError('Method get_decorated_scientific_lineage return value ' +
'returnVal is not type dict as required.')
# return the results
return [returnVal]
def get_decorated_children(self, ctx, params):
"""
:param params: instance of type "GetDecoratedChildrenParams" ->
structure: parameter "ref" of type "ObjectReference"
:returns: instance of type "DecoratedChildren" -> structure:
parameter "decorated_children" of list of type "TaxonInfo" ->
structure: parameter "ref" of type "ObjectReference", parameter
"scientific_name" of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_decorated_children
ref = params['ref']
children_refs = self.get_reffers_type(ref, self._TAXON_TYPES)
decorated_children = []
for child_ref in children_refs:
decorated_children.append({
'ref': child_ref,
'scientific_name': self.get_data(child_ref)['scientific_name']
})
returnVal = {'decorated_children': decorated_children}
#END get_decorated_children
# At some point might do deeper type checking...
if not isinstance(returnVal, dict):
raise ValueError('Method get_decorated_children return value ' +
'returnVal is not type dict as required.')
# return the results
return [returnVal]
def status(self, ctx):
#BEGIN_STATUS
returnVal = {'state': "OK", 'message': "", 'version': self.VERSION,
'git_url': self.GIT_URL, 'git_commit_hash': self.GIT_COMMIT_HASH}
#END_STATUS
return [returnVal]
class CompoundSetUtils:
'''
Module Name:
CompoundSetUtils
Module Description:
A KBase module: CompoundSetUtils
Contains tools for import & export of compound sets
'''
######## WARNING FOR GEVENT USERS ####### noqa
# Since asynchronous IO can lead to methods - even the same method -
# interrupting each other, you must be *very* careful when using global
# state. A method could easily clobber the state set by another while
# the latter method is running.
######################################### noqa
VERSION = "0.0.1"
GIT_URL = "https://github.com/kbaseapps/CompoundSetUtils.git"
GIT_COMMIT_HASH = "53bac077a8efaaea9ead90d5557b1af1c0b23394"
#BEGIN_CLASS_HEADER
@staticmethod
def _check_required_param(in_params, param_list):
"""
Check if each of the params in the list are in the input params
"""
for param in param_list:
if param not in in_params or not in_params[param]:
raise ValueError('{} parameter is required'.format(param))
def _save_to_ws_and_report(self, ctx, method, workspace, source, compoundset):
"""Save compound set to the workspace and make report"""
provenance = [{}]
if 'provenance' in ctx:
provenance = ctx['provenance']
if 'model' in method:
provenance[0]['input_ws_objects'] = workspace + '/' + source
provenance[0]['service'] = 'CompoundSetUtils'
provenance[0]['method'] = method
info = self.ws_client.save_objects(
{'workspace': workspace,
"objects": [{
"type": "KBaseBiochem.CompoundSet",
"data": compoundset,
"name": compoundset['name']
}]})[0]
compoundset_ref = "%s/%s/%s" % (info[6], info[0], info[4])
report_params = {
'objects_created': [{'ref': compoundset_ref,
'description': 'Compound Set'}],
'message': 'Imported %s as %s' % (source, compoundset_ref),
'workspace_name': workspace,
'report_object_name': 'compound_set_creation_report'
}
# Construct the output to send back
report_client = KBaseReport(self.callback_url)
report_info = report_client.create_extended_report(report_params)
output = {'report_name': report_info['name'],
'report_ref': report_info['ref'],
'compoundset_ref': compoundset_ref}
return output
#END_CLASS_HEADER
# config contains contents of config file in a hash or None if it couldn't
# be found
def __init__(self, config):
#BEGIN_CONSTRUCTOR
self.config = config
self.scratch = config['scratch']
self.callback_url = os.environ['SDK_CALLBACK_URL']
self.ws_url = config['workspace-url']
self.ws_client = Workspace(self.ws_url)
self.dfu = DataFileUtil(self.callback_url)
#END_CONSTRUCTOR
pass
def compound_set_from_file(self, ctx, params):
"""
CompoundSetFromFile
string staging_file_path
:param params: instance of type "compoundset_upload_params" ->
structure: parameter "workspace_name" of String, parameter
"staging_file_path" of String, parameter "compound_set_name" of
String
:returns: instance of type "compoundset_upload_results" -> structure:
parameter "report_name" of String, parameter "report_ref" of
String, parameter "compoundset_ref" of type "obj_ref"
"""
# ctx is the context object
# return variables are: output
#BEGIN compound_set_from_file
self._check_required_param(params, ['workspace_name',
'staging_file_path',
'compound_set_name'])
scratch_file_path = self.dfu.download_staging_file(
{'staging_file_subdir_path': params['staging_file_path']}
).get('copy_file_path')
# I probably should be uploading the raw files to shock
ext = os.path.splitext(scratch_file_path)[1]
file_name = os.path.basename(scratch_file_path)
if ext == '.sdf':
compounds = parse.read_sdf(scratch_file_path)
elif ext == '.tsv':
compounds = parse.read_tsv(scratch_file_path)
else:
raise ValueError('Invalid input file type. Expects .tsv or .sdf')
compoundset = {
'id': params['compound_set_name'],
'name': params['compound_set_name'],
'description': 'Compound Set produced from %s' % file_name,
'compounds': compounds,
}
output = self._save_to_ws_and_report(ctx, 'compound_set_from_file',
params['workspace_name'],
params['staging_file_path'],
compoundset)
#END compound_set_from_file
# At some point might do deeper type checking...
if not isinstance(output, dict):
raise ValueError('Method compound_set_from_file return value ' +
'output is not type dict as required.')
# return the results
return [output]
def compound_set_to_file(self, ctx, params):
"""
CompoundSetToFile
string compound_set_name
string output_format
:param params: instance of type "compoundset_download_params" ->
structure: parameter "workspace_name" of String, parameter
"compound_set_name" of String, parameter "output_format" of String
:returns: instance of type "compoundset_download_results" ->
structure: parameter "report_name" of String, parameter
"report_ref" of String
"""
# ctx is the context object
# return variables are: output
#BEGIN compound_set_to_file
self._check_required_param(params, ['workspace_name', 'compound_set_name',
'output_format'])
compoundset = self.ws_client.get_objects2({'objects': [
{'workspace': params['workspace_name'],
'name': params['compound_set_name']}]})['data'][0]['data']
ext = params['output_format']
out = "%s/%s.%s" % (self.scratch, compoundset['name'], ext)
if ext == 'sdf':
outfile_path = parse.write_sdf(compoundset, out)
elif ext == 'tsv':
outfile_path = parse.write_tsv(compoundset, out)
else:
raise ValueError('Invalid output file type. Expects tsv or sdf')
report_files = [{'path': outfile_path,
'name': os.path.basename(outfile_path),
'label': os.path.basename(outfile_path),
'description': 'A compound set in %s format' % ext}]
report_params = {
'objects_created': [],
'message': 'Converted %s compound set to %s format.' % (
params['compound_set_name'], params['output_format']),
'file_links': report_files,
'workspace_name': params['workspace_name'],
'report_object_name': 'compound_set_download_report'
}
# Construct the output to send back
report_client = KBaseReport(self.callback_url)
report_info = report_client.create_extended_report(report_params)
output = {'report_name': report_info['name'],
'report_ref': report_info['ref'],
}
#END compound_set_to_file
# At some point might do deeper type checking...
if not isinstance(output, dict):
raise ValueError('Method compound_set_to_file return value ' +
'output is not type dict as required.')
# return the results
return [output]
def compound_set_from_model(self, ctx, params):
"""
CompoundSetFromModel
required:
string workspace_name
string model_name
string compound_set_name
:param params: instance of type "compoundset_from_model_params" ->
structure: parameter "workspace_name" of String, parameter
"model_name" of String, parameter "compound_set_name" of String
:returns: instance of type "compoundset_upload_results" -> structure:
parameter "report_name" of String, parameter "report_ref" of
String, parameter "compoundset_ref" of type "obj_ref"
"""
# ctx is the context object
# return variables are: output
#BEGIN compound_set_from_model
self._check_required_param(params, ['workspace_name', 'model_name',
'compound_set_name'])
model = self.ws_client.get_objects2({'objects': [
{'workspace': params['workspace_name'],
'name': params['model_name']}]})['data'][0]['data']
compounds, undef = parse.parse_model(model)
compoundset = {
'id': params['compound_set_name'],
'name': params['compound_set_name'],
'description': 'Compound Set produced from %s, a metabolic model'
% model['id'],
'compounds': compounds,
}
output = self._save_to_ws_and_report(ctx, 'compound_set_from_model',
params['workspace_name'],
params['model_name'], compoundset)
#END compound_set_from_model
# At some point might do deeper type checking...
if not isinstance(output, dict):
raise ValueError('Method compound_set_from_model return value ' +
'output is not type dict as required.')
# return the results
return [output]
def status(self, ctx):
#BEGIN_STATUS
returnVal = {'state': "OK",
'message': "",
'version': self.VERSION,
'git_url': self.GIT_URL,
'git_commit_hash': self.GIT_COMMIT_HASH}
#END_STATUS
return [returnVal]
from Workspace.WorkspaceClient import Workspace
import json
wsid = 16962
upa = '16962/3'
upa2 = '16962/23'
ws = Workspace('https://ci.kbase.us/services/ws')
d = ws.get_workspace_info({'id': wsid})
with open('get_workspace_info.json', 'w') as f:
f.write(json.dumps(d))
d = ws.list_objects({'ids': [wsid]})
with open('list_objects.json', 'w') as f:
f.write(json.dumps(d))
d = ws.get_objects2({'objects': [{'ref': upa}]})
with open('get_objects.json', 'w') as f:
f.write(json.dumps(d))
d = ws.get_objects2({'objects': [{'ref': upa2}]})
with open('narrative_object.json', 'w') as f:
f.write(json.dumps(d))
def fetch_reads_refs_from_sampleset(ref, ws_url, callback_url, params):
"""
From the given object ref, return a list of all reads objects that are a part of that
object. E.g., if ref is a ReadsSet, return a list of all PairedEndLibrary or SingleEndLibrary
refs that are a member of that ReadsSet. This is returned as a list of dictionaries as follows:
{
"ref": reads object reference,
"condition": condition string associated with that reads object
}
The only one required is "ref", all other keys may or may not be present, based on the reads
object or object type in initial ref variable. E.g. a RNASeqSampleSet might have condition info
for each reads object, but a single PairedEndLibrary may not have that info.
If ref is already a Reads library, just returns a list with ref as a single element.
"""
obj_type = get_object_type(ref, ws_url)
ws = Workspace(ws_url)
refs = list()
refs_for_ws_info = list()
if "KBaseSets.ReadsSet" in obj_type:
print("Looking up reads references in ReadsSet object")
set_client = SetAPI(callback_url)
reads_set = set_client.get_reads_set_v1({
"ref": ref,
"include_item_info": 0
})
for reads in reads_set["data"]["items"]:
refs.append({"ref": reads["ref"], "condition": reads["label"]})
refs_for_ws_info.append({'ref': reads['ref']})
elif "KBaseRNASeq.RNASeqSampleSet" in obj_type:
print("Looking up reads references in RNASeqSampleSet object")
sample_set = ws.get_objects2({"objects": [{
"ref": ref
}]})["data"][0]["data"]
for i in range(len(sample_set["sample_ids"])):
refs.append({
"ref": sample_set["sample_ids"][i],
"condition": sample_set["condition"][i]
})
refs_for_ws_info.append({'ref': sample_set['sample_ids'][i]})
elif ("KBaseAssembly.SingleEndLibrary" in obj_type
or "KBaseFile.SingleEndLibrary" in obj_type
or "KBaseFile.SingleEndLibrary-2.0" in obj_type
or "KBaseFile.SingleEndLibrary-2.1" in obj_type
or "KBaseAssembly.PairedEndLibrary" in obj_type
or "KBaseFile.PairedEndLibrary" in obj_type
or "KBaseFile.PairedEndLibrary-2.0" in obj_type
or "KBaseFile.PairedEndLibrary-2.1" in obj_type):
refs.append({"ref": ref})
refs_for_ws_info.append({'ref': ref})
else:
raise ValueError("Unable to fetch reads reference from object {} "
"which is a {}".format(ref, obj_type))
# get object info so we can name things properly
infos = ws.get_object_info3({'objects': refs_for_ws_info})['infos']
name_ext = '_alignment'
if ('alignment_suffix' in params
and params['alignment_suffix'] is not None):
ext = params['alignment_suffix'].replace(' ', '')
if ext:
name_ext = ext
unique_names = get_unique_names(infos)
for k in range(0, len(refs)):
refs[k]['info'] = infos[k]
name = unique_names[k] + name_ext
refs[k]['alignment_output_name'] = name
return refs
class DiffExprMatrixUtils:
"""
Constains a set of functions for expression levels calculations.
"""
PARAM_IN_WS_NAME = 'workspace_name'
PARAM_IN_OBJ_NAME = 'output_obj_name'
PARAM_IN_DIFFEXPMATSET_REF = 'diffExprMatrixSet_ref'
def __init__(self, config, logger=None):
self.config = config
self.logger = logger
self.scratch = os.path.join(config['scratch'],
'DEM_' + str(uuid.uuid4()))
self.ws_url = config['workspace-url']
self.serviceWizardURL = config['srv-wiz-url']
self._mkdir_p(self.scratch)
pass
def _mkdir_p(self, path):
"""
_mkdir_p: make directory for given path
"""
if not path:
return
try:
os.makedirs(path)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
def process_params(self, params):
"""
validates params passed to gen expression matrix method
"""
for p in [self.PARAM_IN_DIFFEXPMATSET_REF]:
if p not in params:
raise ValueError(
'"{}" parameter is required, but missing'.format(p))
def get_expressionset_data(self, expressionset_ref):
expr_set_obj = self.ws_client.get_objects2(
{'objects': [{
'ref': expressionset_ref
}]})['data'][0]
expr_set_obj_type = expr_set_obj.get('info')[2]
expr_set_data = dict()
expr_set_data['ws_name'] = expr_set_obj.get('info')[7]
expr_set_data['obj_name'] = expr_set_obj.get('info')[1]
if re.match('KBaseRNASeq.RNASeqExpressionSet-\d.\d',
expr_set_obj_type):
expr_set_data['genome_ref'] = expr_set_obj['data']['genome_id']
expr_obj_refs = list()
for expr_obj in expr_set_obj['data']['mapped_expression_ids']:
expr_obj_refs.append(expr_obj.values()[0])
expr_set_data['expr_obj_refs'] = expr_obj_refs
elif re.match('KBaseSets.ExpressionSet-\d.\d', expr_set_obj_type):
items = expr_set_obj.get('data').get('items')
expr_obj_refs = list()
for item in items:
expr_obj_refs.append(item['ref'])
expr_obj = self.ws_client.get_objects2(
{'objects': [{
'ref': expr_obj_refs[0]
}]})['data'][0]
expr_set_data['genome_ref'] = expr_obj['data']['genome_id']
expr_set_data['expr_obj_refs'] = expr_obj_refs
else:
raise TypeError(self.PARAM_IN_EXPSET_REF + ' should be of type ' +
'KBaseRNASeq.RNASeqExpressionSet ' +
'or KBaseSets.ExpressionSet')
return expr_set_data
def get_diffexpr_matrixset(self, params, token):
self.ws_client = Workspace(self.ws_url, token=token)
col_names = {
'gene_id': 'gene',
'log2_fold_change': 'log2fc_f',
'p_value': 'p_value_f',
'q_value': 'q_value'
}
json_fields = ['log2fc_f', 'p_value_f', 'q_value']
self.process_params(params)
diffexprmatset_list = list()
diffexprmatset_ref = params.get(self.PARAM_IN_DIFFEXPMATSET_REF)
diffexprmatset_obj = self.ws_client.get_objects2(
{'objects': [{
'ref': diffexprmatset_ref
}]})['data'][0]
items = diffexprmatset_obj.get('data').get('items')
diffexprmat_refs = list()
for item in items:
diffexprmat_refs.append(item['ref'])
self.logger.info('DiffExprMatrix ref: ' + item['ref'])
for diffexprmat_ref in diffexprmat_refs:
diffexprmat_dict = dict()
diffexprmat_obj = self.ws_client.get_objects2(
{'objects': [{
'ref': diffexprmat_ref
}]})['data'][0]
diffexprmat = diffexprmat_obj.get('data')
diffexprmat_dict['condition_1'] = diffexprmat.get(
'condition_mapping').keys()[0]
diffexprmat_dict['condition_2'] = diffexprmat.get(
'condition_mapping').values()[0]
voldata = list()
data = diffexprmat.get('data')
for row_index, row_id in enumerate(data.get('row_ids')):
row_data = dict()
row_data['gene'] = row_id
values = data.get('values')[row_index]
for col_index in range(len(values)):
row_data[json_fields[col_index]] = values[col_index]
voldata.append(row_data)
diffexprmat_dict['voldata'] = voldata
diffexprmatset_list.append(diffexprmat_dict)
return diffexprmatset_list
def get_matrix_stats(self, raw_row):
"""
returns a list of [ min, max, mean, std.dev, is_data_missing] for one row of conditional
expression values
"""
has_missing = "No"
row = []
for r in raw_row:
if r == None or numpy.isnan(
r
): # careful here - r can be 0 which is a legitimate value
has_missing = "Yes"
else:
row.append(r)
if len(row) < 1:
return (['NA', 'NA', 'NA', 'NA', 'Yes'])
if len(row) == 1:
sd = 0
else:
sd = numpy.std(row, ddof=1)
return ([min(row), max(row), numpy.mean(row), sd, has_missing])
def convert_dem_to_dict(self, dem):
"""
returns a dict that maps feature_id -> [ fc, q ]
"""
row_ids = dem.get('row_ids')
vals = dem.get('values')
n_rows = len(row_ids)
if (len(vals) != n_rows):
raise Exception(
"length discrepancy in differential expression matrix: {0} row_ids but {1} values"
.format(n_rows, len(fvals)))
dem_dict = {}
for _id, val in zip(row_ids, vals):
dem_dict[_id] = [
val[0], val[2]
] # [fc,q]. (not bothering to check for dups here)
return dem_dict
def get_enhancedFEM(self, params, tok):
"""
implements get_enhancedFilteredExpressionMatrix() method
"""
self.ws_client = Workspace(self.ws_url, token=tok)
if 'fem_object_ref' not in params:
raise ValueError(
"fem_object_ref parameter not given to get_enhancedFilteredExpressionMatrix"
)
fem_object_ref = params.get('fem_object_ref')
fem_obj_ret = self.ws_client.get_objects2(
{'objects': [{
'ref': fem_object_ref
}]})['data'][0]
fem = fem_obj_ret.get('data')
prov = fem_obj_ret.get('provenance')[0]
# create the enhanced FEM, starting with the FEM
efem = {}
for k in ['genome_ref', 'scale', 'type']:
efem[k] = fem.get(k)
efem['data'] = {}
efem['data']['col_ids'] = [
"description", "fold-change", "q-value", "min", "max", "mean",
"std_dev", "is_missing_values"
]
efem['data']['column_labels'] = [
"Description", "Fold change", "Q value", "Min. expression",
"Max. expression", "Mean expression", "Std. dev.",
"Missing values?"
]
fm = fem.get('data')
efem['data']['row_ids'] = fm.get('row_ids')
efem['data']['values'] = []
n_efem_rows = len(efem['data']['row_ids'])
fvals = fm.get('values')
if (len(fvals) != n_efem_rows):
raise Exception(
"length discrepancy in filtered expression matrix: {0} row_ids but {1} values"
.format(n_efem_rows, len(fvals)))
# Get genome object and feature descriptions as a handy feature-indexed dict
# moved from constructor
gaa = GenomeAnnotationAPI(self.serviceWizardURL, token=tok)
feat_dict = gaa.get_feature_functions({
'ref': fem.get('genome_ref'),
'feature_id_list': None
})
# if this FEM has a "resolved_ws_objects" record in its provenance,
# then that should be a list of one DEM reference from which we get the FC and q values
# as a feature (=row_id) -indexed dict.
if fem.get('diff_expr_matrix_ref'):
dem_ref = fem.get('diff_expr_matrix_ref')
dem_obj_ret = self.ws_client.get_objects2(
{'objects': [{
'ref': dem_ref
}]})['data'][0]
dem = dem_obj_ret.get('data')
dem_dict = self.convert_dem_to_dict(
dem.get('data')) # convert to dictionary for quick lookups
else:
dem_dict = {} # empty dictionary
# for each row
for row_id, fm_val_row in zip(fm.get('row_ids'), fvals):
# make a new row with NA for description, FC and q
new_values_row = ['NA', 'NA', 'NA'
] + self.get_matrix_stats(fm_val_row)
# if we have a description for this feature (row_id) put it in the first column
desc = feat_dict.get(row_id)
if desc:
new_values_row[
0] = desc # leave as 'NA' if no entry in feat_dict
# if we have a DEM entry for this row, put FC and q into 2nd and 3rd columns
d = dem_dict.get(row_id)
if d:
new_values_row[1], new_values_row[2] = d
# finally, add this row to the eFEM
efem['data']['values'].append(new_values_row)
return efem
class ExprMatrixUtils:
"""
Constains a set of functions for expression levels calculations.
"""
PARAM_IN_WS_NAME = 'workspace_name'
PARAM_IN_OBJ_NAME = 'output_obj_name'
PARAM_IN_EXPSET_REF = 'expressionset_ref'
def __init__(self, config, logger=None):
self.config = config
self.logger = logger
self.callback_url = os.environ['SDK_CALLBACK_URL']
self.scratch = os.path.join(config['scratch'], 'EM_' + str(uuid.uuid4()))
self.ws_url = config['workspace-url']
self.ws_client = Workspace(self.ws_url)
self.dfu = DataFileUtil(self.callback_url)
pass
def process_params(self, params):
"""
validates params passed to gen expression matrix method
"""
for p in [self.PARAM_IN_EXPSET_REF,
self.PARAM_IN_OBJ_NAME,
self.PARAM_IN_WS_NAME
]:
if p not in params:
raise ValueError('"{}" parameter is required, but missing'.format(p))
ws_name_id = params.get(self.PARAM_IN_WS_NAME)
if not isinstance(ws_name_id, int):
try:
ws_name_id = self.dfu.ws_name_to_id(ws_name_id)
except DFUError as se:
prefix = se.message.split('.')[0]
raise ValueError(prefix)
self.ws_id = ws_name_id
def get_expressionset_data(self, expressionset_ref):
expr_set_obj = self.ws_client.get_objects2(
{'objects': [{'ref': expressionset_ref}]})['data'][0]
expr_set_obj_type = expr_set_obj.get('info')[2]
expr_set_data = dict()
expr_set_data['ws_name'] = expr_set_obj.get('info')[7]
expr_set_data['obj_name'] = expr_set_obj.get('info')[1]
if re.match('KBaseRNASeq.RNASeqExpressionSet-\d.\d', expr_set_obj_type):
expr_set_data['genome_ref'] = expr_set_obj['data']['genome_id']
expr_obj_refs = list()
for expr_obj in expr_set_obj['data']['mapped_expression_ids']:
expr_obj_refs.append(expr_obj.values()[0])
expr_set_data['expr_obj_refs'] = expr_obj_refs
elif re.match('KBaseSets.ExpressionSet-\d.\d', expr_set_obj_type):
items = expr_set_obj.get('data').get('items')
expr_obj_refs = list()
for item in items:
expr_obj_refs.append(item['ref'])
expr_obj = self.ws_client.get_objects2(
{'objects': [{'ref': expr_obj_refs[0]}]})['data'][0]
expr_set_data['genome_ref'] = expr_obj['data']['genome_id']
expr_set_data['expr_obj_refs'] = expr_obj_refs
else:
raise TypeError(self.PARAM_IN_EXPSET_REF + ' should be of type ' +
'KBaseRNASeq.RNASeqExpressionSet ' +
'or KBaseSets.ExpressionSet')
return expr_set_data
def save_expression_matrix(self, tables, expr_set_data, em_obj_name, hidden = 0):
all_rows = {} # build a dictionary of keys only which is a union of all row ids (gene_ids)
self.logger.info( '***** length of tables is {0}'.format( len( tables )))
for table in tables:
for r in table.keys():
all_rows[r] = []
for gene_id in all_rows.keys():
row = []
for table in tables:
if ( gene_id in table ):
#logger.info( 'append ' + gene_id )
#logger.info( pformat( table[gene_id]))
#all_rows[gene_id].append( table[gene_id] )
row.append( table[gene_id] )
else:
#logger.info( 'append 0' )
row.append( 0 )
all_rows[gene_id] = row
#logger.info( all_rows[gene_id])
em_data = {
'genome_ref': expr_set_data['genome_ref'],
'scale': 'log2',
'type': 'level',
'data': {
'row_ids': [],
'values': [],
'col_ids': expr_set_data['expr_obj_names']
},
'feature_mapping' : {},
'condition_mapping': expr_set_data['condition_map']
}
# we need to load row-by-row to preserve the order
self.logger.info('loading expression matrix data')
for gene_id in all_rows.keys():
em_data['feature_mapping'][gene_id] = gene_id
em_data['data']['row_ids'].append(gene_id)
em_data['data']['values'].append(all_rows[gene_id])
try:
self.logger.info( 'saving em_data em_name {0}'.format(em_obj_name))
obj_info = self.dfu.save_objects({'id': self.ws_id,
'objects': [
{ 'type': 'KBaseFeatureValues.ExpressionMatrix',
'data': em_data,
'name': em_obj_name,
'hidden': hidden,
'extra_provenance_input_refs': [
em_data.get('genome_ref'),
self.params[self.PARAM_IN_EXPSET_REF]]
}
]})[0]
self.logger.info('ws save return:\n' + pformat(obj_info))
except Exception as e:
self.logger.exception(e)
raise Exception('Failed Saving Expression Matrix to Workspace')
return str(obj_info[6]) + '/' + str(obj_info[0]) + '/' + str(obj_info[4])
def get_expression_matrix(self, params):
self.process_params(params)
self.params = params
expressionset_ref = params.get(self.PARAM_IN_EXPSET_REF)
expr_set_data = self.get_expressionset_data(expressionset_ref)
expr_obj_names = list()
fpkm_tables = list()
tpm_tables = list()
condition_map = dict()
tpm_table = None
for expr_obj_ref in expr_set_data['expr_obj_refs']:
try:
self.logger.info('*** getting expression set {0} from workspace ****'
.format(expr_obj_ref))
expr = self.ws_client.get_objects2(
{'objects':
[{'ref': expr_obj_ref}]})['data'][0]
except Exception, e:
self.logger.exception(e)
raise Exception('Unable to download expression object {0} from workspace {1}'.
format(expr_obj_ref, expr_set_data['ws_name']))
expr_name = expr.get('info')[1]
expr_obj_names.append(expr_name)
condition_map.update({expr_name: expr.get('data').get('condition')})
num_interp = expr.get('data').get('numerical_interpretation')
if num_interp != 'FPKM':
raise Exception(
'Did not get expected FPKM value from numerical interpretation key from \
Expression object {0}, instead got '.format(expr_obj_ref, num_interp))
pr_comments = expr.get('data').get('processing_comments', None) # log2 Normalized
if pr_comments is not None:
self.logger.info('pr_comments are {0}'.format(pr_comments))
fpkm_table = expr.get('data').get('expression_levels') # QUESTION: is this really FPKM levels?
self.logger.info('FPKM keycount: {0}'.format(len(fpkm_table.keys())))
fpkm_tables.append(fpkm_table)
tpm_table = None # Cufflinks doesn't generate TPM
if 'tpm_expression_levels' in expr['data']: # so we need to check for this key
tpm_table = expr.get('data').get('tpm_expression_levels')
self.logger.info('TPM keycount: {0}'.format(len(tpm_table.keys())))
tpm_tables.append(tpm_table)
expr_set_data['expr_obj_names'] = expr_obj_names
expr_set_data['condition_map'] = condition_map
output_obj_name = params.get(self.PARAM_IN_OBJ_NAME)
fpkm_ref = self.save_expression_matrix(fpkm_tables,
expr_set_data,
'{0}_FPKM_ExpressionMatrix'.format(output_obj_name))
tpm_ref = None
if tpm_table is not None:
tpm_ref = self.save_expression_matrix(tpm_tables,
expr_set_data,
'{0}_TPM_ExpressionMatrix'.format(output_obj_name))
return fpkm_ref, tpm_ref
def fetch_reads_refs_from_sampleset(ref, ws_url, callback_url):
"""
From the given object ref, return a list of all reads objects that are a part of that
object. E.g., if ref is a ReadsSet, return a list of all PairedEndLibrary or SingleEndLibrary
refs that are a member of that ReadsSet. This is returned as a list of dictionaries as follows:
{
"ref": reads object reference,
"condition": condition string associated with that reads object,
"name": reads object name (needed for saving an AlignmentSet)
}
The only one required is "ref", all other keys may or may not be present, based on the reads
object or object type in initial ref variable. E.g. a RNASeqSampleSet might have condition info
for each reads object, but a single PairedEndLibrary may not have that info.
If ref is already a Reads library, just returns a list with ref as a single element.
"""
obj_type = get_object_type(ref, ws_url)
refs = list()
if "KBaseSets.ReadsSet" in obj_type:
print("Looking up reads references in ReadsSet object")
set_client = SetAPI(callback_url)
reads_set = set_client.get_reads_set_v1({
"ref": ref,
"include_item_info": 0
})
print("Got results from ReadsSet object")
pprint(reads_set)
ref_list = [r["ref"] for r in reads_set["data"]["items"]]
reads_names = get_object_names(ref_list, ws_url)
for reads in reads_set["data"]["items"]:
ref = reads["ref"]
refs.append({
"ref": ref,
"condition": reads["label"],
"name": reads_names[ref]
})
elif "KBaseRNASeq.RNASeqSampleSet" in obj_type:
print("Looking up reads references in RNASeqSampleSet object")
ws = Workspace(ws_url)
sample_set = ws.get_objects2({"objects": [{
"ref": ref
}]})["data"][0]["data"]
sample_names = get_object_names(sample_set["sample_ids"], ws_url)
for i in range(len(sample_set["sample_ids"])):
ref = sample_set["sample_ids"][i]
refs.append({
"ref": ref,
"condition": sample_set["condition"][i],
"name": sample_names[ref]
})
elif ("KBaseAssembly.SingleEndLibrary" in obj_type
or "KBaseFile.SingleEndLibrary" in obj_type
or "KBaseAssembly.PairedEndLibrary" in obj_type
or "KBaseFile.PairedEndLibrary" in obj_type):
refs.append({"ref": ref, "name": get_object_names([ref], ws_url)[ref]})
else:
raise ValueError("Unable to fetch reads reference from object {} "
"which is a {}".format(ref, obj_type))
return refs
def exec_remove_adapters(self, ctx, params):
"""
:param params: instance of type "RemoveAdaptersParams" -> structure:
parameter "output_workspace" of String, parameter
"output_object_name" of String, parameter "input_reads" of type
"ws_ref" (@ref ws), parameter "five_prime" of type
"FivePrimeOptions" (unfortunately, we have to name the fields
uniquely between 3' and 5' options due to the current
implementation of grouped parameters) -> structure: parameter
"adapter_sequence_5P" of String, parameter "anchored_5P" of type
"boolean" (@range (0, 1)), parameter "three_prime" of type
"ThreePrimeOptions" -> structure: parameter "adapter_sequence_3P"
of String, parameter "anchored_3P" of type "boolean" (@range (0,
1)), parameter "error_tolerance" of Double, parameter
"min_overlap_length" of Long, parameter "min_read_length" of Long,
parameter "discard_untrimmed" of type "boolean" (@range (0, 1))
:returns: instance of type "exec_RemoveAdaptersResult" -> structure:
parameter "report" of String, parameter "output_reads_ref" of
String
"""
# ctx is the context object
# return variables are: result
#BEGIN exec_remove_adapters
console = []
self.log(console, 'Running exec_remove_adapters() with parameters: ')
self.log(console, "\n" + pformat(params))
self.log(console, "-----------------------------------------------\n")
report = ''
returnVal = dict()
returnVal['output_reads_ref'] = None
token = ctx['token']
wsClient = workspaceService(self.config['workspace-url'], token=token)
ws = Workspace(self.config['workspace-url'], token=token)
#setAPI_Client = SetAPI (url=self.config['SDK_CALLBACK_URL'], token=token) # for SDK local, doesn't work for SetAPI
setAPI_Client = SetAPI(url=self.config['service-wizard-url'],
token=token) # for dynamic service
headers = {'Authorization': 'OAuth ' + token}
env = os.environ.copy()
env['KB_AUTH_TOKEN'] = token
# 0. param checks
required_params = [
'output_workspace', 'input_reads', 'output_object_name'
]
for arg in required_params:
if arg not in params or params[arg] == None or params[arg] == '':
raise ValueError("Must define required param: '" + arg + "'")
# 1. load provenance
provenance = [{}]
if 'provenance' in ctx:
provenance = ctx['provenance']
# add additional info to provenance here, in this case the input data object reference
provenance[0]['input_ws_objects'] = [str(params['input_reads'])]
# 2. Determine whether read library, ReadsSet or RNASeqSampleSet is input object
#
try:
# object_info tuple
[
OBJID_I, NAME_I, TYPE_I, SAVE_DATE_I, VERSION_I, SAVED_BY_I,
WSID_I, WORKSPACE_I, CHSUM_I, SIZE_I, META_I
] = range(11)
input_reads_obj_info = wsClient.get_object_info_new(
{'objects': [{
'ref': params['input_reads']
}]})[0]
input_reads_obj_type = input_reads_obj_info[TYPE_I]
input_reads_obj_type = re.sub(
'-[0-9]+\.[0-9]+$', "",
input_reads_obj_type) # remove trailing version
#input_reads_obj_version = input_reads_obj_info[VERSION_I] # this is object version, not type version
except Exception as e:
raise ValueError(
'Unable to get read library object from workspace: (' +
str(params['input_reads']) + ')' + str(e))
acceptable_types = [
"KBaseSets.ReadsSet", "KBaseRNASeq.RNASeqSampleSet",
"KBaseFile.PairedEndLibrary", "KBaseFile.SingleEndLibrary",
"KBaseAssembly.PairedEndLibrary", "KBaseAssembly.SingleEndLibrary"
]
if input_reads_obj_type not in acceptable_types:
raise ValueError("Input reads of type: '" + input_reads_obj_type +
"'. Must be one of " +
", ".join(acceptable_types))
# 3. Retrieve the set details
#
readsSet_ref_list = []
readsSet_names_list = []
readsSet_types_list = []
if "KBaseSets.ReadsSet" in input_reads_obj_type:
try:
input_readsSet_obj = setAPI_Client.get_reads_set_v1({
'ref':
params['input_reads'],
'include_item_info':
1
})
except Exception as e:
raise ValueError(
'SetAPI FAILURE: Unable to get read library set object from workspace: ('
+ str(params['input_reads']) + ")\n" + str(e))
for readsLibrary_obj in input_readsSet_obj['data']['items']:
readsSet_ref_list.append(readsLibrary_obj['ref'])
NAME_I = 1
TYPE_I = 2
readsSet_names_list.append(readsLibrary_obj['info'][NAME_I])
this_type = readsLibrary_obj['info'][TYPE_I]
this_type = re.sub('-[0-9]+\.[0-9]+$', "",
this_type) # remove trailing version
readsSet_types_list.append(this_type)
elif "KBaseRNASeq.RNASeqSampleSet" in input_reads_obj_type:
sample_set = ws.get_objects2(
{"objects": [{
"ref": params['input_reads']
}]})["data"][0]["data"]
sample_refs = list()
for i in range(len(sample_set["sample_ids"])):
readsSet_ref_list.append(sample_set["sample_ids"][i])
sample_refs.append({"ref": sample_set["sample_ids"][i]})
info = ws.get_object_info3({"objects": sample_refs})
for j in range(len(info["infos"])):
NAME_I = 1
TYPE_I = 2
readsSet_names_list.append(info["infos"][j][NAME_I])
sample_type = info["infos"][j][TYPE_I]
sample_type = re.sub('-[0-9]+\.[0-9]+$', "",
sample_type) # remove trailing version
readsSet_types_list.append(sample_type)
else:
readsSet_ref_list = [params['input_reads']]
readsSet_names_list = [params['output_object_name']]
readsSet_types_list = [input_reads_obj_type]
# 4. Iterate through readsLibrary memebers of set
#
report = ''
cutadapt_readsSet_ref = None
cutadapt_readsLib_refs = []
for reads_item_i, input_reads_library_ref in enumerate(
readsSet_ref_list):
exec_remove_adapters_OneLibrary_params = {
'output_workspace': params['output_workspace'],
'input_reads': input_reads_library_ref,
'reads_type': readsSet_types_list[reads_item_i]
}
if (input_reads_obj_type != "KBaseSets.ReadsSet"
and input_reads_obj_type != "KBaseRNASeq.RNASeqSampleSet"):
exec_remove_adapters_OneLibrary_params[
'output_object_name'] = params['output_object_name']
else:
exec_remove_adapters_OneLibrary_params[
'output_object_name'] = readsSet_names_list[
reads_item_i] + "_cutadapt"
optional_params = [
'float error_tolerance', 'min_overlap_length',
'min_read_length', 'discard_untrimmed'
]
optional_g_params = {
'five_prime': ['adapter_sequence_5P', 'anchored_5P'],
'three_prime': ['adapter_sequence_3P', 'anchored_3P']
}
for arg in optional_params:
if arg in params and params[arg] != None:
exec_remove_adapters_OneLibrary_params[arg] = params[arg]
for group in optional_g_params.keys():
if group in params and params[group] != None:
exec_remove_adapters_OneLibrary_params[group] = dict()
for arg in optional_g_params[group]:
if arg in params[group] and params[group][arg] != None:
exec_remove_adapters_OneLibrary_params[group][
arg] = params[group][arg]
msg = "\n\nRUNNING exec_remove_adapters_OneLibrary() ON LIBRARY: " + str(
input_reads_library_ref) + " " + str(
readsSet_names_list[reads_item_i]) + "\n"
msg += "----------------------------------------------------------------------------\n"
report += msg
self.log(console, msg)
# RUN
exec_remove_adapters_OneLibrary_retVal = self.exec_remove_adapters_OneLibrary(
ctx, exec_remove_adapters_OneLibrary_params)[0]
report += exec_remove_adapters_OneLibrary_retVal['report'] + "\n\n"
cutadapt_readsLib_refs.append(
exec_remove_adapters_OneLibrary_retVal['output_reads_ref'])
# 5. Conclude
# Just one Library
if (input_reads_obj_type != "KBaseSets.ReadsSet"
and input_reads_obj_type != "KBaseRNASeq.RNASeqSampleSet"):
# create return output object
result = {
'report': report,
'output_reads_ref': cutadapt_readsLib_refs[0],
}
# ReadsSet or SampleSet
else:
# save cutadapt readsSet
some_cutadapt_output_created = False
items = []
for i, lib_ref in enumerate(cutadapt_readsLib_refs):
if lib_ref == None:
#items.append(None) # can't have 'None' items in ReadsSet
continue
else:
some_cutadapt_output_created = True
try:
label = input_readsSet_obj['data']['items'][i]['label']
except:
NAME_I = 1
label = ws.get_object_info3(
{'objects': [{
'ref': lib_ref
}]})['infos'][0][NAME_I]
label = label + "_cutadapt"
items.append({
'ref': lib_ref,
'label': label
#'data_attachment': ,
#'info':
})
if some_cutadapt_output_created:
reads_desc_ext = " + Cutadapt"
#reads_name_ext = "_cutadapt"
descText = ""
reads_name_ext = ""
try:
descText = input_readsSet_obj['data']['description']
except:
NAME_I = 1
descText = ws.get_object_info3(
{'objects': [{
'ref': params['input_reads']
}]})['infos'][0][NAME_I]
descText = descText + reads_desc_ext
output_readsSet_obj = {'description': descText, 'items': items}
output_readsSet_name = str(
params['output_object_name']) + reads_name_ext
cutadapt_readsSet_ref = setAPI_Client.save_reads_set_v1({
'workspace_name':
params['output_workspace'],
'output_object_name':
output_readsSet_name,
'data':
output_readsSet_obj
})['set_ref']
else:
raise ValueError("No cutadapt output created")
# create return output object
result = {
'report': report,
'output_reads_ref': cutadapt_readsSet_ref
}
#END exec_remove_adapters
# At some point might do deeper type checking...
if not isinstance(result, dict):
raise ValueError('Method exec_remove_adapters return value ' +
'result is not type dict as required.')
# return the results
return [result]
class TaxonAPI:
'''
Module Name:
TaxonAPI
Module Description:
A KBase module: TaxonAPI
'''
######## WARNING FOR GEVENT USERS ####### noqa
# Since asynchronous IO can lead to methods - even the same method -
# interrupting each other, you must be *very* careful when using global
# state. A method could easily clobber the state set by another while
# the latter method is running.
######################################### noqa
VERSION = "1.0.1"
GIT_URL = "[email protected]:kbase/taxon_api.git"
GIT_COMMIT_HASH = "de62da0a5e2d7e098927862ec0e0c09be1103f68"
#BEGIN_CLASS_HEADER
_GENOME_TYPES = [
'KBaseGenomes.Genome', 'KBaseGenomeAnnotations.GenomeAnnotation'
]
_TAXON_TYPES = ['KBaseGenomeAnnotations.Taxon']
@functools32.lru_cache(maxsize=1000)
def get_object(self, ref):
res = self.ws.get_objects2({'objects': [{'ref': ref}]})['data'][0]
return res
def get_data(self, ref):
obj = self.get_object(ref)
return obj['data']
@functools32.lru_cache(maxsize=1000)
def translate_to_MD5_types(self, ktype):
return self.ws.translate_to_MD5_types([ktype]).values()[0]
def get_referrers(self, ref):
referrers = self.ws.list_referencing_objects([{"ref": ref}])[0]
object_refs_by_type = dict()
tlist = []
for x in referrers:
tlist.append(x[2])
typemap = self.ws.translate_to_MD5_types(tlist)
for x in referrers:
typestring = typemap[x[2]]
if typestring not in object_refs_by_type:
object_refs_by_type[typestring] = list()
upa = '%d/%d/%d' % (x[6], x[0], x[4])
object_refs_by_type[typestring].append(upa)
return object_refs_by_type
def get_reffers_type(self, ref, types):
referrers = self.get_referrers(ref)
children = list()
for object_type in referrers:
if object_type.split('-')[0] in types:
children.extend(referrers[object_type])
return children
def make_hash(self, i):
omd = i[10]
if i[10] == {}:
omd = None
return {
'type_string': i[2],
'workspace_id': i[6],
'object_checksum': i[8],
'object_reference': '%d/%d' % (i[6], i[0]),
'object_size': i[9],
'saved_by': i[5],
'object_id': i[0],
'save_date': i[3],
'object_metadata': omd,
'object_name': i[1],
'version': i[4],
'workspace_name': i[7],
'object_reference_versioned': '%d/%d/%d' % (i[6], i[0], i[4])
}
#END_CLASS_HEADER
# config contains contents of config file in a hash or None if it couldn't
# be found
def __init__(self, config):
#BEGIN_CONSTRUCTOR
self.workspaceURL = config['workspace-url']
self.ws = Workspace(self.workspaceURL)
self.shockURL = config['shock-url']
self.logger = logging.getLogger()
log_handler = logging.StreamHandler()
log_handler.setFormatter(
logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
self.logger.addHandler(log_handler)
#END_CONSTRUCTOR
pass
def get_parent(self, ctx, ref):
"""
Retrieve parent Taxon.
@return Reference to parent Taxon.
:param ref: instance of type "ObjectReference"
:returns: instance of type "ObjectReference"
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_parent
data = self.get_data(ref)
try:
returnVal = data['parent_taxon_ref']
# returnVal=taxon_api.get_parent(ref_only=True)
except:
returnVal = ''
#END get_parent
# At some point might do deeper type checking...
if not isinstance(returnVal, basestring):
raise ValueError('Method get_parent return value ' +
'returnVal is not type basestring as required.')
# return the results
return [returnVal]
def get_children(self, ctx, ref):
"""
Retrieve children Taxon.
@return List of references to child Taxons.
:param ref: instance of type "ObjectReference"
:returns: instance of list of type "ObjectReference"
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_children
returnVal = self.get_reffers_type(ref, self._TAXON_TYPES)
#END get_children
# At some point might do deeper type checking...
if not isinstance(returnVal, list):
raise ValueError('Method get_children return value ' +
'returnVal is not type list as required.')
# return the results
return [returnVal]
def get_genome_annotations(self, ctx, ref):
"""
funcdef GenomeAnnotation(s) that refer to this Taxon.
If this is accessing a KBaseGenomes.Genome object, it will
return an empty list (this information is not available).
@return List of references to GenomeAnnotation objects.
:param ref: instance of type "ObjectReference"
:returns: instance of list of type "ObjectReference"
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_genome_annotations
returnVal = self.get_reffers_type(ref, self._GENOME_TYPES)
#END get_genome_annotations
# At some point might do deeper type checking...
if not isinstance(returnVal, list):
raise ValueError('Method get_genome_annotations return value ' +
'returnVal is not type list as required.')
# return the results
return [returnVal]
def get_scientific_lineage(self, ctx, ref):
"""
Retrieve the scientific lineage.
@return Strings for each 'unit' of the lineage, ordered in
the usual way from Domain to Kingdom to Phylum, etc.
:param ref: instance of type "ObjectReference"
:returns: instance of list of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_scientific_lineage
o = self.ws.get_objects2({'objects': [{
'ref': ref
}]})['data'][0]['data']
returnVal = [x.strip() for x in o['scientific_lineage'].split(";")]
#END get_scientific_lineage
# At some point might do deeper type checking...
if not isinstance(returnVal, list):
raise ValueError('Method get_scientific_lineage return value ' +
'returnVal is not type list as required.')
# return the results
return [returnVal]
def get_scientific_name(self, ctx, ref):
"""
Retrieve the scientific name.
@return The scientific name, e.g., "Escherichia Coli K12 str. MG1655"
:param ref: instance of type "ObjectReference"
:returns: instance of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_scientific_name
obj = self.ws.get_objects2({'objects': [{
'ref': ref
}]})['data'][0]['data']
returnVal = obj['scientific_name']
#END get_scientific_name
# At some point might do deeper type checking...
if not isinstance(returnVal, basestring):
raise ValueError('Method get_scientific_name return value ' +
'returnVal is not type basestring as required.')
# return the results
return [returnVal]
def get_taxonomic_id(self, ctx, ref):
"""
Retrieve the NCBI taxonomic ID of this Taxon.
For type KBaseGenomes.Genome, the ``source_id`` will be returned.
@return Integer taxonomic ID.
:param ref: instance of type "ObjectReference"
:returns: instance of Long
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_taxonomic_id
obj = self.ws.get_objects2({'objects': [{
'ref': ref
}]})['data'][0]['data']
returnVal = obj['taxonomy_id']
#END get_taxonomic_id
# At some point might do deeper type checking...
if not isinstance(returnVal, int):
raise ValueError('Method get_taxonomic_id return value ' +
'returnVal is not type int as required.')
# return the results
return [returnVal]
def get_kingdom(self, ctx, ref):
"""
Retrieve the kingdom.
:param ref: instance of type "ObjectReference"
:returns: instance of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_kingdom
obj = self.ws.get_objects2({'objects': [{
'ref': ref
}]})['data'][0]['data']
returnVal = obj['kingdom']
#END get_kingdom
# At some point might do deeper type checking...
if not isinstance(returnVal, basestring):
raise ValueError('Method get_kingdom return value ' +
'returnVal is not type basestring as required.')
# return the results
return [returnVal]
def get_domain(self, ctx, ref):
"""
Retrieve the domain.
:param ref: instance of type "ObjectReference"
:returns: instance of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_domain
obj = self.ws.get_objects2({'objects': [{
'ref': ref
}]})['data'][0]['data']
returnVal = obj['domain']
#END get_domain
# At some point might do deeper type checking...
if not isinstance(returnVal, basestring):
raise ValueError('Method get_domain return value ' +
'returnVal is not type basestring as required.')
# return the results
return [returnVal]
def get_genetic_code(self, ctx, ref):
"""
Retrieve the genetic code.
:param ref: instance of type "ObjectReference"
:returns: instance of Long
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_genetic_code
obj = self.ws.get_objects2({'objects': [{
'ref': ref
}]})['data'][0]['data']
returnVal = obj['genetic_code']
#END get_genetic_code
# At some point might do deeper type checking...
if not isinstance(returnVal, int):
raise ValueError('Method get_genetic_code return value ' +
'returnVal is not type int as required.')
# return the results
return [returnVal]
def get_aliases(self, ctx, ref):
"""
Retrieve the aliases.
:param ref: instance of type "ObjectReference"
:returns: instance of list of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_aliases
obj = self.ws.get_objects2({'objects': [{
'ref': ref
}]})['data'][0]['data']
if 'aliases' in obj:
returnVal = obj['aliases']
else:
returnVal = list()
#END get_aliases
# At some point might do deeper type checking...
if not isinstance(returnVal, list):
raise ValueError('Method get_aliases return value ' +
'returnVal is not type list as required.')
# return the results
return [returnVal]
def get_info(self, ctx, ref):
"""
Retrieve object info.
@skip documentation
:param ref: instance of type "ObjectReference"
:returns: instance of type "ObjectInfo" (* @skip documentation) ->
structure: parameter "object_id" of Long, parameter "object_name"
of String, parameter "object_reference" of String, parameter
"object_reference_versioned" of String, parameter "type_string" of
String, parameter "save_date" of String, parameter "version" of
Long, parameter "saved_by" of String, parameter "workspace_id" of
Long, parameter "workspace_name" of String, parameter
"object_checksum" of String, parameter "object_size" of Long,
parameter "object_metadata" of mapping from String to String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_info
# returnVal = self.ws.get_objects2({'objects': [{'ref': ref}]})['data'][0]['info']
i = self.get_object(ref)['info']
#md5_typestr = self.ws.translate_to_MD5_types([i[2]]).values()[0]
returnVal = self.make_hash(i)
#END get_info
# At some point might do deeper type checking...
if not isinstance(returnVal, dict):
raise ValueError('Method get_info return value ' +
'returnVal is not type dict as required.')
# return the results
return [returnVal]
def get_history(self, ctx, ref):
"""
Retrieve object history.
@skip documentation
:param ref: instance of type "ObjectReference"
:returns: instance of type "ObjectHistory" (* @skip documentation) ->
list of type "ObjectInfo" (* @skip documentation) -> structure:
parameter "object_id" of Long, parameter "object_name" of String,
parameter "object_reference" of String, parameter
"object_reference_versioned" of String, parameter "type_string" of
String, parameter "save_date" of String, parameter "version" of
Long, parameter "saved_by" of String, parameter "workspace_id" of
Long, parameter "workspace_name" of String, parameter
"object_checksum" of String, parameter "object_size" of Long,
parameter "object_metadata" of mapping from String to String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_history
# returnVal = self.ws.get_object_history({'ref': ref})
returnVal = []
for i in self.ws.get_object_history({'ref': ref}):
returnVal.append(self.make_hash(i))
#END get_history
# At some point might do deeper type checking...
if not isinstance(returnVal, list):
raise ValueError('Method get_history return value ' +
'returnVal is not type list as required.')
# return the results
return [returnVal]
def get_provenance(self, ctx, ref):
"""
Retrieve object provenance.
@skip documentation
:param ref: instance of type "ObjectReference"
:returns: instance of type "ObjectProvenance" (* @skip documentation)
-> list of type "ObjectProvenanceAction" (* @skip documentation)
-> structure: parameter "time" of String, parameter "service_name"
of String, parameter "service_version" of String, parameter
"service_method" of String, parameter "method_parameters" of list
of String, parameter "script_name" of String, parameter
"script_version" of String, parameter "script_command_line" of
String, parameter "input_object_references" of list of String,
parameter "validated_object_references" of list of String,
parameter "intermediate_input_ids" of list of String, parameter
"intermediate_output_ids" of list of String, parameter
"external_data" of list of type "ExternalDataUnit" (* @skip
documentation) -> structure: parameter "resource_name" of String,
parameter "resource_url" of String, parameter "resource_version"
of String, parameter "resource_release_date" of String, parameter
"data_url" of String, parameter "data_id" of String, parameter
"description" of String, parameter "description" of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_provenance
prov = self.ws.get_object_provenance([{"ref": ref}])[0]['provenance']
returnVal = []
copy_keys = {
"time": "time",
"service": "service_name",
"service_ver": "service_version",
"method": "service_method",
"method_params": "method_parameters",
"script": "script_name",
"script_ver": "script_version",
"script_command_line": "script_command_line",
"input_ws_objects": "input_object_references",
"resolved_ws_objects": "validated_object_references",
"intermediate_incoming": "intermediate_input_ids",
"intermediate_outgoing": "intermediate_output_ids",
"external_data": "external_data",
"description": "description"
}
for object_provenance in prov:
action = dict()
for k in copy_keys:
if k in object_provenance:
if isinstance(object_provenance[k], list) and len(
object_provenance[k]) == 0:
continue
action[copy_keys[k]] = object_provenance[k]
returnVal.append(action)
#END get_provenance
# At some point might do deeper type checking...
if not isinstance(returnVal, list):
raise ValueError('Method get_provenance return value ' +
'returnVal is not type list as required.')
# return the results
return [returnVal]
def get_id(self, ctx, ref):
"""
Retrieve object identifier.
@skip documentation
:param ref: instance of type "ObjectReference"
:returns: instance of Long
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_id
returnVal = self.get_object(ref)['info'][0]
#END get_id
# At some point might do deeper type checking...
if not isinstance(returnVal, int):
raise ValueError('Method get_id return value ' +
'returnVal is not type int as required.')
# return the results
return [returnVal]
def get_name(self, ctx, ref):
"""
Retrieve object name.
@skip documentation
:param ref: instance of type "ObjectReference"
:returns: instance of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_name
returnVal = self.get_object(ref)['info'][1]
#END get_name
# At some point might do deeper type checking...
if not isinstance(returnVal, basestring):
raise ValueError('Method get_name return value ' +
'returnVal is not type basestring as required.')
# return the results
return [returnVal]
def get_version(self, ctx, ref):
"""
Retrieve object version.
@skip documentation
:param ref: instance of type "ObjectReference"
:returns: instance of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_version
returnVal = str(self.get_object(ref)['info'][4])
#END get_version
# At some point might do deeper type checking...
if not isinstance(returnVal, basestring):
raise ValueError('Method get_version return value ' +
'returnVal is not type basestring as required.')
# return the results
return [returnVal]
def get_all_data(self, ctx, params):
"""
:param params: instance of type "GetAllDataParams" -> structure:
parameter "ref" of type "ObjectReference", parameter
"include_decorated_scientific_lineage" of type "boolean" (A
boolean. 0 = false, other = true.), parameter
"include_decorated_children" of type "boolean" (A boolean. 0 =
false, other = true.), parameter "exclude_children" of type
"boolean" (A boolean. 0 = false, other = true.)
:returns: instance of type "TaxonData" -> structure: parameter
"parent" of type "ObjectReference", parameter "children" of list
of type "ObjectReference", parameter "decorated_children" of list
of type "TaxonInfo" -> structure: parameter "ref" of type
"ObjectReference", parameter "scientific_name" of String,
parameter "scientific_lineage" of list of String, parameter
"decorated_scientific_lineage" of list of type "TaxonInfo" ->
structure: parameter "ref" of type "ObjectReference", parameter
"scientific_name" of String, parameter "scientific_name" of
String, parameter "taxonomic_id" of Long, parameter "kingdom" of
String, parameter "domain" of String, parameter "genetic_code" of
Long, parameter "aliases" of list of String, parameter "obj_info"
of type "ObjectInfo" (* @skip documentation) -> structure:
parameter "object_id" of Long, parameter "object_name" of String,
parameter "object_reference" of String, parameter
"object_reference_versioned" of String, parameter "type_string" of
String, parameter "save_date" of String, parameter "version" of
Long, parameter "saved_by" of String, parameter "workspace_id" of
Long, parameter "workspace_name" of String, parameter
"object_checksum" of String, parameter "object_size" of Long,
parameter "object_metadata" of mapping from String to String
"""
# ctx is the context object
# return variables are: d
#BEGIN get_all_data
d = {}
ref = params['ref']
obj = self.get_object(ref)
data = obj['data']
try:
d['parent'] = data['parent_taxon_ref']
except KeyError:
print('Error getting parent for ' + ref)
# +':\n'+ str(traceback.format_exc()))
d['parent'] = None
if 'exclude_children' in params and params['exclude_children'] == 1:
pass
else:
d['children'] = self.get_reffers_type(ref, self._TAXON_TYPES)
d['scientific_lineage'] = data['scientific_lineage']
d['scientific_name'] = data['scientific_name']
d['taxonomic_id'] = data['taxonomy_id']
try:
d['kingdom'] = data['kingdom']
# throws error if not found, so catch and log it
except KeyError:
print('Error getting kingdom for ' + ref)
# +':\n'+ str(traceback.format_exc()))
d['kingdom'] = None
d['domain'] = data['domain']
d['genetic_code'] = data['genetic_code']
d['aliases'] = None
if 'aliases' in data:
d['aliases'] = data['aliases']
d['info'] = self.make_hash(obj['info'])
key = 'include_decorated_scientific_lineage'
if key in params and params[key] == 1:
l = self.get_decorated_scientific_lineage(ctx, {'ref': ref})[0]
d['decorated_scientific_lineage'] = l[
'decorated_scientific_lineage']
key = 'include_decorated_children'
if key in params and params[key] == 1:
l = self.get_decorated_children(ctx, {'ref': ref})[0]
d['decorated_children'] = l['decorated_children']
#END get_all_data
# At some point might do deeper type checking...
if not isinstance(d, dict):
raise ValueError('Method get_all_data return value ' +
'd is not type dict as required.')
# return the results
return [d]
def get_decorated_scientific_lineage(self, ctx, params):
"""
:param params: instance of type "GetDecoratedScientificLineageParams"
-> structure: parameter "ref" of type "ObjectReference"
:returns: instance of type "DecoratedScientificLineage" (list starts
at the root, and goes on down to this) -> structure: parameter
"decorated_scientific_lineage" of list of type "TaxonInfo" ->
structure: parameter "ref" of type "ObjectReference", parameter
"scientific_name" of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_decorated_scientific_lineage
lineageList = []
ref = params['ref']
while True:
parent_data = None
try:
# note: doesn't look like there is a way to get a reference
# of a Taxon directly (without constructing it from
# object_info), so first get reference, then instantiate
# another API object
parent_ref = self.get_data(ref)['parent_taxon_ref']
if parent_ref is not None:
data = self.get_data(ref)
scientific_name = data['scientific_name']
if scientific_name != 'root':
parent_data = {
'ref': parent_ref,
'scientific_name': scientific_name
}
ref = parent_ref
except KeyError:
# case where parent is not found
pass
if parent_data is not None:
lineageList.append(parent_data)
else:
break
lineageList.reverse() # reverse list to match scientific_lineage style
returnVal = {'decorated_scientific_lineage': lineageList[:-1]}
#END get_decorated_scientific_lineage
# At some point might do deeper type checking...
if not isinstance(returnVal, dict):
raise ValueError(
'Method get_decorated_scientific_lineage return value ' +
'returnVal is not type dict as required.')
# return the results
return [returnVal]
def get_decorated_children(self, ctx, params):
"""
:param params: instance of type "GetDecoratedChildrenParams" ->
structure: parameter "ref" of type "ObjectReference"
:returns: instance of type "DecoratedChildren" -> structure:
parameter "decorated_children" of list of type "TaxonInfo" ->
structure: parameter "ref" of type "ObjectReference", parameter
"scientific_name" of String
"""
# ctx is the context object
# return variables are: returnVal
#BEGIN get_decorated_children
ref = params['ref']
children_refs = self.get_reffers_type(ref, self._TAXON_TYPES)
decorated_children = []
for child_ref in children_refs:
decorated_children.append({
'ref':
child_ref,
'scientific_name':
self.get_data(child_ref)['scientific_name']
})
returnVal = {'decorated_children': decorated_children}
#END get_decorated_children
# At some point might do deeper type checking...
if not isinstance(returnVal, dict):
raise ValueError('Method get_decorated_children return value ' +
'returnVal is not type dict as required.')
# return the results
return [returnVal]
def status(self, ctx):
#BEGIN_STATUS
returnVal = {
'state': "OK",
'message': "",
'version': self.VERSION,
'git_url': self.GIT_URL,
'git_commit_hash': self.GIT_COMMIT_HASH
}
#END_STATUS
return [returnVal]
class KnowledgeEngineAppsUtil:
METRIC = [
"braycurtis", "canberra", "chebyshev", "cityblock", "correlation",
"cosine", "dice", "euclidean", "hamming", "jaccard", "kulsinski",
"matching", "rogerstanimoto", "russellrao", "sokalmichener",
"sokalsneath", "sqeuclidean", "yule"
]
METHOD = [
"single", "complete", "average", "weighted", "centroid", "median",
"ward"
]
CRITERION = ["inconsistent", "distance", "maxclust"]
def _mkdir_p(self, path):
"""
_mkdir_p: make directory for given path
"""
if not path:
return
try:
os.makedirs(path)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
def _validate_run_pca_params(self, params):
"""
_validate_run_pca_params:
validates params passed to run_pca method
"""
log('start validating run_pca params')
# check for required parameters
for p in ['cluster_set_ref', 'workspace_name', 'pca_matrix_name']:
if p not in params:
raise ValueError(
'"{}" parameter is required, but missing'.format(p))
def _validate_run_kmeans_cluster_params(self, params):
"""
_validate_run_kmeans_cluster_params:
validates params passed to run_kmeans_cluster method
"""
log('start validating run_kmeans_cluster params')
# check for required parameters
for p in ['matrix_ref', 'workspace_name', 'cluster_set_name', 'k_num']:
if p not in params:
raise ValueError(
'"{}" parameter is required, but missing'.format(p))
# check metric validation
metric = params.get('dist_metric')
if metric and metric not in self.METRIC:
error_msg = 'INPUT ERROR:\nInput metric function [{}] is not valid.\n'.format(
metric)
error_msg += 'Available metric: {}'.format(self.METRIC)
raise ValueError(error_msg)
def _validate_run_hierarchical_cluster_params(self, params):
"""
_validate_run_hierarchical_cluster_params:
validates params passed to run_hierarchical_cluster method
"""
log('start validating run_hierarchical_cluster params')
# check for required parameters
for p in [
'matrix_ref', 'workspace_name', 'cluster_set_name',
'dist_cutoff_rate'
]:
if p not in params:
raise ValueError(
'"{}" parameter is required, but missing'.format(p))
# check metric validation
metric = params.get('dist_metric')
if metric and metric not in self.METRIC:
error_msg = 'INPUT ERROR:\nInput metric function [{}] is not valid.\n'.format(
metric)
error_msg += 'Available metric: {}'.format(self.METRIC)
raise ValueError(error_msg)
# check method validation
method = params.get('linkage_method')
if method and method not in self.METHOD:
error_msg = 'INPUT ERROR:\nInput linkage algorithm [{}] is not valid.\n'.format(
method)
error_msg += 'Available metric: {}'.format(self.METHOD)
raise ValueError(error_msg)
# check criterion validation
criterion = params.get('fcluster_criterion')
if criterion and criterion not in self.CRITERION:
error_msg = 'INPUT ERROR:\nInput criterion [{}] is not valid.\n'.format(
criterion)
error_msg += 'Available metric: {}'.format(self.CRITERION)
raise ValueError(error_msg)
def _gen_clusters(self, clusters, conditionset_mapping):
clusters_list = list()
for cluster in clusters.values():
labeled_cluster = {}
labeled_cluster.update({'id_to_data_position': cluster})
if conditionset_mapping:
id_to_condition = {
k: v
for k, v in conditionset_mapping.items()
if k in cluster.keys()
}
labeled_cluster.update({'id_to_condition': id_to_condition})
clusters_list.append(labeled_cluster)
return clusters_list
def _gen_hierarchical_clusters(self, clusters, conditionset_mapping,
data_matrix):
clusters_list = list()
df = pd.read_json(data_matrix)
index = df.index.tolist()
for cluster in clusters.values():
labeled_cluster = {}
id_to_data_position = {}
for item in cluster:
id_to_data_position.update({item: index.index(item)})
labeled_cluster.update(
{'id_to_data_position': id_to_data_position})
if conditionset_mapping:
id_to_condition = {
k: v
for k, v in conditionset_mapping.items() if k in cluster
}
labeled_cluster.update({'id_to_condition': id_to_condition})
clusters_list.append(labeled_cluster)
return clusters_list
def _build_hierarchical_cluster_set(self, clusters, cluster_set_name,
genome_ref, matrix_ref,
conditionset_mapping, conditionset_ref,
workspace_name, clustering_parameters,
data_matrix):
"""
_build_kmeans_cluster_set: build KBaseExperiments.ClusterSet object
"""
log('start saving KBaseExperiments.ClusterSet object')
if isinstance(workspace_name, int) or workspace_name.isdigit():
workspace_id = workspace_name
else:
workspace_id = self.dfu.ws_name_to_id(workspace_name)
clusters_list = self._gen_hierarchical_clusters(
clusters, conditionset_mapping, data_matrix)
cluster_set_data = {
'clusters': clusters_list,
'clustering_parameters': clustering_parameters,
'original_data': matrix_ref,
'condition_set_ref': conditionset_ref,
'genome_ref': genome_ref
}
cluster_set_data = {k: v for k, v in cluster_set_data.items() if v}
object_type = 'KBaseExperiments.ClusterSet'
save_object_params = {
'id':
workspace_id,
'objects': [{
'type': object_type,
'data': cluster_set_data,
'name': cluster_set_name
}]
}
dfu_oi = self.dfu.save_objects(save_object_params)[0]
cluster_set_ref = str(dfu_oi[6]) + '/' + str(dfu_oi[0]) + '/' + str(
dfu_oi[4])
return cluster_set_ref
def _build_kmeans_cluster_set(self, clusters, cluster_set_name, genome_ref,
matrix_ref, conditionset_mapping,
conditionset_ref, workspace_name,
clustering_parameters):
"""
_build_kmeans_cluster_set: build KBaseExperiments.ClusterSet object
"""
log('start saving KBaseExperiments.ClusterSet object')
if isinstance(workspace_name, int) or workspace_name.isdigit():
workspace_id = workspace_name
else:
workspace_id = self.dfu.ws_name_to_id(workspace_name)
clusters_list = self._gen_clusters(clusters, conditionset_mapping)
cluster_set_data = {
'clusters': clusters_list,
'clustering_parameters': clustering_parameters,
'original_data': matrix_ref,
'condition_set_ref': conditionset_ref,
'genome_ref': genome_ref
}
cluster_set_data = {k: v for k, v in cluster_set_data.items() if v}
object_type = 'KBaseExperiments.ClusterSet'
save_object_params = {
'id':
workspace_id,
'objects': [{
'type': object_type,
'data': cluster_set_data,
'name': cluster_set_name
}]
}
dfu_oi = self.dfu.save_objects(save_object_params)[0]
cluster_set_ref = str(dfu_oi[6]) + '/' + str(dfu_oi[0]) + '/' + str(
dfu_oi[4])
return cluster_set_ref
def _generate_visualization_content(self, output_directory,
row_dendrogram_path,
row_dendrogram_truncate_path,
col_dendrogram_path,
col_dendrogram_truncate_path):
"""
_generate_visualization_content: generate visualization html content
"""
visualization_content = ''
if row_dendrogram_path:
row_dendrogram_name = 'row_dendrogram.png'
row_dendrogram_display_name = 'row dendrogram'
shutil.copy2(row_dendrogram_path,
os.path.join(output_directory, row_dendrogram_name))
visualization_content += '<div class="gallery">'
visualization_content += '<a target="_blank" href="{}">'.format(
row_dendrogram_name)
visualization_content += '<img src="{}" '.format(
row_dendrogram_name)
visualization_content += 'alt="{}" width="600" height="400">'.format(
row_dendrogram_display_name)
visualization_content += '</a><div class="desc">{}</div></div>'.format(
row_dendrogram_display_name)
if row_dendrogram_truncate_path:
row_den_truncate_name = 'row_dendrogram_last12.png'
row_den_truncate_display_name = 'row dendrogram truncated (last 12 merges)'
shutil.copy2(row_dendrogram_truncate_path,
os.path.join(output_directory, row_den_truncate_name))
visualization_content += '<div class="gallery">'
visualization_content += '<a target="_blank" href="{}">'.format(
row_den_truncate_name)
visualization_content += '<img src="{}" '.format(
row_den_truncate_name)
visualization_content += 'alt="{}" width="600" height="400">'.format(
row_den_truncate_display_name)
visualization_content += '</a><div class="desc">{}</div></div>'.format(
row_den_truncate_display_name)
if col_dendrogram_path:
col_dendrogram_name = 'column_dendrogram.png'
col_dendrogram_display_name = 'column dendrogram'
shutil.copy2(col_dendrogram_path,
os.path.join(output_directory, col_dendrogram_name))
visualization_content += '<div class="gallery">'
visualization_content += '<a target="_blank" href="{}">'.format(
col_dendrogram_name)
visualization_content += '<img src="{}" '.format(
col_dendrogram_name)
visualization_content += 'alt="{}" width="600" height="400">'.format(
col_dendrogram_display_name)
visualization_content += '</a><div class="desc">{}</div></div>'.format(
col_dendrogram_display_name)
if col_dendrogram_truncate_path:
col_den_truncate_name = 'column_dendrogram_last12.png'
col_den_truncate_display_name = 'column dendrogram truncated (last 12 merges)'
shutil.copy2(col_dendrogram_truncate_path,
os.path.join(output_directory, col_den_truncate_name))
visualization_content += '<div class="gallery">'
visualization_content += '<a target="_blank" href="{}">'.format(
col_den_truncate_name)
visualization_content += '<img src="{}" '.format(
col_den_truncate_name)
visualization_content += 'alt="{}" width="600" height="400">'.format(
col_den_truncate_display_name)
visualization_content += '</a><div class="desc">{}</div></div>'.format(
col_den_truncate_display_name)
if not visualization_content:
visualization_content = '<p>Dendrogram is too large to be printed.</p>'
return visualization_content
def _generate_hierarchical_html_report(self, cluster_set_refs,
row_dendrogram_path,
row_dendrogram_truncate_path,
col_dendrogram_path,
col_dendrogram_truncate_path):
"""
_generate_hierarchical_html_report: generate html summary report for hierarchical
clustering app
"""
log('start generating html report')
html_report = list()
output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(output_directory)
result_file_path = os.path.join(output_directory, 'hier_report.html')
visualization_content = self._generate_visualization_content(
output_directory, row_dendrogram_path,
row_dendrogram_truncate_path, col_dendrogram_path,
col_dendrogram_truncate_path)
with open(result_file_path, 'w') as result_file:
with open(
os.path.join(os.path.dirname(__file__),
'hier_report_template.html'),
'r') as report_template_file:
report_template = report_template_file.read()
report_template = report_template.replace(
'<p>Visualization_Content</p>', visualization_content)
result_file.write(report_template)
report_shock_id = self.dfu.file_to_shock({
'file_path': output_directory,
'pack': 'zip'
})['shock_id']
html_report.append({
'shock_id':
report_shock_id,
'name':
os.path.basename(result_file_path),
'label':
os.path.basename(result_file_path),
'description':
'HTML summary report for ExpressionMatrix Cluster App'
})
return html_report
def _generate_hierarchical_cluster_report(self, cluster_set_refs,
workspace_name,
row_dendrogram_path,
row_dendrogram_truncate_path,
col_dendrogram_path,
col_dendrogram_truncate_path):
"""
_generate_hierarchical_cluster_report: generate summary report
"""
log('creating report')
output_html_files = self._generate_hierarchical_html_report(
cluster_set_refs, row_dendrogram_path,
row_dendrogram_truncate_path, col_dendrogram_path,
col_dendrogram_truncate_path)
objects_created = []
for cluster_set_ref in cluster_set_refs:
objects_created.append({
'ref': cluster_set_ref,
'description': 'Hierarchical ClusterSet'
})
report_params = {
'message': '',
'workspace_name': workspace_name,
'objects_created': objects_created,
'html_links': output_html_files,
'direct_html_link_index': 0,
'html_window_height': 333,
'report_object_name': 'kb_hier_cluster_report_' + str(uuid.uuid4())
}
kbase_report_client = KBaseReport(self.callback_url)
output = kbase_report_client.create_extended_report(report_params)
report_output = {
'report_name': output['name'],
'report_ref': output['ref']
}
return report_output
def _generate_kmeans_cluster_report(self, cluster_set_refs,
workspace_name):
"""
_generate_kmeans_cluster_report: generate summary report
"""
objects_created = []
for cluster_set_ref in cluster_set_refs:
objects_created.append({
'ref': cluster_set_ref,
'description': 'Kmeans ClusterSet'
})
report_params = {
'message': '',
'objects_created': objects_created,
'workspace_name': workspace_name,
'report_object_name': 'run_kmeans_cluster_' + str(uuid.uuid4())
}
kbase_report_client = KBaseReport(self.callback_url, token=self.token)
output = kbase_report_client.create_extended_report(report_params)
report_output = {
'report_name': output['name'],
'report_ref': output['ref']
}
return report_output
def _generate_pca_html_files(self, pca_plots, n_components):
log('start generating html report')
html_report = list()
output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(output_directory)
result_file_path = os.path.join(output_directory, 'pca_report.html')
visualization_content = ''
for pca_plot in pca_plots:
pca_plot_name = os.path.basename(pca_plot)
pca_plot_display_name = '{} Component PCA'.format(n_components)
shutil.copy2(pca_plot, os.path.join(output_directory,
pca_plot_name))
visualization_content += '<div class="gallery">'
visualization_content += '<a target="_blank" href="{}">'.format(
pca_plot_name)
visualization_content += '<img src="{}" '.format(pca_plot_name)
visualization_content += 'alt="{}" width="600" height="600">'.format(
pca_plot_display_name)
visualization_content += '</a><div class="desc">{}</div></div>'.format(
pca_plot_display_name)
with open(result_file_path, 'w') as result_file:
with open(
os.path.join(os.path.dirname(__file__),
'pca_report_template.html'),
'r') as report_template_file:
report_template = report_template_file.read()
report_template = report_template.replace(
'<p>Visualization_Content</p>', visualization_content)
result_file.write(report_template)
report_shock_id = self.dfu.file_to_shock({
'file_path': output_directory,
'pack': 'zip'
})['shock_id']
html_report.append({
'shock_id':
report_shock_id,
'name':
os.path.basename(result_file_path),
'label':
os.path.basename(result_file_path),
'description':
'HTML summary report for ExpressionMatrix Cluster App'
})
return html_report
def _generate_pca_plot(self, pca_matrix_data):
"""
_generate_pca_plot: generate a plot for PCA data
"""
pca_plots = []
output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(output_directory)
df = pd.DataFrame(pca_matrix_data.get('values'),
index=pca_matrix_data.get('row_ids'),
columns=pca_matrix_data.get('col_ids'))
n_components = range(1, df.columns.size)
all_pairs = list(itertools.combinations(n_components, 2))
for pair in all_pairs:
first_component = pair[0]
second_component = pair[1]
pca_plot = os.path.join(
output_directory,
'pca_{}_{}.png'.format(first_component, second_component))
plt.switch_backend('agg')
fig = plt.figure(figsize=(8, 8))
ax = fig.add_subplot(1, 1, 1)
ax.set_xlabel('Principal Component {}'.format(first_component),
fontsize=15)
ax.set_ylabel('Principal Component {}'.format(second_component),
fontsize=15)
ax.set_title('{} component PCA'.format(len(n_components)),
fontsize=20)
clusters = list(
set(['cluster_{}'.format(x) for x in df['cluster'].tolist()]))
colors = [
'red', 'green', 'blue', 'orange', 'yellow', 'pink',
'lightcyan', 'cyan'
]
if len(clusters) > len(colors):
np.random.seed(19680801)
N = len(clusters)
colors = []
for i in range(N):
colors.append(np.random.rand(3, ))
for cluster, color in zip(clusters, colors):
indicesToKeep = df['cluster'] == int(cluster.split('_')[-1])
ax.scatter(
df.loc[indicesToKeep,
'principal_component_{}'.format(first_component)],
df.loc[indicesToKeep,
'principal_component_{}'.format(second_component)],
c=color,
s=50)
ax.legend(clusters, loc='best')
ax.grid()
plt.savefig(pca_plot)
pca_plots.append(pca_plot)
return pca_plots, len(n_components)
def _generate_pca_report(self, pca_ref, pca_matrix_data, workspace_name):
"""
_generate_kmeans_cluster_report: generate summary report
"""
objects_created = []
objects_created.append({'ref': pca_ref, 'description': 'PCA Matrix'})
pca_plots, n_components = self._generate_pca_plot(pca_matrix_data)
output_html_files = self._generate_pca_html_files(
pca_plots, n_components)
report_params = {
'message': '',
'objects_created': objects_created,
'workspace_name': workspace_name,
'html_links': output_html_files,
'direct_html_link_index': 0,
'report_object_name': 'run_pca_' + str(uuid.uuid4())
}
kbase_report_client = KBaseReport(self.callback_url, token=self.token)
output = kbase_report_client.create_extended_report(report_params)
report_output = {
'report_name': output['name'],
'report_ref': output['ref']
}
return report_output
def _save_2D_matrix(self, df, clusters, workspace_name, pca_matrix_name):
"""
_save_2D_matrix: save dataframe as KBaseFeatureValues.FloatMatrix2D object
"""
log('start saving KBaseFeatureValues.FloatMatrix2D object')
if isinstance(workspace_name, int) or workspace_name.isdigit():
workspace_id = workspace_name
else:
workspace_id = self.dfu.ws_name_to_id(workspace_name)
row_ids = df.index.tolist()
col_ids = df.columns.tolist()
col_ids.append('cluster')
values = df.values.tolist()
idx = 0
for cluster in clusters:
cluster_items = cluster.get('id_to_condition').keys()
for cluster_item in cluster_items:
pos = row_ids.index(cluster_item)
values[pos].append(idx)
idx += 1
pca_matrix_data = {
'row_ids': row_ids,
'col_ids': col_ids,
'values': values
}
object_type = 'KBaseFeatureValues.FloatMatrix2D'
save_object_params = {
'id':
workspace_id,
'objects': [{
'type': object_type,
'data': pca_matrix_data,
'name': pca_matrix_name
}]
}
dfu_oi = self.dfu.save_objects(save_object_params)[0]
float_matrix_ref = str(dfu_oi[6]) + '/' + str(dfu_oi[0]) + '/' + str(
dfu_oi[4])
return float_matrix_ref, pca_matrix_data
def _build_flat_cluster(self,
data_matrix,
dist_cutoff_rate,
dist_metric=None,
linkage_method=None,
fcluster_criterion=None):
"""
_build_cluster: build flat clusters and dendrogram for data_matrix
"""
# calculate distance matrix
pdist_params = {'data_matrix': data_matrix, 'metric': dist_metric}
pdist_ret = self.ke_util.run_pdist(pdist_params)
dist_matrix = pdist_ret['dist_matrix']
labels = pdist_ret['labels']
# performs hierarchical/agglomerative clustering
linkage_params = {'dist_matrix': dist_matrix, 'method': linkage_method}
linkage_ret = self.ke_util.run_linkage(linkage_params)
linkage_matrix = linkage_ret['linkage_matrix']
newick = self.ke_util.linkage_2_newick({
'linkage_matrix': linkage_matrix,
'labels': labels
})['newick']
height = max([item[2] for item in linkage_matrix])
dist_threshold = height * dist_cutoff_rate
log('Height: {} Setting dist_threshold: {}'.format(
height, dist_threshold))
merges = len(linkage_matrix)
# generate flat clusters
fcluster_params = {
'linkage_matrix': linkage_matrix,
'dist_threshold': dist_threshold,
'labels': labels,
'criterion': fcluster_criterion
}
fcluster_ret = self.ke_util.run_fcluster(fcluster_params)
flat_cluster = fcluster_ret['flat_cluster']
# generate dendrogram
try:
dendrogram_params = {
'linkage_matrix': linkage_matrix,
'dist_threshold': dist_threshold,
'labels': labels
}
dendrogram_ret = self.ke_util.run_dendrogram(dendrogram_params)
dendrogram_path = dendrogram_ret['result_plots'][0]
except:
dendrogram_path = None
# generate truncated (last 12 merges) dendrogram
if merges > 256:
dendrogram_truncate_params = {
'linkage_matrix': linkage_matrix,
'dist_threshold': dist_threshold,
'labels': labels,
'last_merges': 12
}
dendrogram_truncate_ret = self.ke_util.run_dendrogram(
dendrogram_truncate_params)
dendrogram_truncate_path = dendrogram_truncate_ret['result_plots'][
0]
else:
dendrogram_truncate_path = None
return flat_cluster, labels, newick, dendrogram_path, dendrogram_truncate_path
def _build_kmeans_cluster(self, data_matrix, k_num, dist_metric=None):
"""
_build_kmeans_cluster: Build Kmeans cluster
"""
# calculate distance matrix
pdist_params = {'data_matrix': data_matrix, 'metric': dist_metric}
pdist_ret = self.ke_util.run_pdist(pdist_params)
dist_matrix = pdist_ret['dist_matrix']
labels = pdist_ret['labels']
# run kmeans algorithm
kmeans_params = {'dist_matrix': dist_matrix, 'k_num': k_num}
kmeans_ret = self.ke_util.run_kmeans2(kmeans_params)
centroid = kmeans_ret.get('kmeans_ret')
idx = kmeans_ret.get('idx')
df = pd.read_json(data_matrix)
rows = df.index.tolist()
clusters = {}
for list_index, value in enumerate(idx):
cluster = clusters.get(value)
if not cluster:
clusters.update({value: {rows[list_index]: list_index}})
else:
cluster.update({rows[list_index]: list_index})
return clusters
def __init__(self, config):
self.ws_url = config["workspace-url"]
self.callback_url = config['SDK_CALLBACK_URL']
self.token = config['KB_AUTH_TOKEN']
self.shock_url = config['shock-url']
self.srv_wiz_url = config['srv-wiz-url']
self.scratch = config['scratch']
self.dfu = DataFileUtil(self.callback_url)
self.ke_util = kb_ke_util(self.callback_url, service_ver="dev")
self.gen_api = GenericsAPI(self.callback_url, service_ver="dev")
self.ws = Workspace(self.ws_url, token=self.token)
self.set_client = SetAPI(self.srv_wiz_url)
def run_pca(self, params):
"""
run_pca: generates PCA matrix for KBaseExperiments.ClusterSet data object
cluster_set_ref: KBaseExperiments.ClusterSet object references
workspace_name: the name of the workspace
pca_matrix_name: name of PCA (KBaseFeatureValues.FloatMatrix2D) object
n_components - number of components (default 2)
pca_ref: PCA object reference (as KBaseFeatureValues.FloatMatrix2D data type)
report_name: report name generated by KBaseReport
report_ref: report reference generated by KBaseReport
"""
self._validate_run_pca_params(params)
cluster_set_ref = params.get('cluster_set_ref')
workspace_name = params.get('workspace_name')
pca_matrix_name = params.get('pca_matrix_name')
n_components = int(params.get('n_components', 2))
cluster_set_source = self.dfu.get_objects(
{"object_refs": [cluster_set_ref]})['data'][0]
cluster_set_info = cluster_set_source.get('info')
cluster_set_name = cluster_set_info[1]
cluster_set_data = cluster_set_source.get('data')
clusters = cluster_set_data.get('clusters')
matrix_ref = cluster_set_data.get('original_data')
data_matrix = self.gen_api.fetch_data({
'obj_ref': matrix_ref
}).get('data_matrix')
if '_column' in cluster_set_name:
data_matrix = pd.read_json(
data_matrix).T.to_json() # transpose matrix
# run pca algorithm
pca_params = {'data_matrix': data_matrix, 'n_components': n_components}
PCA_matrix = self.ke_util.run_PCA(pca_params).get('PCA_matrix')
df = pd.read_json(PCA_matrix)
df.fillna(0, inplace=True)
pca_ref, pca_matrix_data = self._save_2D_matrix(
df, clusters, workspace_name, pca_matrix_name)
returnVal = {'pca_ref': pca_ref}
report_output = self._generate_pca_report(pca_ref, pca_matrix_data,
workspace_name)
returnVal.update(report_output)
return returnVal
def run_kmeans_cluster(self, params):
"""
run_kmeans_cluster: generates Kmeans clusters for Matrix data object
matrix_ref: Matrix object reference
workspace_name: the name of the workspace
cluster_set_name: KBaseExperiments.ClusterSet object name
k_num: number of clusters to form
Optional arguments:
dist_metric: The distance metric to use. Default set to 'euclidean'.
The distance function can be
["braycurtis", "canberra", "chebyshev", "cityblock", "correlation", "cosine",
"dice", "euclidean", "hamming", "jaccard", "kulsinski", "matching",
"rogerstanimoto", "russellrao", "sokalmichener", "sokalsneath", "sqeuclidean",
"yule"]
Details refer to:
https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.pdist.html
return:
cluster_set_refs: KBaseExperiments.ClusterSet object references
report_name: report name generated by KBaseReport
report_ref: report reference generated by KBaseReport
"""
self._validate_run_kmeans_cluster_params(params)
matrix_ref = params.get('matrix_ref')
workspace_name = params.get('workspace_name')
cluster_set_name = params.get('cluster_set_name')
k_num = params.get('k_num')
dist_metric = params.get('dist_metric')
matrix_object = self.ws.get_objects2(
{'objects': [{
'ref': matrix_ref
}]})['data'][0]
matrix_data = matrix_object['data']
data_matrix = self.gen_api.fetch_data({
'obj_ref': matrix_ref
}).get('data_matrix')
transpose_data_matrix = pd.read_json(data_matrix).T.to_json()
row_kmeans_clusters = self._build_kmeans_cluster(
data_matrix, k_num, dist_metric=dist_metric)
col_kmeans_clusters = self._build_kmeans_cluster(
transpose_data_matrix, k_num, dist_metric=dist_metric)
genome_ref = matrix_data.get('genome_ref')
clustering_parameters = {
'k_num': str(k_num),
'dist_metric': str(dist_metric)
}
cluster_set_refs = []
row_cluster_set_name = cluster_set_name + '_row'
row_cluster_set = self._build_kmeans_cluster_set(
row_kmeans_clusters, row_cluster_set_name, genome_ref, matrix_ref,
matrix_data.get('row_mapping'),
matrix_data.get('row_conditionset_ref'), workspace_name,
clustering_parameters)
cluster_set_refs.append(row_cluster_set)
col_cluster_set_name = cluster_set_name + '_column'
col_cluster_set = self._build_kmeans_cluster_set(
col_kmeans_clusters, col_cluster_set_name, genome_ref, matrix_ref,
matrix_data.get('col_mapping'),
matrix_data.get('col_conditionset_ref'), workspace_name,
clustering_parameters)
cluster_set_refs.append(col_cluster_set)
returnVal = {'cluster_set_refs': cluster_set_refs}
report_output = self._generate_kmeans_cluster_report(
cluster_set_refs, workspace_name)
returnVal.update(report_output)
return returnVal
def run_hierarchical_cluster(self, params):
"""
run_hierarchical_cluster: generates hierarchical clusters for Matrix data object
matrix_ref: Matrix object reference
workspace_name: the name of the workspace
cluster_set_name: KBaseExperiments.ClusterSet object name
dist_cutoff_rate: the threshold to apply when forming flat clusters
Optional arguments:
dist_metric: The distance metric to use. Default set to 'euclidean'.
The distance function can be
["braycurtis", "canberra", "chebyshev", "cityblock", "correlation", "cosine",
"dice", "euclidean", "hamming", "jaccard", "kulsinski", "matching",
"rogerstanimoto", "russellrao", "sokalmichener", "sokalsneath",
"sqeuclidean", "yule"]
Details refer to:
https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.pdist.html
linkage_method: The linkage algorithm to use. Default set to 'ward'.
The method can be
["single", "complete", "average", "weighted", "centroid", "median", "ward"]
Details refer to:
https://docs.scipy.org/doc/scipy/reference/generated/scipy.cluster.hierarchy.linkage.html
fcluster_criterion: The criterion to use in forming flat clusters.
Default set to 'inconsistent'.
The criterion can be
["inconsistent", "distance", "maxclust"]
Details refer to:
https://docs.scipy.org/doc/scipy/reference/generated/scipy.cluster.hierarchy.fcluster.html
return:
cluster_set_refs: KBaseExperiments.ClusterSet object references
report_name: report name generated by KBaseReport
report_ref: report reference generated by KBaseReport
"""
log('--->\nrunning run_hierarchical_cluster\n' +
'params:\n{}'.format(json.dumps(params, indent=1)))
self._validate_run_hierarchical_cluster_params(params)
matrix_ref = params.get('matrix_ref')
workspace_name = params.get('workspace_name')
cluster_set_name = params.get('cluster_set_name')
dist_cutoff_rate = float(params.get('dist_cutoff_rate'))
dist_metric = params.get('dist_metric')
linkage_method = params.get('linkage_method')
fcluster_criterion = params.get('fcluster_criterion')
matrix_object = self.ws.get_objects2(
{'objects': [{
'ref': matrix_ref
}]})['data'][0]
matrix_data = matrix_object['data']
data_matrix = self.gen_api.fetch_data({
'obj_ref': matrix_ref
}).get('data_matrix')
transpose_data_matrix = pd.read_json(data_matrix).T.to_json()
(row_flat_cluster, row_labels, row_newick, row_dendrogram_path,
row_dendrogram_truncate_path) = self._build_flat_cluster(
data_matrix,
dist_cutoff_rate,
dist_metric=dist_metric,
linkage_method=linkage_method,
fcluster_criterion=fcluster_criterion)
(col_flat_cluster, col_labels, col_newick, col_dendrogram_path,
col_dendrogram_truncate_path) = self._build_flat_cluster(
transpose_data_matrix,
dist_cutoff_rate,
dist_metric=dist_metric,
linkage_method=linkage_method,
fcluster_criterion=fcluster_criterion)
genome_ref = matrix_data.get('genome_ref')
clustering_parameters = {
'dist_cutoff_rate': str(dist_cutoff_rate),
'dist_metric': dist_metric,
'linkage_method': linkage_method,
'fcluster_criterion': fcluster_criterion
}
cluster_set_refs = []
row_cluster_set_name = cluster_set_name + '_row'
row_cluster_set = self._build_hierarchical_cluster_set(
row_flat_cluster, row_cluster_set_name, genome_ref, matrix_ref,
matrix_data.get('row_mapping'),
matrix_data.get('row_conditionset_ref'), workspace_name,
clustering_parameters, data_matrix)
cluster_set_refs.append(row_cluster_set)
col_cluster_set_name = cluster_set_name + '_column'
col_cluster_set = self._build_hierarchical_cluster_set(
col_flat_cluster, col_cluster_set_name, genome_ref, matrix_ref,
matrix_data.get('col_mapping'),
matrix_data.get('col_conditionset_ref'), workspace_name,
clustering_parameters, transpose_data_matrix)
cluster_set_refs.append(col_cluster_set)
returnVal = {'cluster_set_refs': cluster_set_refs}
report_output = self._generate_hierarchical_cluster_report(
cluster_set_refs, workspace_name, row_dendrogram_path,
row_dendrogram_truncate_path, col_dendrogram_path,
col_dendrogram_truncate_path)
returnVal.update(report_output)
return returnVal
from Workspace.WorkspaceClient import Workspace
import json
import os
def grab(upa, file):
if not os.path.exists(file):
d = ws.get_objects2({'objects': [{'ref': upa}]})
with open(file, 'w') as f:
f.write(json.dumps(d, indent=4))
ws = Workspace('https://ci.kbase.us/services/ws')
grab('36815/4/1', './test/mock_data/media_object.json')
grab('17335/21/2', './test/mock_data/fbamodel_object.json')
grab('4/23/1', './test/mock_data/media2_object.json')
upa = '16174/15/1'
l = ['scientific_name', 'taxonomy', 'id']
fname = './test/mock_data/genome_sub_object.json'
if not os.path.exists(fname):
d = ws.get_objects2({'objects': [{'ref': upa, 'included': l}]})
with open(fname, 'w') as f:
f.write(json.dumps(d, indent=2))
class CufflinksUtils:
CUFFLINKS_TOOLKIT_PATH = '/opt/cufflinks/'
GFFREAD_TOOLKIT_PATH = '/opt/cufflinks/'
def __init__(self, config):
"""
:param config:
:param logger:
:param directory: Working directory
:param urls: Service urls
"""
# BEGIN_CONSTRUCTOR
self.ws_url = config["workspace-url"]
self.ws_url = config["workspace-url"]
self.callback_url = config['SDK_CALLBACK_URL']
self.srv_wiz_url = config['srv-wiz-url']
self.token = config['KB_AUTH_TOKEN']
self.shock_url = config['shock-url']
self.dfu = DataFileUtil(self.callback_url)
self.gfu = GenomeFileUtil(self.callback_url)
self.au = AssemblyUtil(self.callback_url)
self.rau = ReadsAlignmentUtils(self.callback_url)
self.set_api = SetAPI(self.srv_wiz_url, service_ver='dev')
self.eu = ExpressionUtils(self.callback_url)
self.ws = Workspace(self.ws_url, token=self.token)
self.scratch = os.path.join(config['scratch'], str(uuid.uuid4()))
self._mkdir_p(self.scratch)
self.tool_used = "Cufflinks"
self.tool_version = os.environ['VERSION']
# END_CONSTRUCTOR
pass
def parse_FPKMtracking_calc_TPM(self, filename):
"""
Generates TPM from FPKM
:return:
"""
fpkm_dict = {}
tpm_dict = {}
gene_col = 0
fpkm_col = 9
sum_fpkm = 0.0
with open(filename) as f:
next(f)
for line in f:
larr = line.split("\t")
gene_id = larr[gene_col]
if gene_id != "":
fpkm = float(larr[fpkm_col])
sum_fpkm = sum_fpkm + fpkm
fpkm_dict[gene_id] = math.log(fpkm + 1, 2)
tpm_dict[gene_id] = fpkm
if sum_fpkm == 0.0:
log("Warning: Unable to calculate TPM values as sum of FPKM values is 0"
)
else:
for g in tpm_dict:
tpm_dict[g] = math.log((tpm_dict[g] / sum_fpkm) * 1e6 + 1, 2)
return fpkm_dict, tpm_dict
def _mkdir_p(self, path):
"""
_mkdir_p: make directory for given path
"""
if not path:
return
try:
os.makedirs(path)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
def _validate_run_cufflinks_params(self, params):
"""
_validate_run_cufflinks_params:
Raises an exception if params are invalid
"""
log('Start validating run_cufflinks params')
# check for required parameters
for p in ['alignment_object_ref', 'workspace_name', 'genome_ref']:
if p not in params:
raise ValueError(
'"{}" parameter is required, but missing'.format(p))
def _run_command(self, command):
"""
_run_command: run command and print result
"""
log('Start executing command:\n{}'.format(command))
pipe = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
output = pipe.communicate()[0]
exitCode = pipe.returncode
if (exitCode == 0):
log('Executed command:\n{}\n'.format(command) +
'Exit Code: {}\nOutput:\n{}'.format(exitCode, output))
else:
error_msg = 'Error running command:\n{}\n'.format(command)
error_msg += 'Exit Code: {}\nOutput:\n{}'.format(exitCode, output)
raise ValueError(error_msg)
def _run_gffread(self, gff_path, gtf_path):
"""
_run_gffread: run gffread script
ref: http://cole-trapnell-lab.github.io/cufflinks/file_formats/#the-gffread-utility
"""
log('converting gff to gtf')
command = self.GFFREAD_TOOLKIT_PATH + '/gffread '
command += "-E {0} -T -o {1}".format(gff_path, gtf_path)
self._run_command(command)
def _create_gtf_annotation_from_genome(self, genome_ref):
"""
Create reference annotation file from genome
"""
ref = self.ws.get_object_subset([{
'ref':
genome_ref,
'included': ['contigset_ref', 'assembly_ref']
}])
if 'contigset_ref' in ref[0]['data']:
contig_id = ref[0]['data']['contigset_ref']
elif 'assembly_ref' in ref[0]['data']:
contig_id = ref[0]['data']['assembly_ref']
if contig_id is None:
raise ValueError(
"Genome at {0} does not have reference to the assembly object".
format(genome_ref))
print(contig_id)
log("Generating GFF file from Genome")
try:
ret = self.au.get_assembly_as_fasta({'ref': contig_id})
output_file = ret['path']
mapping_filename = c_mapping.create_sanitized_contig_ids(
output_file)
os.remove(output_file)
# get the GFF
ret = self.gfu.genome_to_gff({'genome_ref': genome_ref})
genome_gff_file = ret['file_path']
c_mapping.replace_gff_contig_ids(genome_gff_file,
mapping_filename,
to_modified=True)
gtf_ext = ".gtf"
if not genome_gff_file.endswith(gtf_ext):
gtf_path = os.path.splitext(genome_gff_file)[0] + '.gtf'
self._run_gffread(genome_gff_file, gtf_path)
else:
gtf_path = genome_gff_file
log("gtf file : " + gtf_path)
except Exception:
raise ValueError(
"Generating GTF file from Genome Annotation object Failed : {}"
.format("".join(traceback.format_exc())))
return gtf_path
def _get_gtf_file(self, alignment_ref):
"""
_get_gtf_file: get the reference annotation file (in GTF or GFF3 format)
"""
result_directory = self.scratch
alignment_data = self.ws.get_objects2(
{'objects': [{
'ref': alignment_ref
}]})['data'][0]['data']
genome_ref = alignment_data.get('genome_id')
# genome_name = self.ws.get_object_info([{"ref": genome_ref}], includeMetadata=None)[0][1]
# ws_gtf = genome_name+"_GTF_Annotation"
genome_data = self.ws.get_objects2({'objects': [{
'ref': genome_ref
}]})['data'][0]['data']
gff_handle_ref = genome_data.get('gff_handle_ref')
if gff_handle_ref:
log('getting reference annotation file from genome')
annotation_file = self.dfu.shock_to_file({
'handle_id': gff_handle_ref,
'file_path': result_directory,
'unpack': 'unpack'
})['file_path']
else:
annotation_file = self._create_gtf_annotation_from_genome(
genome_ref)
return annotation_file
def _get_gtf_file_from_genome_ref(self, genome_ref):
"""
_get_gtf_file: get the reference annotation file (in GTF or GFF3 format)
"""
result_directory = self.scratch
genome_data = self.ws.get_objects2({'objects': [{
'ref': genome_ref
}]})['data'][0]['data']
gff_handle_ref = genome_data.get('gff_handle_ref')
if gff_handle_ref:
log('getting reference annotation file from genome')
annotation_file = self.dfu.shock_to_file({
'handle_id': gff_handle_ref,
'file_path': result_directory,
'unpack': 'unpack'
})['file_path']
else:
annotation_file = self._create_gtf_annotation_from_genome(
genome_ref)
return annotation_file
def _get_input_file(self, alignment_ref):
"""
_get_input_file: get input BAM file from Alignment object
"""
bam_file_dir = self.rau.download_alignment(
{'source_ref': alignment_ref})['destination_dir']
files = os.listdir(bam_file_dir)
bam_file_list = [
file for file in files if re.match(r'.*\_sorted\.bam', file)
]
if not bam_file_list:
bam_file_list = [
file for file in files if re.match(r'.*(?<!sorted)\.bam', file)
]
if not bam_file_list:
raise ValueError('Cannot find .bam file from alignment {}'.format(
alignment_ref))
bam_file_name = bam_file_list[0]
bam_file = os.path.join(bam_file_dir, bam_file_name)
return bam_file
def _generate_command(self, params):
"""
_generate_command: generate cufflinks command
"""
cufflinks_command = '/opt/cufflinks/cufflinks'
cufflinks_command += (' -q --no-update-check -p ' +
str(params.get('num_threads', 1)))
if 'max_intron_length' in params and params[
'max_intron_length'] is not None:
cufflinks_command += (' --max-intron-length ' +
str(params['max_intron_length']))
if 'min_intron_length' in params and params[
'min_intron_length'] is not None:
cufflinks_command += (' --min-intron-length ' +
str(params['min_intron_length']))
if 'overhang_tolerance' in params and params[
'overhang_tolerance'] is not None:
cufflinks_command += (' --overhang-tolerance ' +
str(params['overhang_tolerance']))
cufflinks_command += " -o {0} -G {1} {2}".format(
params['result_directory'], params['gtf_file'],
params['input_file'])
log('Generated cufflinks command: {}'.format(cufflinks_command))
return cufflinks_command
def _process_rnaseq_alignment_object(self, params):
"""
_process_alignment_object: process KBaseRNASeq.RNASeqAlignment type input object
"""
log('start processing RNASeqAlignment object\nparams:\n{}'.format(
json.dumps(params, indent=1)))
alignment_ref = params.get('alignment_ref')
result_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(result_directory)
params['result_directory'] = str(result_directory)
# input files
params['input_file'] = self._get_input_file(alignment_ref)
if not params.get('gtf_file'):
params['gtf_file'] = self._get_gtf_file(alignment_ref)
if '/' not in params['genome_ref']:
params['genome_ref'] = params['workspace_name'] + '/' + params[
'genome_ref']
command = self._generate_command(params)
self._run_command(command)
expression_obj_ref = self._save_rnaseq_expression(
result_directory, alignment_ref, params.get('workspace_name'),
params.get('genome_ref'), params['gtf_file'],
params['expression_suffix'])
returnVal = {
'result_directory': result_directory,
'expression_obj_ref': expression_obj_ref,
'alignment_ref': alignment_ref
}
expression_name = self.ws.get_object_info([{
"ref": expression_obj_ref
}],
includeMetadata=None)[0][1]
widget_params = {
"output": expression_name,
"workspace": params.get('workspace_name')
}
returnVal.update(widget_params)
return returnVal
def _process_kbasesets_alignment_object(self, params):
"""
_process_alignment_object: process KBaseRNASeq.RNASeqAlignment type input object
"""
log('start processing KBaseSets object\nparams:\n{}'.format(
json.dumps(params, indent=1)))
alignment_ref = params.get('alignment_ref')
result_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(result_directory)
params['result_directory'] = str(result_directory)
# input files
params['input_file'] = self._get_input_file(alignment_ref)
if not params.get('gtf_file'):
params['gtf_file'] = self._get_gtf_file(alignment_ref)
command = self._generate_command(params)
self._run_command(command)
expression_obj_ref = self._save_kbasesets_expression(
result_directory, alignment_ref, params.get('workspace_name'),
params.get('genome_ref'), params.get('gtf_file'),
params.get('expression_suffix'))
returnVal = {
'result_directory': result_directory,
'expression_obj_ref': expression_obj_ref,
'alignment_ref': alignment_ref
}
expression_name = self.ws.get_object_info([{
"ref": expression_obj_ref
}],
includeMetadata=None)[0][1]
widget_params = {
"output": expression_name,
"workspace": params.get('workspace_name')
}
returnVal.update(widget_params)
return returnVal
def _generate_html_report(self, result_directory, obj_ref):
"""
_generate_html_report: generate html summary report
"""
log('Start generating html report')
html_report = list()
output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(output_directory)
result_file_path = os.path.join(output_directory, 'report.html')
expression_object = self.ws.get_objects2(
{'objects': [{
'ref': obj_ref
}]})['data'][0]
expression_object_type = expression_object.get('info')[2]
Overview_Content = ''
if re.match('KBaseRNASeq.RNASeqExpression-\d.\d',
expression_object_type):
Overview_Content += '<p>Generated Expression Object:</p><p>{}</p>'.format(
expression_object.get('info')[1])
elif re.match('KBaseRNASeq.RNASeqExpressionSet-\d.\d',
expression_object_type):
Overview_Content += '<p>Generated Expression Set Object:</p><p>{}</p>'.format(
expression_object.get('info')[1])
Overview_Content += '<br><p>Generated Expression Object:</p>'
for expression_ref in expression_object['data'][
'sample_expression_ids']:
expression_name = self.ws.get_object_info(
[{
"ref": expression_ref
}], includeMetadata=None)[0][1]
Overview_Content += '<p>{}</p>'.format(expression_name)
elif re.match('KBaseSets.ExpressionSet-\d.\d', expression_object_type):
pprint(expression_object)
Overview_Content += '<p>Generated Expression Set Object:</p><p>{}</p>'.format(
expression_object.get('info')[1])
Overview_Content += '<br><p>Generated Expression Object:</p>'
for expression_ref in expression_object['data']['items']:
expression_name = self.ws.get_object_info(
[{
"ref": expression_ref['ref']
}], includeMetadata=None)[0][1]
condition = expression_ref['label']
Overview_Content += '<p>condition:{0}; expression_name: {1}</p>'.format(
condition, expression_name)
with open(result_file_path, 'w') as result_file:
with open(
os.path.join(os.path.dirname(__file__),
'report_template.html'),
'r') as report_template_file:
report_template = report_template_file.read()
report_template = report_template.replace(
'<p>Overview_Content</p>', Overview_Content)
result_file.write(report_template)
html_report.append({
'path':
result_file_path,
'name':
os.path.basename(result_file_path),
'label':
os.path.basename(result_file_path),
'description':
'HTML summary report for Cufflinks App'
})
return html_report
def _save_rnaseq_expression(self, result_directory, alignment_ref,
workspace_name, genome_ref, gtf_file,
expression_suffix):
"""
_save_rnaseq_expression: save Expression object to workspace
"""
log('start saving Expression object')
alignment_object_name = self.ws.get_object_info(
[{
"ref": alignment_ref
}], includeMetadata=None)[0][1]
# set expression name
if re.match('.*_[Aa]lignment$', alignment_object_name):
expression_name = re.sub('_[Aa]lignment$', expression_suffix,
alignment_object_name)
else: # assume user specified suffix
expression_name = alignment_object_name + expression_suffix
expression_ref = self.eu.upload_expression({
'destination_ref':
workspace_name + '/' + expression_name,
'source_dir':
result_directory,
'alignment_ref':
alignment_ref,
'tool_used':
self.tool_used,
'tool_version':
self.tool_version
})['obj_ref']
return expression_ref
def _save_kbasesets_expression(self, result_directory, alignment_ref,
workspace_name, genome_ref, gtf_file,
expression_suffix):
"""
_save_kbasesets_expression: save Expression object to workspace using ExpressionUtils
and SetAPI
"""
log('start saving Expression object')
alignment_info = self.ws.get_object_info3(
{'objects': [{
"ref": alignment_ref
}]})
alignment_object_name = alignment_info['infos'][0][1]
# set expression name
if re.match('.*_[Aa]lignment$', alignment_object_name):
expression_name = re.sub('_[Aa]lignment$', expression_suffix,
alignment_object_name)
else: # assume user specified suffix
expression_name = alignment_object_name + expression_suffix
expression_ref = self.eu.upload_expression({
'destination_ref':
workspace_name + '/' + expression_name,
'source_dir':
result_directory,
'alignment_ref':
alignment_ref,
'tool_used':
self.tool_used,
'tool_version':
self.tool_version
})['obj_ref']
return expression_ref
def _save_rnaseq_expression_set(self, alignment_expression_map,
alignment_set_ref, workspace_name,
expression_set_name):
"""
_save_rnaseq_expression_set: save ExpressionSet object to workspace
"""
log('start saving ExpressionSet object')
if isinstance(workspace_name, int) or workspace_name.isdigit():
workspace_id = workspace_name
else:
workspace_id = self.dfu.ws_name_to_id(workspace_name)
expression_set_data = self._generate_expression_set_data(
alignment_expression_map, alignment_set_ref, expression_set_name)
object_type = 'KBaseRNASeq.RNASeqExpressionSet'
save_object_params = {
'id':
workspace_id,
'objects': [{
'type': object_type,
'data': expression_set_data,
'name': expression_set_name
}]
}
dfu_oi = self.dfu.save_objects(save_object_params)[0]
expression_set_ref = str(dfu_oi[6]) + '/' + str(dfu_oi[0]) + '/' + str(
dfu_oi[4])
return expression_set_ref
def _save_kbasesets_expression_set(self, alignment_expression_map,
alignment_set_ref, workspace_name,
expression_set_name):
"""
_save_kbasesets_expression_set: save ExpressionSet object to workspace
"""
log('start saving ExpressionSet object')
if isinstance(workspace_name, int) or workspace_name.isdigit():
workspace_id = workspace_name
else:
workspace_id = self.dfu.ws_name_to_id(workspace_name)
expression_set_data = self._generate_expression_set_data(
alignment_expression_map, alignment_set_ref, expression_set_name)
object_type = 'KBaseRNASeq.RNASeqExpressionSet'
save_object_params = {
'id':
workspace_id,
'objects': [{
'type': object_type,
'data': expression_set_data,
'name': expression_set_name
}]
}
dfu_oi = self.dfu.save_objects(save_object_params)[0]
expression_set_ref = str(dfu_oi[6]) + '/' + str(dfu_oi[0]) + '/' + str(
dfu_oi[4])
return expression_set_ref
def _generate_report(self,
obj_ref,
workspace_name,
result_directory,
exprMatrix_FPKM_ref=None,
exprMatrix_TPM_ref=None):
"""
_generate_report: generate summary report
"""
log('creating report')
output_files = self._generate_output_file_list(result_directory)
output_html_files = self._generate_html_report(result_directory,
obj_ref)
expression_object = self.ws.get_objects2(
{'objects': [{
'ref': obj_ref
}]})['data'][0]
expression_info = expression_object['info']
expression_data = expression_object['data']
expression_object_type = expression_info[2]
if re.match('KBaseRNASeq.RNASeqExpression-\d+.\d+',
expression_object_type):
objects_created = [{
'ref':
obj_ref,
'description':
'Expression generated by Cufflinks'
}]
elif re.match('KBaseRNASeq.RNASeqExpressionSet-\d+.\d+',
expression_object_type):
objects_created = [{
'ref':
obj_ref,
'description':
'Expression generated by Cufflinks'
}]
elif re.match('KBaseSets.ExpressionSet-\d+.\d+',
expression_object_type):
objects_created = [{
'ref':
obj_ref,
'description':
'ExpressionSet generated by Cufflinks'
}]
items = expression_data['items']
for item in items:
objects_created.append({
'ref':
item['ref'],
'description':
'Expression generated by Cufflinks'
})
objects_created.append({
'ref':
exprMatrix_FPKM_ref,
'description':
'FPKM ExpressionMatrix generated by Cufflinks'
})
objects_created.append({
'ref':
exprMatrix_TPM_ref,
'description':
'TPM ExpressionMatrix generated by Cufflinks'
})
report_params = {
'message': '',
'workspace_name': workspace_name,
'file_links': output_files,
'objects_created': objects_created,
'html_links': output_html_files,
'direct_html_link_index': 0,
'html_window_height': 366,
'report_object_name': 'kb_cufflinks_report_' + str(uuid.uuid4())
}
kbase_report_client = KBaseReport(self.callback_url, token=self.token)
output = kbase_report_client.create_extended_report(report_params)
report_output = {
'report_name': output['name'],
'report_ref': output['ref']
}
return report_output
def _parse_FPKMtracking(self, filename, metric):
result = {}
pos1 = 0
if metric == 'FPKM':
pos2 = 7
if metric == 'TPM':
pos2 = 8
with open(filename) as f:
next(f)
for line in f:
larr = line.split("\t")
if larr[pos1] != "":
try:
result[larr[pos1]] = math.log(float(larr[pos2]) + 1, 2)
except ValueError:
result[larr[pos1]] = math.log(1, 2)
return result
def _generate_output_file_list(self, result_directory):
"""
_generate_output_file_list: zip result files and generate file_links for report
"""
log('Start packing result files')
output_files = list()
output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(output_directory)
result_file = os.path.join(output_directory, 'cufflinks_result.zip')
with zipfile.ZipFile(result_file,
'w',
zipfile.ZIP_DEFLATED,
allowZip64=True) as zip_file:
for root, dirs, files in os.walk(result_directory):
for file in files:
if not (file.endswith('.DS_Store')):
zip_file.write(
os.path.join(root, file),
os.path.join(os.path.basename(root), file))
output_files.append({
'path': result_file,
'name': os.path.basename(result_file),
'label': os.path.basename(result_file),
'description': 'File(s) generated by Cufflinks App'
})
return output_files
def _generate_expression_data(self, result_directory, alignment_ref,
gtf_file, workspace_name, expression_suffix):
"""
_generate_expression_data: generate Expression object with cufflinks output files
"""
alignment_data_object = self.ws.get_objects2(
{'objects': [{
'ref': alignment_ref
}]})['data'][0]
# set expression name
alignment_object_name = alignment_data_object['info'][1]
if re.match('.*_[Aa]lignment$', alignment_object_name):
expression_name = re.sub('_[Aa]lignment$', expression_suffix,
alignment_object_name)
else: # assume user specified suffix
expression_name = alignment_object_name + expression_suffix
expression_data = {
'id': expression_name,
'type': 'RNA-Seq',
'numerical_interpretation': 'FPKM',
'processing_comments': 'log2 Normalized',
'tool_used': self.tool_used,
'tool_version': self.tool_version
}
alignment_data = alignment_data_object['data']
condition = alignment_data.get('condition')
expression_data.update({'condition': condition})
genome_id = alignment_data.get('genome_id')
expression_data.update({'genome_id': genome_id})
read_sample_id = alignment_data.get('read_sample_id')
expression_data.update(
{'mapped_rnaseq_alignment': {
read_sample_id: alignment_ref
}})
exp_dict, tpm_exp_dict = self.parse_FPKMtracking_calc_TPM(
os.path.join(result_directory, 'genes.fpkm_tracking'))
expression_data.update({'expression_levels': exp_dict})
expression_data.update({'tpm_expression_levels': tpm_exp_dict})
handle = self.dfu.file_to_shock({
'file_path': result_directory,
'pack': 'zip',
'make_handle': True
})['handle']
expression_data.update({'file': handle})
return expression_data
def _generate_expression_set_data(self, alignment_expression_map,
alignment_set_ref, expression_set_name):
"""
_generate_expression_set_data: generate ExpressionSet object with cufflinks output files
"""
alignment_set_data_object = self.ws.get_objects2(
{'objects': [{
'ref': alignment_set_ref
}]})['data'][0]
alignment_set_data = alignment_set_data_object['data']
expression_set_data = {
'tool_used': self.tool_used,
'tool_version': self.tool_version,
'id': expression_set_name,
'alignmentSet_id': alignment_set_ref,
'genome_id': alignment_set_data.get('genome_id'),
'sampleset_id': alignment_set_data.get('sampleset_id')
}
sample_expression_ids = []
mapped_expression_objects = []
mapped_expression_ids = []
for alignment_expression in alignment_expression_map:
alignment_ref = alignment_expression.get('alignment_ref')
expression_ref = alignment_expression.get('expression_obj_ref')
sample_expression_ids.append(expression_ref)
mapped_expression_ids.append({alignment_ref: expression_ref})
alignment_name = self.ws.get_object_info(
[{
"ref": alignment_ref
}], includeMetadata=None)[0][1]
expression_name = self.ws.get_object_info(
[{
"ref": expression_ref
}], includeMetadata=None)[0][1]
mapped_expression_objects.append({alignment_name: expression_name})
expression_set_data['sample_expression_ids'] = sample_expression_ids
expression_set_data[
'mapped_expression_objects'] = mapped_expression_objects
expression_set_data['mapped_expression_ids'] = mapped_expression_ids
return expression_set_data
def _process_alignment_set_object(self, params, alignment_object_type):
"""
_process_alignment_set_object: process KBaseRNASeq.RNASeqAlignmentSet type input object
and KBaseSets.ReadsAlignmentSet type object
"""
log('start processing KBaseRNASeq.RNASeqAlignmentSet object or KBaseSets.ReadsAlignmentSet object'
'\nparams:\n{}'.format(json.dumps(params, indent=1)))
alignment_set_ref = params.get('alignment_set_ref')
if re.match('^KBaseRNASeq.RNASeqAlignmentSet-\d*',
alignment_object_type):
params['gtf_file'] = self._get_gtf_file(alignment_set_ref)
else:
if not '/' in params['genome_ref']:
params['genome_ref'] = params['workspace_name'] + '/' + params[
'genome_ref']
params['gtf_file'] = self._get_gtf_file_from_genome_ref(
params['genome_ref'])
alignment_set = self.set_api.get_reads_alignment_set_v1({
'ref':
alignment_set_ref,
'include_item_info':
0,
'include_set_item_ref_paths':
1
})
mul_processor_params = []
for alignment in alignment_set["data"]["items"]:
alignment_ref = alignment['ref_path']
alignment_upload_params = params.copy()
alignment_upload_params['alignment_ref'] = alignment_ref
mul_processor_params.append(alignment_upload_params)
# use the following when you want to run the cmd sequentially
# self._process_kbasesets_alignment_object(mul_processor_params[0])
cpus = min(params.get('num_threads'), multiprocessing.cpu_count())
pool = Pool(ncpus=cpus)
log('running _process_alignment_object with {} cpus'.format(cpus))
alignment_expression_map = pool.map(
self._process_kbasesets_alignment_object, mul_processor_params)
result_directory = os.path.join(self.scratch, str(uuid.uuid4()))
self._mkdir_p(result_directory)
expression_items = list()
for proc_alignment_return in alignment_expression_map:
expression_obj_ref = proc_alignment_return.get(
'expression_obj_ref')
alignment_ref = proc_alignment_return.get('alignment_ref')
alignment_info = self.ws.get_object_info3({
'objects': [{
"ref": alignment_ref
}],
'includeMetadata':
1
})
condition = alignment_info['infos'][0][10]['condition']
expression_items.append({
"ref": expression_obj_ref,
"label": condition,
})
expression_name = self.ws.get_object_info(
[{
"ref": expression_obj_ref
}], includeMetadata=None)[0][1]
self._run_command('cp -R {} {}'.format(
proc_alignment_return.get('result_directory'),
os.path.join(result_directory, expression_name)))
expression_set = {
"description": "generated by kb_cufflinks",
"items": expression_items
}
expression_set_info = self.set_api.save_expression_set_v1({
"workspace":
params['workspace_name'],
"output_object_name":
params['expression_set_name'],
"data":
expression_set
})
returnVal = {
'result_directory': result_directory,
'expression_obj_ref': expression_set_info['set_ref']
}
widget_params = {
"output": params.get('expression_set_name'),
"workspace": params.get('workspace_name')
}
returnVal.update(widget_params)
return returnVal
def _generate_output_object_name(self, params, alignment_object_type,
alignment_object_name):
"""
Generates the output object name based on input object type and name and stores it in
params with key equal to 'expression' or 'expression_set' based on whether the input
object is an alignment or alignment_set.
:param params: module input params
:param alignment_object_type: input alignment object type
:param alignment_object_name: input alignment object name
:param alignment_object_data: input alignment object data
"""
expression_set_suffix = params['expression_set_suffix']
expression_suffix = params['expression_suffix']
if re.match('^KBaseRNASeq.RNASeqAlignment-\d*', alignment_object_type):
if re.match('.*_[Aa]lignment$', alignment_object_name):
params['expression_name'] = re.sub('_[Aa]lignment$',
expression_suffix,
alignment_object_name)
else: # assume user specified suffix
params[
'expression_name'] = alignment_object_name + expression_suffix
if re.match('^KBaseRNASeq.RNASeqAlignmentSet-\d*',
alignment_object_type):
if re.match('.*_[Aa]lignment_[Ss]et$', alignment_object_name):
# set expression set name
params['expression_set_name'] = re.sub('_[Aa]lignment_[Ss]et$',
expression_set_suffix,
alignment_object_name)
else: # assume user specified suffix
params[
'expression_set_name'] = alignment_object_name + expression_set_suffix
if re.match('^KBaseSets.ReadsAlignmentSet-\d*', alignment_object_type):
if re.match('.*_[Aa]lignment_[Ss]et$', alignment_object_name):
# set expression set name
params['expression_set_name'] = re.sub('_[Aa]lignment_[Ss]et$',
expression_set_suffix,
alignment_object_name)
else: # assume user specified suffix
params[
'expression_set_name'] = alignment_object_name + expression_set_suffix
def _save_expression_matrix(self, expressionset_ref, workspace_name):
"""
_save_expression_matrix: save FPKM and TPM ExpressionMatrix
"""
log('start saving ExpressionMatrix object')
expression_set_name = self.ws.get_object_info(
[{
"ref": expressionset_ref
}], includeMetadata=None)[0][1]
output_obj_name_prefix = re.sub('_*[Ee]xpression_*[Ss]et', '',
expression_set_name)
upload_expression_matrix_params = {
'expressionset_ref': expressionset_ref,
'output_obj_name': output_obj_name_prefix,
'workspace_name': workspace_name
}
expression_matrix_refs = self.eu.get_expressionMatrix(
upload_expression_matrix_params)
return expression_matrix_refs
def run_cufflinks_app(self, params):
log('--->\nrunning CufflinksUtil.run_cufflinks_app\n' +
'params:\n{}'.format(json.dumps(params, indent=1)))
self._validate_run_cufflinks_params(params)
alignment_object_ref = params.get('alignment_object_ref')
alignment_object_info = self.ws.get_object_info3(
{"objects": [{
"ref": alignment_object_ref
}]})['infos'][0]
alignment_object_type = alignment_object_info[2]
alignment_object_name = alignment_object_info[1]
# get output object name
self._generate_output_object_name(params, alignment_object_type,
alignment_object_name)
log('--->\nalignment object type: \n' +
'{}'.format(alignment_object_type))
if re.match('^KBaseRNASeq.RNASeqAlignment-\d*', alignment_object_type):
params.update({'alignment_ref': alignment_object_ref})
returnVal = self._process_rnaseq_alignment_object(params)
report_output = self._generate_report(
returnVal.get('expression_obj_ref'),
params.get('workspace_name'),
returnVal.get('result_directory'))
returnVal.update(report_output)
elif re.match('^KBaseRNASeq.RNASeqAlignmentSet-\d*', alignment_object_type) or \
re.match('^KBaseSets.ReadsAlignmentSet-\d*', alignment_object_type):
params.update({'alignment_set_ref': alignment_object_ref})
returnVal = self._process_alignment_set_object(
params, alignment_object_type)
expression_matrix_refs = self._save_expression_matrix(
returnVal['expression_obj_ref'], params.get('workspace_name'))
returnVal.update(expression_matrix_refs)
report_output = self._generate_report(
returnVal['expression_obj_ref'], params.get('workspace_name'),
returnVal['result_directory'],
expression_matrix_refs['exprMatrix_FPKM_ref'],
expression_matrix_refs['exprMatrix_TPM_ref'])
returnVal.update(report_output)
else:
raise ValueError(
'None RNASeqAlignment type\nObject info:\n{}'.format(
alignment_object_info))
return returnVal
class Bowtie2Aligner(object):
def __init__(self, scratch_dir, workspace_url, callback_url, srv_wiz_url,
provenance):
self.scratch_dir = scratch_dir
self.workspace_url = workspace_url
self.callback_url = callback_url
self.srv_wiz_url = srv_wiz_url
self.provenance = provenance
# from the provenance, extract out the version to run by exact hash if possible
self.my_version = 'release'
if len(provenance) > 0:
if 'subactions' in provenance[0]:
self.my_version = self.get_version_from_subactions(
'kb_Bowtie2', provenance[0]['subactions'])
print('Running kb_Bowtie2 version = ' + self.my_version)
self.ws = Workspace(self.workspace_url)
self.bowtie2 = Bowtie2Runner(self.scratch_dir)
self.parallel_runner = KBParallel(self.callback_url)
self.qualimap = kb_QualiMap(self.callback_url)
def get_version_from_subactions(self, module_name, subactions):
# go through each sub action looking for
if not subactions:
return 'release' # default to release if we can't find anything
for sa in subactions:
if 'name' in sa:
if sa['name'] == module_name:
# local-docker-image implies that we are running in kb-test, so return 'dev'
if sa['commit'] == 'local-docker-image':
return 'dev'
# to check that it is a valid hash, make sure it is the right
# length and made up of valid hash characters
if re.match('[a-fA-F0-9]{40}$', sa['commit']):
return sa['commit']
# again, default to setting this to release
return 'release'
def align(self, params):
validated_params = self.validate_params(params)
input_info = self.determine_input_info(validated_params)
# input info provides information on the input and tells us if we should
# run as a single_library or as a set:
# input_info = {'run_mode': '', 'info': [..], 'ref': '55/1/2'}
assembly_or_genome_ref = validated_params['assembly_or_genome_ref']
if input_info['run_mode'] == 'single_library':
if 'output_alignment_name' not in validated_params:
suffix = '_alignment'
if 'output_alignment_suffix' in validated_params:
suffix = validated_params['output_alignment_suffix']
validated_params[
'output_alignment_name'] = input_info['info'][1] + suffix
single_lib_result = self.single_reads_lib_run(
input_info,
assembly_or_genome_ref,
validated_params,
create_report=validated_params['create_report'])
return single_lib_result
if input_info['run_mode'] == 'sample_set':
reads = self.fetch_reads_refs_from_sampleset(
input_info['ref'], input_info['info'], validated_params)
self.build_bowtie2_index(assembly_or_genome_ref,
validated_params['output_workspace'])
print('Running on set of reads=')
pprint(reads)
tasks = []
for r in reads:
tasks.append(
self.build_single_execution_task(
r['ref'], params, r['alignment_output_name'],
r['condition']))
batch_run_params = {
'tasks': tasks,
'runner': 'parallel',
'max_retries': 2
}
if validated_params['concurrent_local_tasks'] is not None:
batch_run_params['concurrent_local_tasks'] = validated_params[
'concurrent_local_tasks']
if validated_params['concurrent_njsw_tasks'] is not None:
batch_run_params['concurrent_njsw_tasks'] = validated_params[
'concurrent_njsw_tasks']
results = self.parallel_runner.run_batch(batch_run_params)
print('Batch run results=')
pprint(results)
batch_result = self.process_batch_result(results, validated_params,
reads, input_info['info'])
return batch_result
raise ('Improper run mode')
def build_single_execution_task(self, reads_lib_ref, params, output_name,
condition):
task_params = copy.deepcopy(params)
task_params['input_ref'] = reads_lib_ref
task_params['output_alignment_name'] = output_name
task_params['create_report'] = 0
task_params['condition_label'] = condition
return {
'module_name': 'kb_Bowtie2',
'function_name': 'align_reads_to_assembly_app',
'version': self.my_version,
'parameters': task_params
}
def single_reads_lib_run(self,
read_lib_info,
assembly_or_genome_ref,
validated_params,
create_report=False,
bowtie2_index_info=None):
''' run on one reads '''
# download reads and prepare any bowtie2 index files
input_configuration = self.prepare_single_run(
read_lib_info, assembly_or_genome_ref, bowtie2_index_info,
validated_params['output_workspace'])
# run the actual program
run_output_info = self.run_bowtie2_align_cli(input_configuration,
validated_params)
# process the result and save the output
upload_results = self.save_read_alignment_output(
run_output_info, input_configuration, validated_params)
run_output_info['upload_results'] = upload_results
report_info = None
if create_report:
report_info = self.create_report_for_single_run(
run_output_info, input_configuration, validated_params)
self.clean(run_output_info)
return {'output_info': run_output_info, 'report_info': report_info}
def build_bowtie2_index(self, assembly_or_genome_ref, ws_for_cache):
bowtie2IndexBuilder = Bowtie2IndexBuilder(self.scratch_dir,
self.workspace_url,
self.callback_url,
self.srv_wiz_url,
self.provenance)
return bowtie2IndexBuilder.get_index({
'ref': assembly_or_genome_ref,
'ws_for_cache': ws_for_cache
})
def prepare_single_run(self, input_info, assembly_or_genome_ref,
bowtie2_index_info, ws_for_cache):
''' Given a reads ref and an assembly, setup the bowtie2 index '''
# first setup the bowtie2 index of the assembly
input_configuration = {'bowtie2_index_info': bowtie2_index_info}
if not bowtie2_index_info:
bowtie2IndexBuilder = Bowtie2IndexBuilder(self.scratch_dir,
self.workspace_url,
self.callback_url,
self.srv_wiz_url,
self.provenance)
index_result = bowtie2IndexBuilder.get_index({
'ref':
assembly_or_genome_ref,
'ws_for_cache':
ws_for_cache
})
input_configuration['bowtie2_index_info'] = index_result
# next download the reads
read_lib_ref = input_info['ref']
read_lib_info = input_info['info']
reads_params = {
'read_libraries': [read_lib_ref],
'interleaved': 'false',
'gzipped': None
}
ru = ReadsUtils(self.callback_url)
reads = ru.download_reads(reads_params)['files']
input_configuration['reads_lib_type'] = self.get_type_from_obj_info(
read_lib_info).split('.')[1]
input_configuration['reads_files'] = reads[read_lib_ref]
input_configuration['reads_lib_ref'] = read_lib_ref
return input_configuration
def run_bowtie2_align_cli(self, input_configuration, validated_params):
# pprint('======== input_configuration =====')
# pprint(input_configuration)
options = []
run_output_info = {}
# set the bowtie2 index location
bt2_index_dir = input_configuration['bowtie2_index_info']['output_dir']
bt2_index_basename = input_configuration['bowtie2_index_info'][
'index_files_basename']
options.extend(['-x', bt2_index_basename])
# set the input reads
if input_configuration['reads_lib_type'] == 'SingleEndLibrary':
options.extend(
['-U', input_configuration['reads_files']['files']['fwd']])
run_output_info['library_type'] = 'single_end'
elif input_configuration['reads_lib_type'] == 'PairedEndLibrary':
options.extend(
['-1', input_configuration['reads_files']['files']['fwd']])
options.extend(
['-2', input_configuration['reads_files']['files']['rev']])
run_output_info['library_type'] = 'paired_end'
# setup the output file name
output_dir = os.path.join(
self.scratch_dir,
'bowtie2_alignment_output_' + str(int(time.time() * 10000)))
output_sam_file = os.path.join(output_dir, 'reads_alignment.sam')
os.makedirs(output_dir)
options.extend(['-S', output_sam_file])
run_output_info['output_sam_file'] = output_sam_file
run_output_info['output_dir'] = output_dir
# parse all the other parameters
if 'quality_score' in validated_params:
options.append('--' + str(validated_params['quality_score']))
if 'alignment_type' in validated_params:
options.append('--' + str(validated_params['alignment_type']))
if 'preset_options' in validated_params:
if 'alignment_type' in validated_params and validated_params[
'alignment_type'] == 'local':
options.append('--' + str(validated_params['preset_options'] +
'-local'))
else:
options.append('--' + str(validated_params['preset_options']))
if 'trim5' in validated_params:
options.extend(['--trim5', str(validated_params['trim5'])])
if 'trim3' in validated_params:
options.extend(['--trim3', str(validated_params['trim3'])])
if 'np' in validated_params:
options.extend(['--np', str(validated_params['np'])])
if 'minins' in validated_params:
options.extend(['--minins', str(validated_params['minins'])])
if 'maxins' in validated_params:
options.extend(['--maxins', str(validated_params['maxins'])])
# unfortunately, bowtie2 expects the index files to be in the current directory, and
# you cannot configure it otherwise. So run bowtie out of the index directory, but
# place the output SAM file somewhere else
self.bowtie2.run('bowtie2', options, cwd=bt2_index_dir)
return run_output_info
def save_read_alignment_output(self, run_output_info, input_configuration,
validated_params):
rau = ReadsAlignmentUtils(self.callback_url)
destination_ref = validated_params[
'output_workspace'] + '/' + validated_params[
'output_alignment_name']
condition = 'unknown'
if 'condition_label' in validated_params:
condition = validated_params['condition_label']
upload_params = {
'file_path': run_output_info['output_sam_file'],
'destination_ref': destination_ref,
'read_library_ref': input_configuration['reads_lib_ref'],
'assembly_or_genome_ref':
validated_params['assembly_or_genome_ref'],
'condition': condition
}
upload_results = rau.upload_alignment(upload_params)
return upload_results
def clean(self, run_output_info):
''' Not really necessary on a single run, but if we are running multiple local subjobs, we
should clean up files that have already been saved back up to kbase '''
pass
def create_report_for_single_run(self, run_output_info,
input_configuration, validated_params):
# first run qualimap
qualimap_report = self.qualimap.run_bamqc(
{'input_ref': run_output_info['upload_results']['obj_ref']})
qc_result_zip_info = qualimap_report['qc_result_zip_info']
# create report
report_text = 'Ran on a single reads library.\n\n'
alignment_info = self.get_obj_info(
run_output_info['upload_results']['obj_ref'])
report_text = 'Created ReadsAlignment: ' + str(
alignment_info[1]) + '\n'
report_text = ' ' + run_output_info[
'upload_results']['obj_ref'] + '\n'
kbr = KBaseReport(self.callback_url)
report_info = kbr.create_extended_report({
'message':
report_text,
'objects_created': [{
'ref':
run_output_info['upload_results']['obj_ref'],
'description':
'ReadsAlignment'
}],
'report_object_name':
'kb_Bowtie2_' + str(uuid.uuid4()),
'direct_html_link_index':
0,
'html_links': [{
'shock_id': qc_result_zip_info['shock_id'],
'name': qc_result_zip_info['index_html_file_name'],
'label': qc_result_zip_info['name']
}],
'workspace_name':
validated_params['output_workspace']
})
return {
'report_name': report_info['name'],
'report_ref': report_info['ref']
}
def process_batch_result(self, batch_result, validated_params, reads,
input_set_info):
n_jobs = len(batch_result['results'])
n_success = 0
n_error = 0
ran_locally = 0
ran_njsw = 0
# reads alignment set items
items = []
objects_created = []
for k in range(0, len(batch_result['results'])):
job = batch_result['results'][k]
result_package = job['result_package']
if job['is_error']:
n_error += 1
else:
n_success += 1
output_info = result_package['result'][0]['output_info']
ra_ref = output_info['upload_results']['obj_ref']
# Note: could add a label to the alignment here?
items.append({'ref': ra_ref, 'label': reads[k]['condition']})
objects_created.append({'ref': ra_ref})
if result_package['run_context']['location'] == 'local':
ran_locally += 1
if result_package['run_context']['location'] == 'njsw':
ran_njsw += 1
# Save the alignment set
alignment_set_data = {'description': '', 'items': items}
alignment_set_save_params = {
'data':
alignment_set_data,
'workspace':
validated_params['output_workspace'],
'output_object_name':
str(input_set_info[1]) + validated_params['output_obj_name_suffix']
}
set_api = SetAPI(self.srv_wiz_url)
save_result = set_api.save_reads_alignment_set_v1(
alignment_set_save_params)
print('Saved ReadsAlignment=')
pprint(save_result)
objects_created.append({
'ref':
save_result['set_ref'],
'description':
'Set of all reads alignments generated'
})
set_name = save_result['set_info'][1]
# run qualimap
qualimap_report = self.qualimap.run_bamqc(
{'input_ref': save_result['set_ref']})
qc_result_zip_info = qualimap_report['qc_result_zip_info']
# create the report
report_text = 'Ran on SampleSet or ReadsSet.\n\n'
report_text = 'Created ReadsAlignmentSet: ' + str(set_name) + '\n\n'
report_text += 'Total ReadsLibraries = ' + str(n_jobs) + '\n'
report_text += ' Successful runs = ' + str(n_success) + '\n'
report_text += ' Failed runs = ' + str(n_error) + '\n'
report_text += ' Ran on main node = ' + str(ran_locally) + '\n'
report_text += ' Ran on remote worker = ' + str(ran_njsw) + '\n\n'
print('Report text=')
print(report_text)
kbr = KBaseReport(self.callback_url)
report_info = kbr.create_extended_report({
'message':
report_text,
'objects_created':
objects_created,
'report_object_name':
'kb_Bowtie2_' + str(uuid.uuid4()),
'direct_html_link_index':
0,
'html_links': [{
'shock_id': qc_result_zip_info['shock_id'],
'name': qc_result_zip_info['index_html_file_name'],
'label': qc_result_zip_info['name']
}],
'workspace_name':
validated_params['output_workspace']
})
result = {
'report_info': {
'report_name': report_info['name'],
'report_ref': report_info['ref']
}
}
result['batch_output_info'] = batch_result
return result
def validate_params(self, params):
validated_params = {}
required_string_fields = [
'input_ref', 'assembly_or_genome_ref', 'output_obj_name_suffix',
'output_workspace'
]
for field in required_string_fields:
if field in params and params[field]:
validated_params[field] = params[field]
else:
raise ValueError('"' + field +
'" field required to run bowtie2 aligner app')
optional_fields = [
'quality_score', 'alignment_type', 'preset_options', 'trim5',
'trim3', 'condition_label', 'np', 'minins', 'maxins',
'output_alignment_suffix', 'output_alignment_name'
]
for field in optional_fields:
if field in params:
if params[field] is not None:
validated_params[field] = params[field]
validated_params['create_report'] = True
if 'create_report' in params and params['create_report'] is not None:
if int(params['create_report']) == 1:
validated_params['create_report'] = True
elif int(params['create_report']) == 0:
validated_params['create_report'] = False
else:
raise ValueError(
'"create_report" field, if present, should be set to a boolean value: 0 or 1'
)
validated_params['concurrent_local_tasks'] = None
validated_params['concurrent_njsw_tasks'] = None
if 'concurrent_local_tasks' in params and params[
'concurrent_local_tasks'] is not None:
validated_params['concurrent_local_tasks'] = int(
params['concurrent_local_tasks'])
if 'concurrent_njsw_tasks' in params and params[
'concurrent_njsw_tasks'] is not None:
validated_params['concurrent_njsw_tasks'] = int(
params['concurrent_njsw_tasks'])
return validated_params
def fetch_reads_refs_from_sampleset(self, ref, info, validated_params):
"""
Note: adapted from kbaseapps/kb_hisat2 - file_util.py
From the given object ref, return a list of all reads objects that are a part of that
object. E.g., if ref is a ReadsSet, return a list of all PairedEndLibrary or SingleEndLibrary
refs that are a member of that ReadsSet. This is returned as a list of dictionaries as follows:
{
"ref": reads object reference,
"condition": condition string associated with that reads object
}
The only one required is "ref", all other keys may or may not be present, based on the reads
object or object type in initial ref variable. E.g. a RNASeqSampleSet might have condition info
for each reads object, but a single PairedEndLibrary may not have that info.
If ref is already a Reads library, just returns a list with ref as a single element.
"""
obj_type = self.get_type_from_obj_info(info)
refs = list()
refs_for_ws_info = list()
if "KBaseSets.ReadsSet" in obj_type:
print("Looking up reads references in ReadsSet object")
set_api = SetAPI(self.srv_wiz_url)
reads_set = set_api.get_reads_set_v1({
'ref': ref,
'include_item_info': 0
})
for reads in reads_set["data"]["items"]:
refs.append({'ref': reads['ref'], 'condition': reads['label']})
refs_for_ws_info.append({'ref': reads['ref']})
elif "KBaseRNASeq.RNASeqSampleSet" in obj_type:
print("Looking up reads references in RNASeqSampleSet object")
sample_set = self.ws.get_objects2({"objects": [{
"ref": ref
}]})["data"][0]["data"]
for i in range(len(sample_set["sample_ids"])):
refs.append({
'ref': sample_set["sample_ids"][i],
'condition': sample_set["condition"][i]
})
refs_for_ws_info.append({'ref': sample_set["sample_ids"][i]})
else:
raise ValueError("Unable to fetch reads reference from object {} "
"which is a {}".format(ref, obj_type))
# get object info so we can name things properly
infos = self.ws.get_object_info3({'objects':
refs_for_ws_info})['infos']
name_ext = '_alignment'
if 'output_alignment_suffix' in validated_params \
and validated_params['output_alignment_suffix'] is not None:
ext = validated_params['output_alignment_suffix'].replace(' ', '')
if ext:
name_ext = ext
unique_name_lookup = {}
for k in range(0, len(refs)):
refs[k]['info'] = infos[k]
name = infos[k][1]
if name not in unique_name_lookup:
unique_name_lookup[name] = 1
else:
unique_name_lookup[name] += 1
name = name + '_' + str(unique_name_lookup[name])
name = name + name_ext
refs[k]['alignment_output_name'] = name
return refs
def determine_input_info(self, validated_params):
''' get info on the input_ref object and determine if we run once or run on a set '''
info = self.get_obj_info(validated_params['input_ref'])
obj_type = self.get_type_from_obj_info(info)
if obj_type in [
'KBaseAssembly.PairedEndLibrary',
'KBaseAssembly.SingleEndLibrary', 'KBaseFile.PairedEndLibrary',
'KBaseFile.SingleEndLibrary'
]:
return {
'run_mode': 'single_library',
'info': info,
'ref': validated_params['input_ref']
}
if obj_type == 'KBaseRNASeq.RNASeqSampleSet':
return {
'run_mode': 'sample_set',
'info': info,
'ref': validated_params['input_ref']
}
if obj_type == 'KBaseSets.ReadsSet':
return {
'run_mode': 'sample_set',
'info': info,
'ref': validated_params['input_ref']
}
raise ValueError('Object type of input_ref is not valid, was: ' +
str(obj_type))
def get_type_from_obj_info(self, info):
return info[2].split('-')[0]
def get_obj_info(self, ref):
return self.ws.get_object_info3({'objects': [{
'ref': ref
}]})['infos'][0]
