def get_authors(config, wsid):
ws = Workspace(url=config.narrative_session.ws_url,
token=config.narrative_session.token)
ws_info = ws.get_workspace_info({"id": wsid})
author_id_list = [ws_info[2]]
other_authors = ws.get_permissions({"id": wsid})
for author in sorted(other_authors.keys()):
if author != "*" and other_authors[author] in [
"w", "a"
] and author not in author_id_list:
author_id_list.append(author)
auth = _KBaseAuth(config.narrative_session.auth_url)
disp_names = auth.get_display_names(config.narrative_session.token,
author_id_list)
author_list = []
for author in author_id_list:
author_list.append({
"id":
author,
"name":
html.escape(disp_names.get(author, author)),
"path":
config.narrative_session.profile_page_url + author
})
return author_list
def fetch_narrative_data(endpt: str, token: str, ws_id: int,
outdir: str) -> int:
ws = Workspace(url=endpt + "ws", token=token)
ws_info = ws.get_workspace_info({"id": ws_id})
ws_meta = ws_info[8]
# Narrative object
narr_id = ws_meta["narrative"]
narr_obj = ws.get_objects2({"objects": [{
"ref": f"{ws_id}/{narr_id}"
}]})["data"][0]
narr_ver = narr_obj["info"][4]
narr_outpath = os.path.join(
outdir, f"narrative-{ws_id}.{narr_id}.{narr_ver}.json")
with open(narr_outpath, "w") as fout:
json.dump(narr_obj, fout, indent=4)
# Report objects
for cell in narr_obj["data"]["cells"]:
if "kbase" in cell["metadata"]:
meta = cell["metadata"]["kbase"]
if "appCell" in meta:
job_state = meta["appCell"].get("exec", {}).get("jobState")
result = list()
if "result" in job_state:
result = job_state["result"]
elif "job_output" in job_state and "result" in job_state[
"job_output"]:
result = job_state["job_output"]["result"]
if len(result) > 0 and "report_ref" in result[0]:
report_data = ws.get_objects2(
{"objects": [{
"ref": result[0]["report_ref"]
}]})["data"][0]
report_info = report_data["info"]
ref_dots = f"{report_info[6]}.{report_info[0]}.{report_info[4]}"
report_path = os.path.join(outdir,
f"report-{ref_dots}.json")
with open(report_path, "w") as fout:
json.dump(report_data, fout, indent=4)
# List objects results
service = NarrativeService(url=endpt + "service_wizard", token=token)
# service = ServiceClient(url=endpt + "service_wizard", use_url_lookup=True, token=token)
ws_data = service.list_objects_with_sets({
"ws_id": ws_id,
"includeMetadata": 1
})
# ws_data = service.sync_call(
# "NarrativeService.list_objects_with_sets",
# [{"ws_id": ws_id, "includeMetadata": 1}]
# )[0]
data_outpath = os.path.join(outdir, f"objects-{ws_id}.json")
with open(data_outpath, "w") as fout:
json.dump(ws_data, fout, indent=4)
return 0
def get_static_info(ws_url: str, token: str, ws_id: int) -> Dict:
"""
Looks up the static narrative info for the given Workspace id.
That info is stashed in the Workspace metadata, so that gets fetched, munged into a structure,
and returned.
If there's no static narrative, this returns an empty structure, as there's no info.
If ws_id is not present, or not numeric, raises a ValueError.
If there's a problem when contacting the Workspace (anything that raises a ServerError),
this raises a WorkspaceError.
:param ws_url: the URL for the workspace endpoint
:param token: the user auth token
:param ws_id: the workspace id of the narrative to fetch info for.
:returns: a dictionary with the following keys if a static narrative is present:
ws_id - int - the workspace id
narrative_id - int - the id of the narrative object
version - int - the version of the narrative object made static
url - str - the url of the static narrative
narr_saved - int - the timestamp of when the narrative that the static version is
based on was saved (ms since epoch)
static_saved - int - the timestamp of when the static narrative was saved (ms
since epoch)
"""
if not ws_id or not str(ws_id).isdigit():
raise ValueError(f"The parameter ws_id must be an integer, not {ws_id}")
ws_client = Workspace(url=ws_url, token=token)
try:
ws_info = ws_client.get_workspace_info({"id": ws_id})
except ServerError as err:
raise WorkspaceError(err, ws_id)
info = {}
meta = ws_info[8]
if "static_narrative_ver" in meta:
info = {
"ws_id": ws_id,
"version": int(meta["static_narrative_ver"]),
"narrative_id": int(meta["narrative"]),
"url": meta["static_narrative"],
"static_saved": int(meta["static_narrative_saved"])
}
try:
obj_info = ws_client.get_object_info3({
"objects": [{
"ref": f"{ws_id}/{info['narrative_id']}/{info['version']}"
}]
})
except ServerError as err:
raise WorkspaceError(err, ws_id)
ts = date_parser.isoparse(obj_info["infos"][0][3]).timestamp()
info["narr_saved"] = int(ts*1000)
return info
class WorkspaceAdminUtils:
def __init__(self, config):
wsurl = config.get('workspace-url')
self.atoken = config.get('workspace-admin-token')
self.noadmin = False
if self.atoken is None or self.atoken == '':
self.noadmin = True
self.atoken = os.environ.get('KB_AUTH_TOKEN', None)
self.ws = Workspace(wsurl, token=self.atoken)
def list_objects(self, params):
"""
Provide something that acts like a standard listObjects
"""
if self.noadmin:
return self.ws.list_objects(params)
return self.ws.administer({'command': 'listObjects', 'params': params})
def get_objects2(self, params):
"""
Provide something that acts like a standard getObjects
"""
if self.noadmin:
return self.ws.get_objects2(params)
return self.ws.administer({'command': 'getObjects', 'params': params})
def get_workspace_info(self, params):
"""
Provide something that acts like a standard getObjects
"""
if self.noadmin:
return self.ws.get_workspace_info(params)
return self.ws.administer({
'command': 'getWorkspaceInfo',
'params': params
})
def run_poolfileupload(self, ctx, params):
"""
This example function accepts any number of parameters and returns results in a KBaseReport
:param params: instance of mapping from String to unspecified object
:returns: instance of type "ReportResults" -> structure: parameter
"report_name" of String, parameter "report_ref" of String
"""
# ctx is the context object
# return variables are: output
#BEGIN run_poolfileupload
params['shared_folder'] = self.shared_folder
token = os.environ.get('KB_AUTH_TOKEN', None)
ws = Workspace(self.ws_url, token=token)
params['workspace_id'] = ws.get_workspace_info(
{'workspace': params['workspace_name']})[0]
params['ws_obj'] = ws
params['username'] = ctx['user_id']
params['output_name'] = check_output_name(params['output_name'])
if 'pool_file_type' not in params:
raise Exception("Did not get param pool_file_type")
else:
pft = params['pool_file_type']
if pft == 'poolfile':
pfu = poolfileuploadUtil(params)
result = pfu.upload_poolfile()
elif pft == 'poolcount':
pcfu = poolcountfileuploadUtil(params)
result = pcfu.upload_poolcountfile()
elif pft == 'experiments':
expsfu = expsfileuploadUtil(params)
result = expsfu.upload_expsfile()
else:
raise Exception("Did not recognize pool_file_type for upload")
text_message = "Finished uploading file \n"
text_message += "{} saved as {} on {}\n".format(
result['Name'], result['Type'], result['Date'])
report = KBaseReport(self.callback_url)
report_info = report.create({
'report': {
'objects_created': [],
'text_message': text_message
},
'workspace_name': params['workspace_name']
})
output = {
'report_name': report_info['name'],
'report_ref': report_info['ref'],
}
#END run_poolfileupload
# At some point might do deeper type checking...
if not isinstance(output, dict):
raise ValueError('Method run_poolfileupload return value ' +
'output is not type dict as required.')
# return the results
return [output]
class NarrativeManager:
KB_CELL = 'kb-cell'
KB_TYPE = 'type'
KB_APP_CELL = 'kb_app'
KB_FUNCTION_CELL = 'function_input'
KB_OUTPUT_CELL = 'function_output'
KB_ERROR_CELL = 'kb_error'
KB_CODE_CELL = 'kb_code'
KB_STATE = 'widget_state'
DEBUG = False
DATA_PALETTES_TYPES = DataPaletteTypes(False)
def __init__(self, config, ctx, set_api_client, data_palette_client):
self.narrativeMethodStoreURL = config['narrative-method-store']
self.set_api_cache = set_api_client # DynamicServiceCache type
self.data_palette_client = data_palette_client # DynamicServiceCache type
self.token = ctx["token"]
self.user_id = ctx["user_id"]
self.ws = Workspace(config['workspace-url'], token=self.token)
self.intro_md_file = config['intro-markdown-file']
# We switch DPs on only for internal Continuous Integration environment for now:
if config['kbase-endpoint'].startswith("https://ci.kbase.us/") or \
'USE_DP' in os.environ:
self.DATA_PALETTES_TYPES = DataPaletteTypes(True)
def list_objects_with_sets(self,
ws_id=None,
ws_name=None,
workspaces=None,
types=None,
include_metadata=0,
include_data_palettes=0):
if not workspaces:
if not ws_id and not ws_name:
raise ValueError(
"One and only one of 'ws_id', 'ws_name', 'workspaces' " +
"parameters should be set")
workspaces = [self._get_workspace_name_or_id(ws_id, ws_name)]
return self._list_objects_with_sets(workspaces, types,
include_metadata,
include_data_palettes)
def _list_objects_with_sets(self, workspaces, types, include_metadata,
include_data_palettes):
type_map = None
if types is not None:
type_map = {key: True for key in types}
processed_refs = {}
data = []
if self.DEBUG:
print("NarrativeManager._list_objects_with_sets: processing sets")
t1 = time.time()
set_ret = self.set_api_cache.call_method(
"list_sets", [{
'workspaces': workspaces,
'include_set_item_info': 1,
'include_metadata': include_metadata
}], self.token)
sets = set_ret['sets']
for set_info in sets:
# Process
target_set_items = []
for set_item in set_info['items']:
target_set_items.append(set_item['info'])
if self._check_info_type(set_info['info'], type_map):
data_item = {
'object_info': set_info['info'],
'set_items': {
'set_items_info': target_set_items
}
}
data.append(data_item)
processed_refs[set_info['ref']] = data_item
if self.DEBUG:
print(" (time=" + str(time.time() - t1) + ")")
if self.DEBUG:
print("NarrativeManager._list_objects_with_sets: loading ws_info")
t2 = time.time()
ws_info_list = []
# for ws in workspaces:
if len(workspaces) == 1:
ws = workspaces[0]
ws_id = None
ws_name = None
if str(ws).isdigit():
ws_id = int(ws)
else:
ws_name = str(ws)
ws_info_list.append(
self.ws.get_workspace_info({
"id": ws_id,
"workspace": ws_name
}))
else:
ws_map = {key: True for key in workspaces}
for ws_info in self.ws.list_workspace_info({'perm': 'r'}):
if ws_info[1] in ws_map or str(ws_info[0]) in ws_map:
ws_info_list.append(ws_info)
if self.DEBUG:
print(" (time=" + str(time.time() - t2) + ")")
if self.DEBUG:
print(
"NarrativeManager._list_objects_with_sets: loading workspace objects"
)
t3 = time.time()
for info in WorkspaceListObjectsIterator(
self.ws,
ws_info_list=ws_info_list,
list_objects_params={'includeMetadata': include_metadata}):
item_ref = str(info[6]) + '/' + str(info[0]) + '/' + str(info[4])
if item_ref not in processed_refs and self._check_info_type(
info, type_map):
data_item = {'object_info': info}
data.append(data_item)
processed_refs[item_ref] = data_item
if self.DEBUG:
print(" (time=" + str(time.time() - t3) + ")")
return_data = {"data": data}
if include_data_palettes == 1:
if self.DEBUG:
print(
"NarrativeManager._list_objects_with_sets: processing DataPalettes"
)
t5 = time.time()
dp_ret = self.data_palette_client.call_method(
"list_data", [{
'workspaces': workspaces,
'include_metadata': include_metadata
}], self.token)
for item in dp_ret['data']:
ref = item['ref']
if self._check_info_type(item['info'], type_map):
data_item = None
if ref in processed_refs:
data_item = processed_refs[ref]
else:
data_item = {'object_info': item['info']}
processed_refs[ref] = data_item
data.append(data_item)
dp_info = {}
if 'dp_ref' in item:
dp_info['ref'] = item['dp_ref']
if 'dp_refs' in item:
dp_info['refs'] = item['dp_refs']
data_item['dp_info'] = dp_info
return_data["data_palette_refs"] = dp_ret['data_palette_refs']
if self.DEBUG:
print(" (time=" + str(time.time() - t5) + ")")
return return_data
def _check_info_type(self, info, type_map):
if type_map is None:
return True
obj_type = info[2].split('-')[0]
return type_map.get(obj_type, False)
def copy_narrative(self, newName, workspaceRef, workspaceId):
time_ms = int(round(time.time() * 1000))
newWsName = self.user_id + ':narrative_' + str(time_ms)
# add the 'narrative' field to newWsMeta later.
newWsMeta = {"narrative_nice_name": newName, "searchtags": "narrative"}
# start with getting the existing narrative object.
currentNarrative = self.ws.get_objects([{'ref': workspaceRef}])[0]
if not workspaceId:
workspaceId = currentNarrative['info'][6]
# Let's prepare exceptions for clone the workspace.
# 1) currentNarrative object:
excluded_list = [{'objid': currentNarrative['info'][0]}]
# 2) let's exclude objects of types under DataPalette handling:
## DP CODE
# data_palette_type = "DataPalette.DataPalette"
# excluded_types = [data_palette_type]
# excluded_types.extend(self.DATA_PALETTES_TYPES.keys())
# add_to_palette_list = []
# dp_detected = False
## END DP CODE
# for obj_type in excluded_types:
# list_objects_params = {'type': obj_type}
## DP CODE
# if obj_type == data_palette_type:
# list_objects_params['showHidden'] = 1
## END DP CODE
# for info in WorkspaceListObjectsIterator(self.ws,
# ws_id=workspaceId,
# list_objects_params=list_objects_params):
## DP CODE
# if obj_type == data_palette_type:
# dp_detected = True
# else:
# add_to_palette_list.append({
# 'ref': str(info[6]) + '/' + str(info[0]) + '/' + str(info[4])
# })
## END DP CODE
# excluded_list.append({'objid': info[0]})
# clone the workspace EXCEPT for currentNarrative object
newWsId = self.ws.clone_workspace({
'wsi': {
'id': workspaceId
},
'workspace': newWsName,
'meta': newWsMeta,
'exclude': excluded_list
})[0]
try:
## DP CODE
# if dp_detected:
# self.data_palette_client.call_method(
# "copy_palette",
# [{'from_workspace': str(workspaceId), 'to_workspace': str(newWsId)}],
# self.token
# )
# if len(add_to_palette_list) > 0:
# # There are objects in source workspace that have type under DataPalette handling
# # but these objects are physically stored in source workspace rather that saved
# # in DataPalette object. So they weren't copied by "dps.copy_palette".
# self.data_palette_client.call_method(
# "add_to_palette",
# [{'workspace': str(newWsId), 'new_refs': add_to_palette_list}],
# self.token
# )
## END DP CODE
# update the ref inside the narrative object and the new workspace metadata.
newNarMetadata = currentNarrative['info'][10]
newNarMetadata['name'] = newName
newNarMetadata['ws_name'] = newWsName
newNarMetadata['job_info'] = json.dumps({
'queue_time': 0,
'running': 0,
'completed': 0,
'run_time': 0,
'error': 0
})
is_temporary = newNarMetadata.get('is_temporary', 'false')
if 'is_temporary' not in newNarMetadata:
if newNarMetadata['name'] == 'Untitled' or newNarMetadata[
'name'] is None:
is_temporary = 'true'
newNarMetadata['is_temporary'] = is_temporary
currentNarrative['data']['metadata']['name'] = newName
currentNarrative['data']['metadata']['ws_name'] = newWsName
currentNarrative['data']['metadata']['job_ids'] = {
'apps': [],
'methods': [],
'job_usage': {
'queue_time': 0,
'run_time': 0
}
}
# save the shiny new Narrative so it's at version 1
newNarInfo = self.ws.save_objects({
'id':
newWsId,
'objects': [{
'type': currentNarrative['info'][2],
'data': currentNarrative['data'],
'provenance': currentNarrative['provenance'],
'name': currentNarrative['info'][1],
'meta': newNarMetadata
}]
})
# now, just update the workspace metadata to point
# to the new narrative object
if 'worksheets' in currentNarrative['data']: # handle legacy.
num_cells = len(
currentNarrative['data']['worksheets'][0]['cells'])
else:
num_cells = len(currentNarrative['data']['cells'])
newNarId = newNarInfo[0][0]
self.ws.alter_workspace_metadata({
'wsi': {
'id': newWsId
},
'new': {
'narrative': str(newNarId),
'is_temporary': is_temporary,
'cell_count': str(num_cells)
}
})
return {'newWsId': newWsId, 'newNarId': newNarId}
except Exception:
# let's delete copy of workspace so it's out of the way - it's broken
self.ws.delete_workspace({'id': newWsId})
raise
def create_new_narrative(self, app, method, appparam, appData, markdown,
copydata, importData, includeIntroCell, title):
if app and method:
raise ValueError(
"Must provide no more than one of the app or method params")
if not importData and copydata:
importData = copydata.split(';')
if not appData and appparam:
appData = []
for tmp_item in appparam.split(';'):
tmp_tuple = tmp_item.split(',')
step_pos = None
if tmp_tuple[0]:
try:
step_pos = int(tmp_tuple[0])
except ValueError:
pass
appData.append([step_pos, tmp_tuple[1], tmp_tuple[2]])
cells = None
if app:
cells = [{"app": app}]
elif method:
cells = [{"method": method}]
elif markdown:
cells = [{"markdown": markdown}]
narr_info = self._create_temp_narrative(cells, appData, importData,
includeIntroCell, title)
if title is not None:
# update workspace info so it's not temporary
pass
return narr_info
def _get_intro_markdown(self):
"""
Creates and returns a cell with the introductory text included.
"""
# Load introductory markdown text
with open(self.intro_md_file) as intro_file:
intro_md = intro_file.read()
return intro_md
def _create_temp_narrative(self, cells, parameters, importData,
includeIntroCell, title):
# Migration to python of JavaScript class from https://github.com/kbase/kbase-ui/blob/4d31151d13de0278765a69b2b09f3bcf0e832409/src/client/modules/plugins/narrativemanager/modules/narrativeManager.js#L414
narr_id = int(round(time.time() * 1000))
workspaceName = self.user_id + ':narrative_' + str(narr_id)
narrativeName = "Narrative." + str(narr_id)
ws = self.ws
ws_info = ws.create_workspace({
'workspace': workspaceName,
'description': ''
})
[narrativeObject, metadataExternal
] = self._fetchNarrativeObjects(workspaceName, cells, parameters,
includeIntroCell, title)
is_temporary = 'true'
if title is not None and title != 'Untitled':
is_temporary = 'false'
metadataExternal['is_temporary'] = is_temporary
objectInfo = ws.save_objects({
'workspace':
workspaceName,
'objects': [{
'type':
'KBaseNarrative.Narrative',
'data':
narrativeObject,
'name':
narrativeName,
'meta':
metadataExternal,
'provenance': [{
'script':
'NarrativeManager.py',
'description':
'Created new ' + 'Workspace/Narrative bundle.'
}],
'hidden':
0
}]
})[0]
objectInfo = ServiceUtils.object_info_to_object(objectInfo)
ws_info = self._completeNewNarrative(ws_info[0], objectInfo['id'],
importData, is_temporary, title,
len(narrativeObject['cells']))
return {
'workspaceInfo': ServiceUtils.workspace_info_to_object(ws_info),
'narrativeInfo': objectInfo
}
def _fetchNarrativeObjects(self, workspaceName, cells, parameters,
includeIntroCell, title):
if not cells:
cells = []
if not title:
title = 'Untitled'
# fetchSpecs
appSpecIds = []
methodSpecIds = []
specMapping = {'apps': {}, 'methods': {}}
for cell in cells:
if 'app' in cell:
appSpecIds.append(cell['app'])
elif 'method' in cell:
methodSpecIds.append(cell['method'])
nms = NarrativeMethodStore(self.narrativeMethodStoreURL,
token=self.token)
if len(appSpecIds) > 0:
appSpecs = nms.get_app_spec({'ids': appSpecIds})
for spec in appSpecs:
spec_id = spec['info']['id']
specMapping['apps'][spec_id] = spec
if len(methodSpecIds) > 0:
methodSpecs = nms.get_method_spec({'ids': methodSpecIds})
for spec in methodSpecs:
spec_id = spec['info']['id']
specMapping['methods'][spec_id] = spec
# end of fetchSpecs
metadata = {
'job_ids': {
'methods': [],
'apps': [],
'job_usage': {
'queue_time': 0,
'run_time': 0
}
},
'format': 'ipynb',
'creator': self.user_id,
'ws_name': workspaceName,
'name': title,
'type': 'KBaseNarrative.Narrative',
'description': '',
'data_dependencies': []
}
cellData = self._gatherCellData(cells, specMapping, parameters,
includeIntroCell)
narrativeObject = {
'nbformat_minor': 0,
'cells': cellData,
'metadata': metadata,
'nbformat': 4
}
metadataExternal = {}
for key in metadata:
value = metadata[key]
if isinstance(value, str):
metadataExternal[key] = value
else:
metadataExternal[key] = json.dumps(value)
return [narrativeObject, metadataExternal]
def _gatherCellData(self, cells, specMapping, parameters,
includeIntroCell):
cell_data = []
if includeIntroCell == 1:
cell_data.append({
'cell_type': 'markdown',
'source': self._get_intro_markdown(),
'metadata': {}
})
for cell_pos, cell in enumerate(cells):
if 'app' in cell:
cell_data.append(
self._buildAppCell(len(cell_data),
specMapping['apps'][cell['app']],
parameters))
elif 'method' in cell:
cell_data.append(
self._buildMethodCell(
len(cell_data), specMapping['methods'][cell['method']],
parameters))
elif 'markdown' in cell:
cell_data.append({
'cell_type': 'markdown',
'source': cell['markdown'],
'metadata': {}
})
else:
raise ValueError("cannot add cell #" + str(cell_pos) +
", unrecognized cell content")
return cell_data
def _buildAppCell(self, pos, spec, params):
cellId = 'kb-cell-' + str(pos) + '-' + str(uuid.uuid4())
cell = {
"cell_type":
"markdown",
"source":
"<div id='" + cellId + "'></div>" + "\n<script>" + "$('#" +
cellId + "').kbaseNarrativeAppCell({'appSpec' : '" +
self._safeJSONStringify(spec) + "', 'cellId' : '" + cellId +
"'});" + "</script>",
"metadata": {}
}
cellInfo = {}
widgetState = []
cellInfo[self.KB_TYPE] = self.KB_APP_CELL
cellInfo['app'] = spec
if params:
steps = {}
for param in params:
stepid = 'step_' + str(param[0])
if stepid not in steps:
steps[stepid] = {}
steps[stepid]['inputState'] = {}
steps[stepid]['inputState'][param[1]] = param[2]
state = {'state': {'step': steps}}
widgetState.append(state)
cellInfo[self.KB_STATE] = widgetState
cell['metadata'][self.KB_CELL] = cellInfo
return cell
def _buildMethodCell(self, pos, spec, params):
cellId = "kb-cell-" + str(pos) + "-" + str(uuid.uuid4())
cell = {
"cell_type":
"markdown",
"source":
"<div id='" + cellId + "'></div>" + "\n<script>" + "$('#" +
cellId + "').kbaseNarrativeMethodCell({'method' : '" +
self._safeJSONStringify(spec) + "'});" + "</script>",
"metadata": {}
}
cellInfo = {"method": spec, "widget": spec["widgets"]["input"]}
cellInfo[self.KB_TYPE] = self.KB_FUNCTION_CELL
widgetState = []
if params:
wparams = {}
for param in params:
wparams[param[1]] = param[2]
widgetState.append({"state": wparams})
cellInfo[self.KB_STATE] = widgetState
cell["metadata"][self.KB_CELL] = cellInfo
return cell
def _completeNewNarrative(self, workspaceId, objectId, importData,
is_temporary, title, num_cells):
"""
'Completes' the new narrative by updating workspace metadata with the required fields and
copying in data from the importData list of references.
"""
new_meta = {
'narrative': str(objectId),
'is_temporary': is_temporary,
'searchtags': 'narrative',
'cell_count': str(num_cells)
}
if is_temporary == 'false' and title is not None:
new_meta['narrative_nice_name'] = title
self.ws.alter_workspace_metadata({
'wsi': {
'id': workspaceId
},
'new': new_meta
})
# copy_to_narrative:
if importData:
objectsToCopy = [{'ref': x} for x in importData]
infoList = self.ws.get_object_info_new({
'objects': objectsToCopy,
'includeMetadata': 0
})
for item in infoList:
objectInfo = ServiceUtils.object_info_to_object(item)
self.copy_object(objectInfo['ref'], workspaceId, None, None,
objectInfo)
return self.ws.get_workspace_info({'id': workspaceId})
def _safeJSONStringify(self, obj):
return json.dumps(self._safeJSONStringifyPrepare(obj))
def _safeJSONStringifyPrepare(self, obj):
if isinstance(obj, str):
return obj.replace("'", "'").replace('"', """)
elif isinstance(obj, list):
for pos in range(len(obj)):
obj[pos] = self._safeJSONStringifyPrepare(obj[pos])
elif isinstance(obj, dict):
obj_keys = list(obj.keys())
for key in obj_keys:
obj[key] = self._safeJSONStringifyPrepare(obj[key])
else:
pass # it's boolean/int/float/None
return obj
def _get_workspace_name_or_id(self, ws_id, ws_name):
ret = ws_name
if not ret:
ret = str(ws_id)
return ret
def copy_object(self, ref, target_ws_id, target_ws_name, target_name,
src_info):
"""
Copies an object from one workspace to another.
"""
if not target_ws_id and not target_ws_name:
raise ValueError("Neither target workspace id nor name is defined")
if not src_info:
src_info_tuple = self.ws.get_object_info_new({
'objects': [{
'ref': ref
}],
'includeMetadata':
0
})[0]
src_info = ServiceUtils.object_info_to_object(src_info_tuple)
if not target_name:
target_name = src_info['name']
obj_info_tuple = self.ws.copy_object({
'from': {
'ref': ref
},
'to': {
'wsid': target_ws_id,
'workspace': target_ws_name,
'name': target_name
}
})
obj_info = ServiceUtils.object_info_to_object(obj_info_tuple)
return {'info': obj_info}
def list_available_types(self, workspaces):
data = self.list_objects_with_sets(workspaces=workspaces)['data']
type_stat = {}
for item in data:
info = item['object_info']
obj_type = info[2].split('-')[0]
if obj_type in type_stat:
type_stat[obj_type] += 1
else:
type_stat[obj_type] = 1
return {'type_stat': type_stat}
def run_barseqR(self, ctx, params):
"""
Args:
:param params: instance of mapping from String to unspecified object
ctx:
client_ip: None or 'str',
user_id: str,
'authenticated': 1,
'token': str,
'module': None,
'method': None,
'call_id': None,
'rpc_context': None,
'provenance':list<prov_d>
prov_d: (d)
service: (str)
'method': 'please_never_use_it_in_production',
'method_params': []}]}
:returns: instance of type "ReportResults" -> structure: parameter
"report_name" of String, parameter "report_ref" of String
"""
# ctx is the context object
# return variables are: output
#BEGIN run_barseqR
# SETUP - Unrelated to inputs --------
logging.basicConfig(level=logging.DEBUG)
logging.info("Call back url: " + str(self.callback_url))
# We create important classes
dfu = DataFileUtil(self.callback_url)
logging.info("DFU VARS-- " * 8)
logging.info(vars(dfu))
gfu = GenomeFileUtil(self.callback_url)
smpl_s = SampleService(self.callback_url)
myToken = os.environ.get('KB_AUTH_TOKEN', None)
ws = Workspace(self.ws_url, token=myToken)
ws_id = ws.get_workspace_info({'workspace':
params['workspace_name']})[0]
logging.info(os.environ)
logging.info('ws-url')
logging.info(self.ws_url)
logging.info('ctx')
logging.info(ctx)
# We create indir, outdir, sets_dir (Input, Output, Sets)
indir = os.path.join(self.shared_folder, "indir")
os.mkdir(indir)
outdir = os.path.join(self.shared_folder, "outdir")
os.mkdir(outdir)
sets_dir = os.path.join(indir, "sets_dir")
os.mkdir(sets_dir)
metadir = '/kb/module/lib/RunDir/metadata'
if not (os.path.isdir(metadir)):
raise Exception(
"metadata directory not found at: {}".format(metadir))
# We prepare locations of input files
poolfile_path = os.path.join(indir, "pool.n10")
gene_table_fp = os.path.join(indir, "genes.GC")
exps_file = os.path.join(indir, "FEBA_Barseq.tsv")
# END SETUP
# VALIDATE PARAMS:
logging.info("PARAMS:")
logging.info(params)
# From Util.validate python file
val_par = validate_params(params)
'''
val_par contains keys:
genome_ref
poolfile_ref
exps_ref
sets_ref
output_name
workspace_name
'''
val_par['username'] = ctx['user_id']
# DOWNLOAD FILES
download_dict = {
"dfu": dfu,
"gfu": gfu,
"ws": ws,
"smpl_s": smpl_s,
"sets_dir": sets_dir,
"poolfile_path": poolfile_path,
"gene_table_fp": gene_table_fp,
"exps_file": exps_file,
"scratch_dir": self.shared_folder
}
# We copy input files to proper directories.
# vp must contain genome_ref, poolfile_ref, exps_ref, sets_refs (list)
# DownloadResults must contain keys 'org', 'set_names_list', 'set_fps_list'
# set_names_list value contains the names of the sets without extensions
DownloadResults = download_files(val_par, download_dict)
logging.debug(json.dumps(DownloadResults, indent=2))
# Get args in this format:
# [-org, org_name, -indir, Scratch_Dir_Input, -metadir, Fixed meta dir,
# -outdir, scratch_dir_output, -sets_dir, within scratch_dir_input,
# -sets, set1 (sets_dir), set2 (sets_dir), set3 (sets_dir), ... ]
# Note meta dir is called metadata and is in RunDir
# Running the entire program:
arg_list = [
"-org", DownloadResults['org'], '-indir', indir, '-metadir',
metadir, '-outdir', outdir, '-sets_dir', sets_dir, '-sets'
]
arg_list += DownloadResults['set_names_list']
RunBarSeq(arg_list)
# Returning files to user
report = KBaseReport(self.callback_url)
report_info = report.create({
'report': {
'objects_created': [],
'text_message': params['parameter_1']
},
'workspace_name': params['workspace_name']
})
output = {
'report_name': report_info['name'],
'report_ref': report_info['ref'],
}
#END run_barseqR
# At some point might do deeper type checking...
if not isinstance(output, dict):
raise ValueError('Method run_barseqR return value ' +
'output is not type dict as required.')
# return the results
return [output]
def run_poolcount(self, ctx, params):
"""
This example function accepts any number of parameters and returns
results in a KBaseReport
:param params: instance of mapping from String to unspecified object
:returns: instance of type "ReportResults" -> structure: parameter
"report_name" of String, parameter "report_ref" of String
Args:
params:
['workspace_name']: self.wsName,
"mutantpool_ref": pool_ref (str),
"fastq_files": list<fastq_refs (str)>,
"genome_ref": genome_ref (str),
"KB_BarcodeCount_Bool": "yes"/"no" - create a poolcount file?
"poolcount_description": (str) A text description of the pool file,
"output_name": (str),
## "test_local_bool": test_local_bool Deprecated
## "save_ignore_bool": bool, Deprecated
"maxReads": int or None,
"minQuality": int,
"debug": bool,
"protocol_type": str,
"doOff1": bool
"""
# ctx is the context object
# return variables are: output
logging.basicConfig(level=logging.DEBUG)
logging.warning("INPUT PARAMS:")
logging.warning(params)
#BEGIN run_poolcount
report = KBaseReport(self.callback_url)
myToken = os.environ.get('KB_AUTH_TOKEN', None)
ws = Workspace(self.ws_url, token=myToken)
ws_id = ws.get_workspace_info({'workspace':
params['workspace_name']})[0]
#Creating Data File Util Object
dfu = DataFileUtil(self.callback_url)
#We make the BarcodeCount output directory in scratch:
outputs_dir = os.path.join(self.shared_folder, "BarcodeCount_Outputs")
HTML_dir = os.path.join(self.shared_folder, "HTML_OP")
MC_dir = os.path.join(self.shared_folder, "MC_op_dir")
for x_dir in [outputs_dir, HTML_dir, MC_dir]:
if os.path.isdir(x_dir):
logging.info(f"{x_dir} contents: " +
",\n".join(os.listdir(x_dir)))
else:
os.mkdir(x_dir)
main_HTML_fp = os.path.join(HTML_dir, "index.html")
# parsed_params_dict contains keys:
# mutantpool_ref, fastq_files_refs_list, genes_table_ref, output_name,
# KB_BarcodeCount_Bool, poolcount_description
parsed_params_dict = parse_and_check_params(params)
# We get the username for later
parsed_params_dict['username'] = ctx['user_id']
# We set the mutantpool's path
mutantpool_path = os.path.join(self.shared_folder, "kb_pool.pool")
download_mutantpool(parsed_params_dict['mutantpool_ref'],
mutantpool_path,
dfu,
genome_ref=parsed_params_dict['genome_ref'])
poolcount_prefix = os.path.join(outputs_dir,
parsed_params_dict["output_name"])
fastq_dicts_list = download_fastq_and_prepare_mc(
parsed_params_dict, dfu, self.shared_folder, outputs_dir)
logging.info(fastq_dicts_list)
#First we download all the Fastq Files and the Pool File
report_dict = {}
report_str = ""
mc_run_list = fastq_dicts_list
mc_run_num = len(mc_run_list)
logging.info("Total MultiCodes Runs: {}".format(mc_run_num))
FullRun_d = get_FullRun_d(parsed_params_dict, MC_dir, fastq_dicts_list,
poolcount_prefix, mutantpool_path,
main_HTML_fp)
# Running all programs:
PC_RunAll(FullRun_d)
# Now we upload the poolcount file to KBase to make a BarcodeCount Object
if parsed_params_dict['KB_BarcodeCount_Bool']:
upload_params = {
'username':
parsed_params_dict['username'],
'fastq_refs':
parsed_params_dict['fastq_files'],
'genome_ref':
parsed_params_dict['genome_ref'],
'mutantpool_ref':
parsed_params_dict['mutantpool_ref'],
'poolcount_description':
parsed_params_dict['poolcount_description'],
'workspace_id':
ws_id,
'ws_obj':
ws,
'protocol_type':
parsed_params_dict["protocol_type"],
'poolcount_fp':
poolcount_prefix + ".poolcount",
'poolcount_name':
parsed_params_dict['output_name'],
'dfu':
dfu,
"scratch_dir":
self.shared_folder,
"set_name":
parsed_params_dict['output_name']
}
logging.info("UPLOADING BarcodeCount FILE to KBASE through DFU")
upload_mutantpool_results = upload_poolcount_to_KBase(
upload_params)
logging.info("Upload BarcodeCount File Results:")
logging.info(upload_mutantpool_results)
# DEBUGGING WHAT REPORT DICT LOOKS LIKE
with open(os.path.join(self.shared_folder, "Report.JSON"), 'w') as g:
g.write(json.dumps(report_dict, indent=2))
#Cleaning outputs dir (removing .codes, .close, .counts files)
clean_output_dir(outputs_dir, self.shared_folder)
#Writing report string:
report_fp = os.path.join(outputs_dir, "Run_Report.txt")
with open(report_fp, "w") as f:
f.write(report_str)
#Return Outputs Dir to user:
dir_zip_shock_id = dfu.file_to_shock({
"file_path": outputs_dir,
'pack': 'zip'
})['shock_id']
dir_link_dict = {
'shock_id': dir_zip_shock_id,
'name': parsed_params_dict['output_name'] + ".zip",
'label': 'RBTnSeqBarcodeCount_dir',
'description': 'The folder containing outputs from this app'
}
HTML_report_shock_id = dfu.file_to_shock({
"file_path": HTML_dir,
"pack": "zip"
})['shock_id']
HTML_report_d_l = [{
"shock_id":
HTML_report_shock_id,
"name":
os.path.basename(os.path.join(HTML_dir, "index.html")),
"label":
"BarcodeCount Report",
"description":
"HTML Summary Report for MultiCodes and Combine BarSeq"
}]
extended_report_params = {
'workspace_name': params['workspace_name'],
"html_links": HTML_report_d_l,
"direct_html_link_index": 0,
"html_window_height": 333,
"report_object_name": "KB_BarcodeCount_Report",
'file_links': [dir_link_dict],
'message': ""
}
report_info = report.create_extended_report(extended_report_params)
output = {
'report_name': report_info['name'],
'report_ref': report_info['ref'],
}
#END run_poolcount
# At some point might do deeper type checking...
if not isinstance(output, dict):
raise ValueError('Method run_poolcount return value ' +
'output is not type dict as required.')
# return the results
return [output]
