def get_authors(config, wsid): ws = Workspace(url=config.narrative_session.ws_url, token=config.narrative_session.token) ws_info = ws.get_workspace_info({"id": wsid}) author_id_list = [ws_info[2]] other_authors = ws.get_permissions({"id": wsid}) for author in sorted(other_authors.keys()): if author != "*" and other_authors[author] in [ "w", "a" ] and author not in author_id_list: author_id_list.append(author) auth = _KBaseAuth(config.narrative_session.auth_url) disp_names = auth.get_display_names(config.narrative_session.token, author_id_list) author_list = [] for author in author_id_list: author_list.append({ "id": author, "name": html.escape(disp_names.get(author, author)), "path": config.narrative_session.profile_page_url + author }) return author_list
def fetch_narrative_data(endpt: str, token: str, ws_id: int, outdir: str) -> int: ws = Workspace(url=endpt + "ws", token=token) ws_info = ws.get_workspace_info({"id": ws_id}) ws_meta = ws_info[8] # Narrative object narr_id = ws_meta["narrative"] narr_obj = ws.get_objects2({"objects": [{ "ref": f"{ws_id}/{narr_id}" }]})["data"][0] narr_ver = narr_obj["info"][4] narr_outpath = os.path.join( outdir, f"narrative-{ws_id}.{narr_id}.{narr_ver}.json") with open(narr_outpath, "w") as fout: json.dump(narr_obj, fout, indent=4) # Report objects for cell in narr_obj["data"]["cells"]: if "kbase" in cell["metadata"]: meta = cell["metadata"]["kbase"] if "appCell" in meta: job_state = meta["appCell"].get("exec", {}).get("jobState") result = list() if "result" in job_state: result = job_state["result"] elif "job_output" in job_state and "result" in job_state[ "job_output"]: result = job_state["job_output"]["result"] if len(result) > 0 and "report_ref" in result[0]: report_data = ws.get_objects2( {"objects": [{ "ref": result[0]["report_ref"] }]})["data"][0] report_info = report_data["info"] ref_dots = f"{report_info[6]}.{report_info[0]}.{report_info[4]}" report_path = os.path.join(outdir, f"report-{ref_dots}.json") with open(report_path, "w") as fout: json.dump(report_data, fout, indent=4) # List objects results service = NarrativeService(url=endpt + "service_wizard", token=token) # service = ServiceClient(url=endpt + "service_wizard", use_url_lookup=True, token=token) ws_data = service.list_objects_with_sets({ "ws_id": ws_id, "includeMetadata": 1 }) # ws_data = service.sync_call( # "NarrativeService.list_objects_with_sets", # [{"ws_id": ws_id, "includeMetadata": 1}] # )[0] data_outpath = os.path.join(outdir, f"objects-{ws_id}.json") with open(data_outpath, "w") as fout: json.dump(ws_data, fout, indent=4) return 0
def get_static_info(ws_url: str, token: str, ws_id: int) -> Dict: """ Looks up the static narrative info for the given Workspace id. That info is stashed in the Workspace metadata, so that gets fetched, munged into a structure, and returned. If there's no static narrative, this returns an empty structure, as there's no info. If ws_id is not present, or not numeric, raises a ValueError. If there's a problem when contacting the Workspace (anything that raises a ServerError), this raises a WorkspaceError. :param ws_url: the URL for the workspace endpoint :param token: the user auth token :param ws_id: the workspace id of the narrative to fetch info for. :returns: a dictionary with the following keys if a static narrative is present: ws_id - int - the workspace id narrative_id - int - the id of the narrative object version - int - the version of the narrative object made static url - str - the url of the static narrative narr_saved - int - the timestamp of when the narrative that the static version is based on was saved (ms since epoch) static_saved - int - the timestamp of when the static narrative was saved (ms since epoch) """ if not ws_id or not str(ws_id).isdigit(): raise ValueError(f"The parameter ws_id must be an integer, not {ws_id}") ws_client = Workspace(url=ws_url, token=token) try: ws_info = ws_client.get_workspace_info({"id": ws_id}) except ServerError as err: raise WorkspaceError(err, ws_id) info = {} meta = ws_info[8] if "static_narrative_ver" in meta: info = { "ws_id": ws_id, "version": int(meta["static_narrative_ver"]), "narrative_id": int(meta["narrative"]), "url": meta["static_narrative"], "static_saved": int(meta["static_narrative_saved"]) } try: obj_info = ws_client.get_object_info3({ "objects": [{ "ref": f"{ws_id}/{info['narrative_id']}/{info['version']}" }] }) except ServerError as err: raise WorkspaceError(err, ws_id) ts = date_parser.isoparse(obj_info["infos"][0][3]).timestamp() info["narr_saved"] = int(ts*1000) return info
class WorkspaceAdminUtils: def __init__(self, config): wsurl = config.get('workspace-url') self.atoken = config.get('workspace-admin-token') self.noadmin = False if self.atoken is None or self.atoken == '': self.noadmin = True self.atoken = os.environ.get('KB_AUTH_TOKEN', None) self.ws = Workspace(wsurl, token=self.atoken) def list_objects(self, params): """ Provide something that acts like a standard listObjects """ if self.noadmin: return self.ws.list_objects(params) return self.ws.administer({'command': 'listObjects', 'params': params}) def get_objects2(self, params): """ Provide something that acts like a standard getObjects """ if self.noadmin: return self.ws.get_objects2(params) return self.ws.administer({'command': 'getObjects', 'params': params}) def get_workspace_info(self, params): """ Provide something that acts like a standard getObjects """ if self.noadmin: return self.ws.get_workspace_info(params) return self.ws.administer({ 'command': 'getWorkspaceInfo', 'params': params })
def run_poolfileupload(self, ctx, params): """ This example function accepts any number of parameters and returns results in a KBaseReport :param params: instance of mapping from String to unspecified object :returns: instance of type "ReportResults" -> structure: parameter "report_name" of String, parameter "report_ref" of String """ # ctx is the context object # return variables are: output #BEGIN run_poolfileupload params['shared_folder'] = self.shared_folder token = os.environ.get('KB_AUTH_TOKEN', None) ws = Workspace(self.ws_url, token=token) params['workspace_id'] = ws.get_workspace_info( {'workspace': params['workspace_name']})[0] params['ws_obj'] = ws params['username'] = ctx['user_id'] params['output_name'] = check_output_name(params['output_name']) if 'pool_file_type' not in params: raise Exception("Did not get param pool_file_type") else: pft = params['pool_file_type'] if pft == 'poolfile': pfu = poolfileuploadUtil(params) result = pfu.upload_poolfile() elif pft == 'poolcount': pcfu = poolcountfileuploadUtil(params) result = pcfu.upload_poolcountfile() elif pft == 'experiments': expsfu = expsfileuploadUtil(params) result = expsfu.upload_expsfile() else: raise Exception("Did not recognize pool_file_type for upload") text_message = "Finished uploading file \n" text_message += "{} saved as {} on {}\n".format( result['Name'], result['Type'], result['Date']) report = KBaseReport(self.callback_url) report_info = report.create({ 'report': { 'objects_created': [], 'text_message': text_message }, 'workspace_name': params['workspace_name'] }) output = { 'report_name': report_info['name'], 'report_ref': report_info['ref'], } #END run_poolfileupload # At some point might do deeper type checking... if not isinstance(output, dict): raise ValueError('Method run_poolfileupload return value ' + 'output is not type dict as required.') # return the results return [output]
class NarrativeManager: KB_CELL = 'kb-cell' KB_TYPE = 'type' KB_APP_CELL = 'kb_app' KB_FUNCTION_CELL = 'function_input' KB_OUTPUT_CELL = 'function_output' KB_ERROR_CELL = 'kb_error' KB_CODE_CELL = 'kb_code' KB_STATE = 'widget_state' DEBUG = False DATA_PALETTES_TYPES = DataPaletteTypes(False) def __init__(self, config, ctx, set_api_client, data_palette_client): self.narrativeMethodStoreURL = config['narrative-method-store'] self.set_api_cache = set_api_client # DynamicServiceCache type self.data_palette_client = data_palette_client # DynamicServiceCache type self.token = ctx["token"] self.user_id = ctx["user_id"] self.ws = Workspace(config['workspace-url'], token=self.token) self.intro_md_file = config['intro-markdown-file'] # We switch DPs on only for internal Continuous Integration environment for now: if config['kbase-endpoint'].startswith("https://ci.kbase.us/") or \ 'USE_DP' in os.environ: self.DATA_PALETTES_TYPES = DataPaletteTypes(True) def list_objects_with_sets(self, ws_id=None, ws_name=None, workspaces=None, types=None, include_metadata=0, include_data_palettes=0): if not workspaces: if not ws_id and not ws_name: raise ValueError( "One and only one of 'ws_id', 'ws_name', 'workspaces' " + "parameters should be set") workspaces = [self._get_workspace_name_or_id(ws_id, ws_name)] return self._list_objects_with_sets(workspaces, types, include_metadata, include_data_palettes) def _list_objects_with_sets(self, workspaces, types, include_metadata, include_data_palettes): type_map = None if types is not None: type_map = {key: True for key in types} processed_refs = {} data = [] if self.DEBUG: print("NarrativeManager._list_objects_with_sets: processing sets") t1 = time.time() set_ret = self.set_api_cache.call_method( "list_sets", [{ 'workspaces': workspaces, 'include_set_item_info': 1, 'include_metadata': include_metadata }], self.token) sets = set_ret['sets'] for set_info in sets: # Process target_set_items = [] for set_item in set_info['items']: target_set_items.append(set_item['info']) if self._check_info_type(set_info['info'], type_map): data_item = { 'object_info': set_info['info'], 'set_items': { 'set_items_info': target_set_items } } data.append(data_item) processed_refs[set_info['ref']] = data_item if self.DEBUG: print(" (time=" + str(time.time() - t1) + ")") if self.DEBUG: print("NarrativeManager._list_objects_with_sets: loading ws_info") t2 = time.time() ws_info_list = [] # for ws in workspaces: if len(workspaces) == 1: ws = workspaces[0] ws_id = None ws_name = None if str(ws).isdigit(): ws_id = int(ws) else: ws_name = str(ws) ws_info_list.append( self.ws.get_workspace_info({ "id": ws_id, "workspace": ws_name })) else: ws_map = {key: True for key in workspaces} for ws_info in self.ws.list_workspace_info({'perm': 'r'}): if ws_info[1] in ws_map or str(ws_info[0]) in ws_map: ws_info_list.append(ws_info) if self.DEBUG: print(" (time=" + str(time.time() - t2) + ")") if self.DEBUG: print( "NarrativeManager._list_objects_with_sets: loading workspace objects" ) t3 = time.time() for info in WorkspaceListObjectsIterator( self.ws, ws_info_list=ws_info_list, list_objects_params={'includeMetadata': include_metadata}): item_ref = str(info[6]) + '/' + str(info[0]) + '/' + str(info[4]) if item_ref not in processed_refs and self._check_info_type( info, type_map): data_item = {'object_info': info} data.append(data_item) processed_refs[item_ref] = data_item if self.DEBUG: print(" (time=" + str(time.time() - t3) + ")") return_data = {"data": data} if include_data_palettes == 1: if self.DEBUG: print( "NarrativeManager._list_objects_with_sets: processing DataPalettes" ) t5 = time.time() dp_ret = self.data_palette_client.call_method( "list_data", [{ 'workspaces': workspaces, 'include_metadata': include_metadata }], self.token) for item in dp_ret['data']: ref = item['ref'] if self._check_info_type(item['info'], type_map): data_item = None if ref in processed_refs: data_item = processed_refs[ref] else: data_item = {'object_info': item['info']} processed_refs[ref] = data_item data.append(data_item) dp_info = {} if 'dp_ref' in item: dp_info['ref'] = item['dp_ref'] if 'dp_refs' in item: dp_info['refs'] = item['dp_refs'] data_item['dp_info'] = dp_info return_data["data_palette_refs"] = dp_ret['data_palette_refs'] if self.DEBUG: print(" (time=" + str(time.time() - t5) + ")") return return_data def _check_info_type(self, info, type_map): if type_map is None: return True obj_type = info[2].split('-')[0] return type_map.get(obj_type, False) def copy_narrative(self, newName, workspaceRef, workspaceId): time_ms = int(round(time.time() * 1000)) newWsName = self.user_id + ':narrative_' + str(time_ms) # add the 'narrative' field to newWsMeta later. newWsMeta = {"narrative_nice_name": newName, "searchtags": "narrative"} # start with getting the existing narrative object. currentNarrative = self.ws.get_objects([{'ref': workspaceRef}])[0] if not workspaceId: workspaceId = currentNarrative['info'][6] # Let's prepare exceptions for clone the workspace. # 1) currentNarrative object: excluded_list = [{'objid': currentNarrative['info'][0]}] # 2) let's exclude objects of types under DataPalette handling: ## DP CODE # data_palette_type = "DataPalette.DataPalette" # excluded_types = [data_palette_type] # excluded_types.extend(self.DATA_PALETTES_TYPES.keys()) # add_to_palette_list = [] # dp_detected = False ## END DP CODE # for obj_type in excluded_types: # list_objects_params = {'type': obj_type} ## DP CODE # if obj_type == data_palette_type: # list_objects_params['showHidden'] = 1 ## END DP CODE # for info in WorkspaceListObjectsIterator(self.ws, # ws_id=workspaceId, # list_objects_params=list_objects_params): ## DP CODE # if obj_type == data_palette_type: # dp_detected = True # else: # add_to_palette_list.append({ # 'ref': str(info[6]) + '/' + str(info[0]) + '/' + str(info[4]) # }) ## END DP CODE # excluded_list.append({'objid': info[0]}) # clone the workspace EXCEPT for currentNarrative object newWsId = self.ws.clone_workspace({ 'wsi': { 'id': workspaceId }, 'workspace': newWsName, 'meta': newWsMeta, 'exclude': excluded_list })[0] try: ## DP CODE # if dp_detected: # self.data_palette_client.call_method( # "copy_palette", # [{'from_workspace': str(workspaceId), 'to_workspace': str(newWsId)}], # self.token # ) # if len(add_to_palette_list) > 0: # # There are objects in source workspace that have type under DataPalette handling # # but these objects are physically stored in source workspace rather that saved # # in DataPalette object. So they weren't copied by "dps.copy_palette". # self.data_palette_client.call_method( # "add_to_palette", # [{'workspace': str(newWsId), 'new_refs': add_to_palette_list}], # self.token # ) ## END DP CODE # update the ref inside the narrative object and the new workspace metadata. newNarMetadata = currentNarrative['info'][10] newNarMetadata['name'] = newName newNarMetadata['ws_name'] = newWsName newNarMetadata['job_info'] = json.dumps({ 'queue_time': 0, 'running': 0, 'completed': 0, 'run_time': 0, 'error': 0 }) is_temporary = newNarMetadata.get('is_temporary', 'false') if 'is_temporary' not in newNarMetadata: if newNarMetadata['name'] == 'Untitled' or newNarMetadata[ 'name'] is None: is_temporary = 'true' newNarMetadata['is_temporary'] = is_temporary currentNarrative['data']['metadata']['name'] = newName currentNarrative['data']['metadata']['ws_name'] = newWsName currentNarrative['data']['metadata']['job_ids'] = { 'apps': [], 'methods': [], 'job_usage': { 'queue_time': 0, 'run_time': 0 } } # save the shiny new Narrative so it's at version 1 newNarInfo = self.ws.save_objects({ 'id': newWsId, 'objects': [{ 'type': currentNarrative['info'][2], 'data': currentNarrative['data'], 'provenance': currentNarrative['provenance'], 'name': currentNarrative['info'][1], 'meta': newNarMetadata }] }) # now, just update the workspace metadata to point # to the new narrative object if 'worksheets' in currentNarrative['data']: # handle legacy. num_cells = len( currentNarrative['data']['worksheets'][0]['cells']) else: num_cells = len(currentNarrative['data']['cells']) newNarId = newNarInfo[0][0] self.ws.alter_workspace_metadata({ 'wsi': { 'id': newWsId }, 'new': { 'narrative': str(newNarId), 'is_temporary': is_temporary, 'cell_count': str(num_cells) } }) return {'newWsId': newWsId, 'newNarId': newNarId} except Exception: # let's delete copy of workspace so it's out of the way - it's broken self.ws.delete_workspace({'id': newWsId}) raise def create_new_narrative(self, app, method, appparam, appData, markdown, copydata, importData, includeIntroCell, title): if app and method: raise ValueError( "Must provide no more than one of the app or method params") if not importData and copydata: importData = copydata.split(';') if not appData and appparam: appData = [] for tmp_item in appparam.split(';'): tmp_tuple = tmp_item.split(',') step_pos = None if tmp_tuple[0]: try: step_pos = int(tmp_tuple[0]) except ValueError: pass appData.append([step_pos, tmp_tuple[1], tmp_tuple[2]]) cells = None if app: cells = [{"app": app}] elif method: cells = [{"method": method}] elif markdown: cells = [{"markdown": markdown}] narr_info = self._create_temp_narrative(cells, appData, importData, includeIntroCell, title) if title is not None: # update workspace info so it's not temporary pass return narr_info def _get_intro_markdown(self): """ Creates and returns a cell with the introductory text included. """ # Load introductory markdown text with open(self.intro_md_file) as intro_file: intro_md = intro_file.read() return intro_md def _create_temp_narrative(self, cells, parameters, importData, includeIntroCell, title): # Migration to python of JavaScript class from https://github.com/kbase/kbase-ui/blob/4d31151d13de0278765a69b2b09f3bcf0e832409/src/client/modules/plugins/narrativemanager/modules/narrativeManager.js#L414 narr_id = int(round(time.time() * 1000)) workspaceName = self.user_id + ':narrative_' + str(narr_id) narrativeName = "Narrative." + str(narr_id) ws = self.ws ws_info = ws.create_workspace({ 'workspace': workspaceName, 'description': '' }) [narrativeObject, metadataExternal ] = self._fetchNarrativeObjects(workspaceName, cells, parameters, includeIntroCell, title) is_temporary = 'true' if title is not None and title != 'Untitled': is_temporary = 'false' metadataExternal['is_temporary'] = is_temporary objectInfo = ws.save_objects({ 'workspace': workspaceName, 'objects': [{ 'type': 'KBaseNarrative.Narrative', 'data': narrativeObject, 'name': narrativeName, 'meta': metadataExternal, 'provenance': [{ 'script': 'NarrativeManager.py', 'description': 'Created new ' + 'Workspace/Narrative bundle.' }], 'hidden': 0 }] })[0] objectInfo = ServiceUtils.object_info_to_object(objectInfo) ws_info = self._completeNewNarrative(ws_info[0], objectInfo['id'], importData, is_temporary, title, len(narrativeObject['cells'])) return { 'workspaceInfo': ServiceUtils.workspace_info_to_object(ws_info), 'narrativeInfo': objectInfo } def _fetchNarrativeObjects(self, workspaceName, cells, parameters, includeIntroCell, title): if not cells: cells = [] if not title: title = 'Untitled' # fetchSpecs appSpecIds = [] methodSpecIds = [] specMapping = {'apps': {}, 'methods': {}} for cell in cells: if 'app' in cell: appSpecIds.append(cell['app']) elif 'method' in cell: methodSpecIds.append(cell['method']) nms = NarrativeMethodStore(self.narrativeMethodStoreURL, token=self.token) if len(appSpecIds) > 0: appSpecs = nms.get_app_spec({'ids': appSpecIds}) for spec in appSpecs: spec_id = spec['info']['id'] specMapping['apps'][spec_id] = spec if len(methodSpecIds) > 0: methodSpecs = nms.get_method_spec({'ids': methodSpecIds}) for spec in methodSpecs: spec_id = spec['info']['id'] specMapping['methods'][spec_id] = spec # end of fetchSpecs metadata = { 'job_ids': { 'methods': [], 'apps': [], 'job_usage': { 'queue_time': 0, 'run_time': 0 } }, 'format': 'ipynb', 'creator': self.user_id, 'ws_name': workspaceName, 'name': title, 'type': 'KBaseNarrative.Narrative', 'description': '', 'data_dependencies': [] } cellData = self._gatherCellData(cells, specMapping, parameters, includeIntroCell) narrativeObject = { 'nbformat_minor': 0, 'cells': cellData, 'metadata': metadata, 'nbformat': 4 } metadataExternal = {} for key in metadata: value = metadata[key] if isinstance(value, str): metadataExternal[key] = value else: metadataExternal[key] = json.dumps(value) return [narrativeObject, metadataExternal] def _gatherCellData(self, cells, specMapping, parameters, includeIntroCell): cell_data = [] if includeIntroCell == 1: cell_data.append({ 'cell_type': 'markdown', 'source': self._get_intro_markdown(), 'metadata': {} }) for cell_pos, cell in enumerate(cells): if 'app' in cell: cell_data.append( self._buildAppCell(len(cell_data), specMapping['apps'][cell['app']], parameters)) elif 'method' in cell: cell_data.append( self._buildMethodCell( len(cell_data), specMapping['methods'][cell['method']], parameters)) elif 'markdown' in cell: cell_data.append({ 'cell_type': 'markdown', 'source': cell['markdown'], 'metadata': {} }) else: raise ValueError("cannot add cell #" + str(cell_pos) + ", unrecognized cell content") return cell_data def _buildAppCell(self, pos, spec, params): cellId = 'kb-cell-' + str(pos) + '-' + str(uuid.uuid4()) cell = { "cell_type": "markdown", "source": "<div id='" + cellId + "'></div>" + "\n<script>" + "$('#" + cellId + "').kbaseNarrativeAppCell({'appSpec' : '" + self._safeJSONStringify(spec) + "', 'cellId' : '" + cellId + "'});" + "</script>", "metadata": {} } cellInfo = {} widgetState = [] cellInfo[self.KB_TYPE] = self.KB_APP_CELL cellInfo['app'] = spec if params: steps = {} for param in params: stepid = 'step_' + str(param[0]) if stepid not in steps: steps[stepid] = {} steps[stepid]['inputState'] = {} steps[stepid]['inputState'][param[1]] = param[2] state = {'state': {'step': steps}} widgetState.append(state) cellInfo[self.KB_STATE] = widgetState cell['metadata'][self.KB_CELL] = cellInfo return cell def _buildMethodCell(self, pos, spec, params): cellId = "kb-cell-" + str(pos) + "-" + str(uuid.uuid4()) cell = { "cell_type": "markdown", "source": "<div id='" + cellId + "'></div>" + "\n<script>" + "$('#" + cellId + "').kbaseNarrativeMethodCell({'method' : '" + self._safeJSONStringify(spec) + "'});" + "</script>", "metadata": {} } cellInfo = {"method": spec, "widget": spec["widgets"]["input"]} cellInfo[self.KB_TYPE] = self.KB_FUNCTION_CELL widgetState = [] if params: wparams = {} for param in params: wparams[param[1]] = param[2] widgetState.append({"state": wparams}) cellInfo[self.KB_STATE] = widgetState cell["metadata"][self.KB_CELL] = cellInfo return cell def _completeNewNarrative(self, workspaceId, objectId, importData, is_temporary, title, num_cells): """ 'Completes' the new narrative by updating workspace metadata with the required fields and copying in data from the importData list of references. """ new_meta = { 'narrative': str(objectId), 'is_temporary': is_temporary, 'searchtags': 'narrative', 'cell_count': str(num_cells) } if is_temporary == 'false' and title is not None: new_meta['narrative_nice_name'] = title self.ws.alter_workspace_metadata({ 'wsi': { 'id': workspaceId }, 'new': new_meta }) # copy_to_narrative: if importData: objectsToCopy = [{'ref': x} for x in importData] infoList = self.ws.get_object_info_new({ 'objects': objectsToCopy, 'includeMetadata': 0 }) for item in infoList: objectInfo = ServiceUtils.object_info_to_object(item) self.copy_object(objectInfo['ref'], workspaceId, None, None, objectInfo) return self.ws.get_workspace_info({'id': workspaceId}) def _safeJSONStringify(self, obj): return json.dumps(self._safeJSONStringifyPrepare(obj)) def _safeJSONStringifyPrepare(self, obj): if isinstance(obj, str): return obj.replace("'", "'").replace('"', """) elif isinstance(obj, list): for pos in range(len(obj)): obj[pos] = self._safeJSONStringifyPrepare(obj[pos]) elif isinstance(obj, dict): obj_keys = list(obj.keys()) for key in obj_keys: obj[key] = self._safeJSONStringifyPrepare(obj[key]) else: pass # it's boolean/int/float/None return obj def _get_workspace_name_or_id(self, ws_id, ws_name): ret = ws_name if not ret: ret = str(ws_id) return ret def copy_object(self, ref, target_ws_id, target_ws_name, target_name, src_info): """ Copies an object from one workspace to another. """ if not target_ws_id and not target_ws_name: raise ValueError("Neither target workspace id nor name is defined") if not src_info: src_info_tuple = self.ws.get_object_info_new({ 'objects': [{ 'ref': ref }], 'includeMetadata': 0 })[0] src_info = ServiceUtils.object_info_to_object(src_info_tuple) if not target_name: target_name = src_info['name'] obj_info_tuple = self.ws.copy_object({ 'from': { 'ref': ref }, 'to': { 'wsid': target_ws_id, 'workspace': target_ws_name, 'name': target_name } }) obj_info = ServiceUtils.object_info_to_object(obj_info_tuple) return {'info': obj_info} def list_available_types(self, workspaces): data = self.list_objects_with_sets(workspaces=workspaces)['data'] type_stat = {} for item in data: info = item['object_info'] obj_type = info[2].split('-')[0] if obj_type in type_stat: type_stat[obj_type] += 1 else: type_stat[obj_type] = 1 return {'type_stat': type_stat}
def run_barseqR(self, ctx, params): """ Args: :param params: instance of mapping from String to unspecified object ctx: client_ip: None or 'str', user_id: str, 'authenticated': 1, 'token': str, 'module': None, 'method': None, 'call_id': None, 'rpc_context': None, 'provenance':list<prov_d> prov_d: (d) service: (str) 'method': 'please_never_use_it_in_production', 'method_params': []}]} :returns: instance of type "ReportResults" -> structure: parameter "report_name" of String, parameter "report_ref" of String """ # ctx is the context object # return variables are: output #BEGIN run_barseqR # SETUP - Unrelated to inputs -------- logging.basicConfig(level=logging.DEBUG) logging.info("Call back url: " + str(self.callback_url)) # We create important classes dfu = DataFileUtil(self.callback_url) logging.info("DFU VARS-- " * 8) logging.info(vars(dfu)) gfu = GenomeFileUtil(self.callback_url) smpl_s = SampleService(self.callback_url) myToken = os.environ.get('KB_AUTH_TOKEN', None) ws = Workspace(self.ws_url, token=myToken) ws_id = ws.get_workspace_info({'workspace': params['workspace_name']})[0] logging.info(os.environ) logging.info('ws-url') logging.info(self.ws_url) logging.info('ctx') logging.info(ctx) # We create indir, outdir, sets_dir (Input, Output, Sets) indir = os.path.join(self.shared_folder, "indir") os.mkdir(indir) outdir = os.path.join(self.shared_folder, "outdir") os.mkdir(outdir) sets_dir = os.path.join(indir, "sets_dir") os.mkdir(sets_dir) metadir = '/kb/module/lib/RunDir/metadata' if not (os.path.isdir(metadir)): raise Exception( "metadata directory not found at: {}".format(metadir)) # We prepare locations of input files poolfile_path = os.path.join(indir, "pool.n10") gene_table_fp = os.path.join(indir, "genes.GC") exps_file = os.path.join(indir, "FEBA_Barseq.tsv") # END SETUP # VALIDATE PARAMS: logging.info("PARAMS:") logging.info(params) # From Util.validate python file val_par = validate_params(params) ''' val_par contains keys: genome_ref poolfile_ref exps_ref sets_ref output_name workspace_name ''' val_par['username'] = ctx['user_id'] # DOWNLOAD FILES download_dict = { "dfu": dfu, "gfu": gfu, "ws": ws, "smpl_s": smpl_s, "sets_dir": sets_dir, "poolfile_path": poolfile_path, "gene_table_fp": gene_table_fp, "exps_file": exps_file, "scratch_dir": self.shared_folder } # We copy input files to proper directories. # vp must contain genome_ref, poolfile_ref, exps_ref, sets_refs (list) # DownloadResults must contain keys 'org', 'set_names_list', 'set_fps_list' # set_names_list value contains the names of the sets without extensions DownloadResults = download_files(val_par, download_dict) logging.debug(json.dumps(DownloadResults, indent=2)) # Get args in this format: # [-org, org_name, -indir, Scratch_Dir_Input, -metadir, Fixed meta dir, # -outdir, scratch_dir_output, -sets_dir, within scratch_dir_input, # -sets, set1 (sets_dir), set2 (sets_dir), set3 (sets_dir), ... ] # Note meta dir is called metadata and is in RunDir # Running the entire program: arg_list = [ "-org", DownloadResults['org'], '-indir', indir, '-metadir', metadir, '-outdir', outdir, '-sets_dir', sets_dir, '-sets' ] arg_list += DownloadResults['set_names_list'] RunBarSeq(arg_list) # Returning files to user report = KBaseReport(self.callback_url) report_info = report.create({ 'report': { 'objects_created': [], 'text_message': params['parameter_1'] }, 'workspace_name': params['workspace_name'] }) output = { 'report_name': report_info['name'], 'report_ref': report_info['ref'], } #END run_barseqR # At some point might do deeper type checking... if not isinstance(output, dict): raise ValueError('Method run_barseqR return value ' + 'output is not type dict as required.') # return the results return [output]
def run_poolcount(self, ctx, params): """ This example function accepts any number of parameters and returns results in a KBaseReport :param params: instance of mapping from String to unspecified object :returns: instance of type "ReportResults" -> structure: parameter "report_name" of String, parameter "report_ref" of String Args: params: ['workspace_name']: self.wsName, "mutantpool_ref": pool_ref (str), "fastq_files": list<fastq_refs (str)>, "genome_ref": genome_ref (str), "KB_BarcodeCount_Bool": "yes"/"no" - create a poolcount file? "poolcount_description": (str) A text description of the pool file, "output_name": (str), ## "test_local_bool": test_local_bool Deprecated ## "save_ignore_bool": bool, Deprecated "maxReads": int or None, "minQuality": int, "debug": bool, "protocol_type": str, "doOff1": bool """ # ctx is the context object # return variables are: output logging.basicConfig(level=logging.DEBUG) logging.warning("INPUT PARAMS:") logging.warning(params) #BEGIN run_poolcount report = KBaseReport(self.callback_url) myToken = os.environ.get('KB_AUTH_TOKEN', None) ws = Workspace(self.ws_url, token=myToken) ws_id = ws.get_workspace_info({'workspace': params['workspace_name']})[0] #Creating Data File Util Object dfu = DataFileUtil(self.callback_url) #We make the BarcodeCount output directory in scratch: outputs_dir = os.path.join(self.shared_folder, "BarcodeCount_Outputs") HTML_dir = os.path.join(self.shared_folder, "HTML_OP") MC_dir = os.path.join(self.shared_folder, "MC_op_dir") for x_dir in [outputs_dir, HTML_dir, MC_dir]: if os.path.isdir(x_dir): logging.info(f"{x_dir} contents: " + ",\n".join(os.listdir(x_dir))) else: os.mkdir(x_dir) main_HTML_fp = os.path.join(HTML_dir, "index.html") # parsed_params_dict contains keys: # mutantpool_ref, fastq_files_refs_list, genes_table_ref, output_name, # KB_BarcodeCount_Bool, poolcount_description parsed_params_dict = parse_and_check_params(params) # We get the username for later parsed_params_dict['username'] = ctx['user_id'] # We set the mutantpool's path mutantpool_path = os.path.join(self.shared_folder, "kb_pool.pool") download_mutantpool(parsed_params_dict['mutantpool_ref'], mutantpool_path, dfu, genome_ref=parsed_params_dict['genome_ref']) poolcount_prefix = os.path.join(outputs_dir, parsed_params_dict["output_name"]) fastq_dicts_list = download_fastq_and_prepare_mc( parsed_params_dict, dfu, self.shared_folder, outputs_dir) logging.info(fastq_dicts_list) #First we download all the Fastq Files and the Pool File report_dict = {} report_str = "" mc_run_list = fastq_dicts_list mc_run_num = len(mc_run_list) logging.info("Total MultiCodes Runs: {}".format(mc_run_num)) FullRun_d = get_FullRun_d(parsed_params_dict, MC_dir, fastq_dicts_list, poolcount_prefix, mutantpool_path, main_HTML_fp) # Running all programs: PC_RunAll(FullRun_d) # Now we upload the poolcount file to KBase to make a BarcodeCount Object if parsed_params_dict['KB_BarcodeCount_Bool']: upload_params = { 'username': parsed_params_dict['username'], 'fastq_refs': parsed_params_dict['fastq_files'], 'genome_ref': parsed_params_dict['genome_ref'], 'mutantpool_ref': parsed_params_dict['mutantpool_ref'], 'poolcount_description': parsed_params_dict['poolcount_description'], 'workspace_id': ws_id, 'ws_obj': ws, 'protocol_type': parsed_params_dict["protocol_type"], 'poolcount_fp': poolcount_prefix + ".poolcount", 'poolcount_name': parsed_params_dict['output_name'], 'dfu': dfu, "scratch_dir": self.shared_folder, "set_name": parsed_params_dict['output_name'] } logging.info("UPLOADING BarcodeCount FILE to KBASE through DFU") upload_mutantpool_results = upload_poolcount_to_KBase( upload_params) logging.info("Upload BarcodeCount File Results:") logging.info(upload_mutantpool_results) # DEBUGGING WHAT REPORT DICT LOOKS LIKE with open(os.path.join(self.shared_folder, "Report.JSON"), 'w') as g: g.write(json.dumps(report_dict, indent=2)) #Cleaning outputs dir (removing .codes, .close, .counts files) clean_output_dir(outputs_dir, self.shared_folder) #Writing report string: report_fp = os.path.join(outputs_dir, "Run_Report.txt") with open(report_fp, "w") as f: f.write(report_str) #Return Outputs Dir to user: dir_zip_shock_id = dfu.file_to_shock({ "file_path": outputs_dir, 'pack': 'zip' })['shock_id'] dir_link_dict = { 'shock_id': dir_zip_shock_id, 'name': parsed_params_dict['output_name'] + ".zip", 'label': 'RBTnSeqBarcodeCount_dir', 'description': 'The folder containing outputs from this app' } HTML_report_shock_id = dfu.file_to_shock({ "file_path": HTML_dir, "pack": "zip" })['shock_id'] HTML_report_d_l = [{ "shock_id": HTML_report_shock_id, "name": os.path.basename(os.path.join(HTML_dir, "index.html")), "label": "BarcodeCount Report", "description": "HTML Summary Report for MultiCodes and Combine BarSeq" }] extended_report_params = { 'workspace_name': params['workspace_name'], "html_links": HTML_report_d_l, "direct_html_link_index": 0, "html_window_height": 333, "report_object_name": "KB_BarcodeCount_Report", 'file_links': [dir_link_dict], 'message': "" } report_info = report.create_extended_report(extended_report_params) output = { 'report_name': report_info['name'], 'report_ref': report_info['ref'], } #END run_poolcount # At some point might do deeper type checking... if not isinstance(output, dict): raise ValueError('Method run_poolcount return value ' + 'output is not type dict as required.') # return the results return [output]