def check_workspace_name(ws_name, ws_url): """ Tests whether the given workspace name actually exists, is accessible by the current user, and is an appropriately formatted name. Really, it just pokes the Workspace with the name and returns True if it can. """ ws = Workspace(ws_url) try: # let the Workspace do the work - if this is NOT a real name, it will raise an exception. ws.get_workspace_info({"workspace": ws_name}) return True except: return False
def list_data(self, ctx, params): ''' ''' token = self._extract_token(ctx) if 'workspaces' not in params: raise ValueError( 'missing required field "workspaces" in parameters to list_data' ) if not isinstance(params['workspaces'], list): raise ValueError('"workspaces" field must be a list') workspaces = params['workspaces'] include_metadata = params.get('include_metadata', 0) ws = Workspace(self.ws_url, token=token) ws_info_list = [] if len(workspaces) == 1: workspace = workspaces[0] list_params = {} if str(workspace).isdigit(): list_params['id'] = int(workspace) else: list_params['workspace'] = str(workspace) ws_info_list.append(ws.get_workspace_info(list_params)) else: ws_map = {key: True for key in workspaces} for ws_info in ws.list_workspace_info({'perm': 'r'}): if ws_info[1] in ws_map or str(ws_info[0]) in ws_map: ws_info_list.append(ws_info) data = [] dp_list_filter = {'include_metadata': include_metadata} data_palette_refs = {} for ws_info in ws_info_list: dp = DataPalette(None, ws_info=ws_info, ws=ws) data = data + dp.list(dp_list_filter) dp_ref = dp._get_root_data_palette_ref() if dp_ref: data_palette_refs[str(ws_info[0])] = dp_ref data = self._remove_duplicate_data(data) return {'data': data, 'data_palette_refs': data_palette_refs}
class WorkspaceAdminUtil: def __init__(self, config): wsurl = config.get('workspace-url') self.atoken = config.get('ws-admin-token') self.noadmin = False if self.atoken is None or self.atoken == '': self.noadmin = True self.atoken = config['token'] self.ws = Workspace(wsurl, token=self.atoken) def list_objects(self, params): """ Provide something that acts like a standard listObjects """ if self.noadmin: return self.ws.list_objects(params) return self.ws.administer({'command': 'listObjects', 'params': params}) def get_objects2(self, params): """ Provide something that acts like a standard getObjects """ if self.noadmin: return self.ws.get_objects2(params) return self.ws.administer({'command': 'getObjects', 'params': params}) def get_workspace_info(self, params): """ Provide something that acts like a standard getObjects """ if self.noadmin: return self.ws.get_workspace_info(params) return self.ws.administer({ 'command': 'getWorkspaceInfo', 'params': params })
class NarrativeManager: KB_CELL = 'kb-cell' KB_TYPE = 'type' KB_APP_CELL = 'kb_app' KB_FUNCTION_CELL = 'function_input' KB_OUTPUT_CELL = 'function_output' KB_ERROR_CELL = 'kb_error' KB_CODE_CELL = 'kb_code' KB_STATE = 'widget_state' DEBUG = False DATA_PALETTES_TYPES = DataPaletteTypes(False) def __init__(self, config, ctx, set_api_cache, dps_cache): self.narrativeMethodStoreURL = config['narrative-method-store'] self.set_api_cache = set_api_cache # DynamicServiceCache type self.dps_cache = dps_cache # DynamicServiceCache type self.token = ctx["token"] self.user_id = ctx["user_id"] self.ws = Workspace(config['workspace-url'], token=self.token) self.intro_md_file = config['intro-markdown-file'] # We switch DPs on only for internal Continuous Integration environment for now: if config['kbase-endpoint'].startswith("https://ci.kbase.us/"): self.DATA_PALETTES_TYPES = DataPaletteTypes(True) def list_objects_with_sets(self, ws_id=None, ws_name=None, workspaces=None, types=None, include_metadata=0): if not workspaces: if (not ws_id) and (not ws_name): raise ValueError( "One and only one of 'ws_id', 'ws_name', 'workspaces' " + "parameters should be set") workspaces = [self._get_workspace_name_or_id(ws_id, ws_name)] return self._list_objects_with_sets(workspaces, types, include_metadata) def _list_objects_with_sets(self, workspaces, types, include_metadata): type_map = None if types is not None: type_map = {key: True for key in types} processed_refs = {} data = [] if self.DEBUG: print("NarrativeManager._list_objects_with_sets: processing sets") t1 = time.time() set_ret = self.set_api_cache.call_method( "list_sets", [{ 'workspaces': workspaces, 'include_set_item_info': 1, 'include_raw_data_palettes': 1, 'include_metadata': include_metadata }], self.token) sets = set_ret['sets'] dp_data = set_ret.get('raw_data_palettes') dp_refs = set_ret.get('raw_data_palette_refs') for set_info in sets: # Process target_set_items = [] for set_item in set_info['items']: target_set_items.append(set_item['info']) if self._check_info_type(set_info['info'], type_map): data_item = { 'object_info': set_info['info'], 'set_items': { 'set_items_info': target_set_items } } data.append(data_item) processed_refs[set_info['ref']] = data_item if self.DEBUG: print(" (time=" + str(time.time() - t1) + ")") if self.DEBUG: print("NarrativeManager._list_objects_with_sets: loading ws_info") t2 = time.time() ws_info_list = [] #for ws in workspaces: if len(workspaces) == 1: ws = workspaces[0] ws_id = None ws_name = None if str(ws).isdigit(): ws_id = int(ws) else: ws_name = str(ws) ws_info_list.append( self.ws.get_workspace_info({ "id": ws_id, "workspace": ws_name })) else: ws_map = {key: True for key in workspaces} for ws_info in self.ws.list_workspace_info({'perm': 'r'}): if ws_info[1] in ws_map or str(ws_info[0]) in ws_map: ws_info_list.append(ws_info) if self.DEBUG: print(" (time=" + str(time.time() - t2) + ")") if self.DEBUG: print( "NarrativeManager._list_objects_with_sets: loading workspace objects" ) t3 = time.time() for info in WorkspaceListObjectsIterator( self.ws, ws_info_list=ws_info_list, list_objects_params={'includeMetadata': include_metadata}): item_ref = str(info[6]) + '/' + str(info[0]) + '/' + str(info[4]) if item_ref not in processed_refs and self._check_info_type( info, type_map): data_item = {'object_info': info} data.append(data_item) processed_refs[item_ref] = data_item if self.DEBUG: print(" (time=" + str(time.time() - t3) + ")") if self.DEBUG: print( "NarrativeManager._list_objects_with_sets: processing DataPalettes" ) t5 = time.time() if dp_data is None or dp_refs is None: dps = self.dps_cache dp_ret = dps.call_method("list_data", [{ 'workspaces': workspaces, 'include_metadata': include_metadata }], self.token) dp_data = dp_ret['data'] dp_refs = dp_ret['data_palette_refs'] for item in dp_data: ref = item['ref'] if self._check_info_type(item['info'], type_map): data_item = None if ref in processed_refs: data_item = processed_refs[ref] else: data_item = {'object_info': item['info']} processed_refs[ref] = data_item data.append(data_item) dp_info = {} if 'dp_ref' in item: dp_info['ref'] = item['dp_ref'] if 'dp_refs' in item: dp_info['refs'] = item['dp_refs'] data_item['dp_info'] = dp_info if self.DEBUG: print(" (time=" + str(time.time() - t5) + ")") return {"data": data, 'data_palette_refs': dp_refs} def _check_info_type(self, info, type_map): if type_map is None: return True obj_type = info[2].split('-')[0] return type_map.get(obj_type, False) def copy_narrative(self, newName, workspaceRef, workspaceId): time_ms = int(round(time.time() * 1000)) newWsName = self.user_id + ':narrative_' + str(time_ms) # add the 'narrative' field to newWsMeta later. newWsMeta = {"is_temporary": "false", "narrative_nice_name": newName} # start with getting the existing narrative object. currentNarrative = self.ws.get_objects([{'ref': workspaceRef}])[0] if not workspaceId: workspaceId = currentNarrative['info'][6] # Let's prepare exceptions for clone the workspace. # 1) currentNarrative object: excluded_list = [{'objid': currentNarrative['info'][0]}] # 2) let's exclude objects of types under DataPalette handling: data_palette_type = "DataPalette.DataPalette" excluded_types = [data_palette_type] excluded_types.extend(self.DATA_PALETTES_TYPES.keys()) add_to_palette_list = [] dp_detected = False for obj_type in excluded_types: list_objects_params = {'type': obj_type} if obj_type == data_palette_type: list_objects_params['showHidden'] = 1 for info in WorkspaceListObjectsIterator( self.ws, ws_id=workspaceId, list_objects_params=list_objects_params): if obj_type == data_palette_type: dp_detected = True else: add_to_palette_list.append({ 'ref': str(info[6]) + '/' + str(info[0]) + '/' + str(info[4]) }) excluded_list.append({'objid': info[0]}) # clone the workspace EXCEPT for currentNarrative object + obejcts of DataPalette types: newWsId = self.ws.clone_workspace({ 'wsi': { 'id': workspaceId }, 'workspace': newWsName, 'meta': newWsMeta, 'exclude': excluded_list })[0] try: if dp_detected: self.dps_cache.call_method( "copy_palette", [{ 'from_workspace': str(workspaceId), 'to_workspace': str(newWsId) }], self.token) if len(add_to_palette_list) > 0: # There are objects in source workspace that have type under DataPalette handling # but these objects are physically stored in source workspace rather that saved # in DataPalette object. So they weren't copied by "dps.copy_palette". self.dps_cache.call_method("add_to_palette", [{ 'workspace': str(newWsId), 'new_refs': add_to_palette_list }], self.token) # update the ref inside the narrative object and the new workspace metadata. newNarMetadata = currentNarrative['info'][10] newNarMetadata['name'] = newName newNarMetadata['ws_name'] = newWsName newNarMetadata['job_info'] = json.dumps({ 'queue_time': 0, 'running': 0, 'completed': 0, 'run_time': 0, 'error': 0 }) currentNarrative['data']['metadata']['name'] = newName currentNarrative['data']['metadata']['ws_name'] = newWsName currentNarrative['data']['metadata']['job_ids'] = { 'apps': [], 'methods': [], 'job_usage': { 'queue_time': 0, 'run_time': 0 } } # save the shiny new Narrative so it's at version 1 newNarInfo = self.ws.save_objects({ 'id': newWsId, 'objects': [{ 'type': currentNarrative['info'][2], 'data': currentNarrative['data'], 'provenance': currentNarrative['provenance'], 'name': currentNarrative['info'][1], 'meta': newNarMetadata }] }) # now, just update the workspace metadata to point # to the new narrative object newNarId = newNarInfo[0][0] self.ws.alter_workspace_metadata({ 'wsi': { 'id': newWsId }, 'new': { 'narrative': str(newNarId) } }) return {'newWsId': newWsId, 'newNarId': newNarId} except: # let's delete copy of workspace so it's out of the way - it's broken self.ws.delete_workspace({'id': newWsId}) raise # continue raising previous exception def create_new_narrative(self, app, method, appparam, appData, markdown, copydata, importData, includeIntroCell): if app and method: raise ValueError( "Must provide no more than one of the app or method params") if (not importData) and copydata: importData = copydata.split(';') if (not appData) and appparam: appData = [] for tmp_item in appparam.split(';'): tmp_tuple = tmp_item.split(',') step_pos = None if tmp_tuple[0]: try: step_pos = int(tmp_tuple[0]) except ValueError: pass appData.append([step_pos, tmp_tuple[1], tmp_tuple[2]]) cells = None if app: cells = [{"app": app}] elif method: cells = [{"method": method}] elif markdown: cells = [{"markdown": markdown}] return self._create_temp_narrative(cells, appData, importData, includeIntroCell) def _get_intro_markdown(self): """ Creates and returns a cell with the introductory text included. """ # Load introductory markdown text with open(self.intro_md_file) as intro_file: intro_md = intro_file.read() return intro_md def _create_temp_narrative(self, cells, parameters, importData, includeIntroCell): # Migration to python of JavaScript class from https://github.com/kbase/kbase-ui/blob/4d31151d13de0278765a69b2b09f3bcf0e832409/src/client/modules/plugins/narrativemanager/modules/narrativeManager.js#L414 narr_id = int(round(time.time() * 1000)) workspaceName = self.user_id + ':narrative_' + str(narr_id) narrativeName = "Narrative." + str(narr_id) ws = self.ws ws_info = ws.create_workspace({ 'workspace': workspaceName, 'description': '' }) newWorkspaceInfo = ServiceUtils.workspaceInfoToObject(ws_info) [narrativeObject, metadataExternal ] = self._fetchNarrativeObjects(workspaceName, cells, parameters, includeIntroCell) objectInfo = ws.save_objects({ 'workspace': workspaceName, 'objects': [{ 'type': 'KBaseNarrative.Narrative', 'data': narrativeObject, 'name': narrativeName, 'meta': metadataExternal, 'provenance': [{ 'script': 'NarrativeManager.py', 'description': 'Created new ' + 'Workspace/Narrative bundle.' }], 'hidden': 0 }] })[0] objectInfo = ServiceUtils.objectInfoToObject(objectInfo) self._completeNewNarrative(newWorkspaceInfo['id'], objectInfo['id'], importData) return {'workspaceInfo': newWorkspaceInfo, 'narrativeInfo': objectInfo} def _fetchNarrativeObjects(self, workspaceName, cells, parameters, includeIntroCell): if not cells: cells = [] # fetchSpecs appSpecIds = [] methodSpecIds = [] specMapping = {'apps': {}, 'methods': {}} for cell in cells: if 'app' in cell: appSpecIds.append(cell['app']) elif 'method' in cell: methodSpecIds.append(cell['method']) nms = NarrativeMethodStore(self.narrativeMethodStoreURL, token=self.token) if len(appSpecIds) > 0: appSpecs = nms.get_app_spec({'ids': appSpecIds}) for spec in appSpecs: spec_id = spec['info']['id'] specMapping['apps'][spec_id] = spec if len(methodSpecIds) > 0: methodSpecs = nms.get_method_spec({'ids': methodSpecIds}) for spec in methodSpecs: spec_id = spec['info']['id'] specMapping['methods'][spec_id] = spec # end of fetchSpecs metadata = { 'job_ids': { 'methods': [], 'apps': [], 'job_usage': { 'queue_time': 0, 'run_time': 0 } }, 'format': 'ipynb', 'creator': self.user_id, 'ws_name': workspaceName, 'name': 'Untitled', 'type': 'KBaseNarrative.Narrative', 'description': '', 'data_dependencies': [] } cellData = self._gatherCellData(cells, specMapping, parameters, includeIntroCell) narrativeObject = { 'nbformat_minor': 0, 'cells': cellData, 'metadata': metadata, 'nbformat': 4 } metadataExternal = {} for key in metadata: value = metadata[key] if isinstance(value, basestring): metadataExternal[key] = value else: metadataExternal[key] = json.dumps(value) return [narrativeObject, metadataExternal] def _gatherCellData(self, cells, specMapping, parameters, includeIntroCell): cell_data = [] if includeIntroCell == 1: cell_data.append({ 'cell_type': 'markdown', 'source': self._get_intro_markdown(), 'metadata': {} }) for cell_pos, cell in enumerate(cells): if 'app' in cell: cell_data.append( self._buildAppCell(len(cell_data), specMapping['apps'][cell['app']], parameters)) elif 'method' in cell: cell_data.append( self._buildMethodCell( len(cell_data), specMapping['methods'][cell['method']], parameters)) elif 'markdown' in cell: cell_data.append({ 'cell_type': 'markdown', 'source': cell['markdown'], 'metadata': {} }) else: raise ValueError("cannot add cell #" + str(cell_pos) + ", unrecognized cell content") return cell_data def _buildAppCell(self, pos, spec, params): cellId = 'kb-cell-' + str(pos) + '-' + str(uuid.uuid4()) cell = { 'cell_type': 'markdown', 'source': "<div id='" + cellId + "'></div>" + "\n<script>" + "$('#" + cellId + "').kbaseNarrativeAppCell({'appSpec' : '" + self._safeJSONStringify(spec) + "', 'cellId' : '" + cellId + "'});" + "</script>", 'metadata': {} } cellInfo = {} widgetState = [] cellInfo[self.KB_TYPE] = self.KB_APP_CELL cellInfo['app'] = spec if params: steps = {} for param in params: stepid = 'step_' + str(param[0]) if stepid not in steps: steps[stepid] = {} steps[stepid]['inputState'] = {} steps[stepid]['inputState'][param[1]] = param[2] state = { 'state': { 'step': steps } } widgetState.append(state) cellInfo[self.KB_STATE] = widgetState cell['metadata'][self.KB_CELL] = cellInfo return cell def _buildMethodCell(self, pos, spec, params): cellId = 'kb-cell-' + str(pos) + '-' + str(uuid.uuid4()) cell = { 'cell_type': 'markdown', 'source': "<div id='" + cellId + "'></div>" + "\n<script>" + "$('#" + cellId + "').kbaseNarrativeMethodCell({'method' : '" + self._safeJSONStringify(spec) + "'});" + "</script>", 'metadata': {} } cellInfo = {'method': spec, 'widget': spec['widgets']['input']} cellInfo[self.KB_TYPE] = self.KB_FUNCTION_CELL widgetState = [] if params: wparams = {} for param in params: wparams[param[1]] = param[2] widgetState.append({'state': wparams}) cellInfo[self.KB_STATE] = widgetState cell['metadata'][self.KB_CELL] = cellInfo return cell def _completeNewNarrative(self, workspaceId, objectId, importData): self.ws.alter_workspace_metadata({ 'wsi': { 'id': workspaceId }, 'new': { 'narrative': str(objectId), 'is_temporary': 'true' } }) # copy_to_narrative: if not importData: return objectsToCopy = [{'ref': x} for x in importData] infoList = self.ws.get_object_info_new({ 'objects': objectsToCopy, 'includeMetadata': 0 }) for item in infoList: objectInfo = ServiceUtils.objectInfoToObject(item) self.copy_object(objectInfo['ref'], workspaceId, None, None, objectInfo) def _safeJSONStringify(self, obj): return json.dumps(self._safeJSONStringifyPrepare(obj)) def _safeJSONStringifyPrepare(self, obj): if isinstance(obj, basestring): return obj.replace("'", "'").replace('"', """) elif isinstance(obj, list): for pos in range(len(obj)): obj[pos] = self._safeJSONStringifyPrepare(obj[pos]) elif isinstance(obj, dict): obj_keys = list(obj.keys()) for key in obj_keys: obj[key] = self._safeJSONStringifyPrepare(obj[key]) else: pass # it's boolean/int/float/None return obj def _get_workspace_name_or_id(self, ws_id, ws_name): ret = ws_name if not ret: ret = str(ws_id) return ret def copy_object(self, ref, target_ws_id, target_ws_name, target_name, src_info): # There should be some logic related to DataPalettes if (not target_ws_id) and (not target_ws_name): raise ValueError("Neither target workspace ID nor name is defined") if not src_info: src_info_tuple = self.ws.get_object_info_new({ 'objects': [{ 'ref': ref }], 'includeMetadata': 0 })[0] src_info = ServiceUtils.objectInfoToObject(src_info_tuple) type_name = src_info['typeModule'] + '.' + src_info['typeName'] type_config = self.DATA_PALETTES_TYPES.get(type_name) if type_config is not None: # Copy with DataPaletteService if target_name: raise ValueError( "'target_name' cannot be defined for DataPalette copy") target_ws_name_or_id = self._get_workspace_name_or_id( target_ws_id, target_ws_name) self.dps_cache.call_method("add_to_palette", [{ 'workspace': target_ws_name_or_id, 'new_refs': [{ 'ref': ref }] }], self.token) return {'info': src_info} else: if not target_name: target_name = src_info['name'] obj_info_tuple = self.ws.copy_object({ 'from': { 'ref': ref }, 'to': { 'wsid': target_ws_id, 'workspace': target_ws_name, 'name': target_name } }) obj_info = ServiceUtils.objectInfoToObject(obj_info_tuple) return {'info': obj_info} def list_available_types(self, workspaces): data = self.list_objects_with_sets(workspaces=workspaces)['data'] type_stat = {} for item in data: info = item['object_info'] obj_type = info[2].split('-')[0] if obj_type in type_stat: type_stat[obj_type] += 1 else: type_stat[obj_type] = 1 return {'type_stat': type_stat}
from Workspace.WorkspaceClient import Workspace import json wsid = 16962 upa = '16962/3' upa2 = '16962/23' ws = Workspace('https://ci.kbase.us/services/ws') d = ws.get_workspace_info({'id': wsid}) with open('get_workspace_info.json', 'w') as f: f.write(json.dumps(d)) d = ws.list_objects({'ids': [wsid]}) with open('list_objects.json', 'w') as f: f.write(json.dumps(d)) d = ws.get_objects2({'objects': [{'ref': upa}]}) with open('get_objects.json', 'w') as f: f.write(json.dumps(d)) d = ws.get_objects2({'objects': [{'ref': upa2}]}) with open('narrative_object.json', 'w') as f: f.write(json.dumps(d))
class DataPalette(): PROVENANCE = [{'service': 'DataPaletteService'}] DATA_PALETTE_WS_METADATA_KEY = 'data_palette_id' DEFAULT_PALETTE_OBJ_NAME = 'data_palette' PALETTE_OBJ_WS_TYPE = 'DataPalette.DataPalette' # set of types that cannot be added to a data palette, add to configuration PROHIBITED_DATA_TYPES = ['KBaseReport.Report', 'KBaseNarrative.Narrative', 'DataPalette.DataPalette'] def __init__(self, ws_name_or_id, ws_url=None, token=None, ws_info=None, ws=None): if ws: self.ws = ws else: if ws_url is None: raise ValueError('ws_url was not defined') if token is None: print('DataPalette warning: token was not set') self.ws = Workspace(ws_url, token=token) if ws_info: if ws_name_or_id: raise ValueError("Either ws_name_or_id or ws_info should be set") self.ws_info = WorkspaceInfo(ws_info) else: if str(ws_name_or_id).isdigit(): self.ws_info = WorkspaceInfo(self.ws.get_workspace_info({'id': int(ws_name_or_id)})) else: self.ws_info = WorkspaceInfo(self.ws.get_workspace_info({ 'workspace': str(ws_name_or_id) })) self.palette_ref = None def list(self, options): # if there is no data palette, return nothing dp_ref = self._get_root_data_palette_ref() if dp_ref is None: return [] palette = self._get_data_palette() include_metadata = options.get('include_metadata', 0) palette = self._attach_palette_data_info(palette, include_metadata) return palette['data'] def add(self, refs=None): ''' Adds the provided references to the data palette. ''' if len(refs) == 0: return {} # make sure the references to add are visible and valid objs = self._get_object_info(refs) self._validate_objects_to_add(objs) # get the existing palette and build an index palette = self._get_data_palette() data_index = self._build_palette_data_index(palette['data']) # changing refs in DataPalette so that they are pointed through # DataPalette object ref as ref-path self._extend_ref_paths_before_saving(palette) # perform the actual update palette update for obj_pos in range(0, len(objs)): o = objs[obj_pos] ws = str(o[6]) obj = str(o[0]) ver = str(o[4]) ref = refs[obj_pos]['ref'] #ws + '/' + obj + '/' + ver if ws + '/' + obj in data_index: # the object is in the palette, so check versions index = data_index[ws + '/' + obj] if index['ver'] == ver: # the version didn't change, so continue continue # the version is different, so update it data_index[ws + '/' + obj]['ver'] = ver palette['data'][index['idx']]['ref'] = ref else: # the object wasn't in the palette, so add it idx = len(palette['data']) palette['data'].append({'ref': ref}) data_index[ws + '/' + obj] = {'ver': ver, 'idx': idx} # save the updated palette and return self._save_data_palette(palette) return {} def remove(self, refs=None): dp_ref = self._get_root_data_palette_ref() if dp_ref is None: raise ValueError('Cannot remove from data_palette- data palette ' + 'for Workspace does not exist') if len(refs) == 0: return {} # right now, we only match on exact refs, so this works palette = self._get_data_palette() data_index = self._build_palette_data_index(palette['data']) index_to_delete = [] for r in range(0, len(refs)): ref = refs[r]['ref'] tokens = ref.split('/') if len(tokens) != 3: raise ValueError('Invalid absolute reference: ' + str(ref) + ' at position ' + str(r) + ' of removal list. References must be full, absolute numerical WS refs.') is_digits = map(lambda x: x.isdigit(), tokens) if False in is_digits: raise ValueError('Invalid absolute reference: ' + str(ref) + ' at position ' + str(r) + ' of removal list. References must be full, absolute numerical WS refs.') ws_slash_id = tokens[0] + '/' + tokens[1] if ws_slash_id in data_index: if data_index[ws_slash_id]['ver'] == tokens[2]: index_to_delete.append(data_index[ws_slash_id]['idx']) else: raise ValueError('Reference: ' + str(ref) + ' at position ' + str(r) + ' of removal list was not found in palette. Object exists, but ' + 'version was not correct.') else: raise ValueError('Reference: ' + str(ref) + ' at position ' + str(r) + ' of removal list was not found in palette.') index_to_delete = set(index_to_delete) for i in sorted(index_to_delete, reverse=True): del palette['data'][i] self._extend_ref_paths_before_saving(palette) self._save_data_palette(palette) return {} def _build_palette_data_index(self, palette_data): data_index = {} for k in range(0, len(palette_data)): tokens = palette_data[k]['ref'].split('/') key = tokens[0] + '/' + tokens[1] value = {'ver': tokens[2], 'idx': k} data_index[key] = value return data_index def _get_object_info(self, objects): return self.ws.get_object_info_new({'objects': objects}) def _validate_objects_to_add(self, object_info_list): for info in object_info_list: # validate type, split and ignore the type version full_type_name = info[2].split('-')[0] if full_type_name in self.PROHIBITED_DATA_TYPES: raise ValueError('Object ' + str(info[1]) + ' (id=' + str(info[6]) + '/' + str(info[0]) + '/' + str(info[4]) + ') is a type (' + full_type_name + ') that cannot be added to a data palette.') def _attach_palette_data_info(self, palette, include_metadata = 0): # TODO: make sure we get object info via reference chain if len(palette['data']) == 0: return palette palette_ref = self._get_root_data_palette_ref() info_input = [{'ref': palette_ref + ';' + obj['ref']} for obj in palette['data']] all_info = self.ws.get_object_info_new({ 'objects': info_input, 'includeMetadata': include_metadata }) dp_ref = self._get_root_data_palette_ref() for k in range(0, len(all_info)): palette['data'][k]['info'] = all_info[k] palette['data'][k]['dp_ref'] = dp_ref palette['data'][k]['dp_refs'] = [dp_ref] return palette def _extend_ref_paths_before_saving(self, palette): dp_ref = self._get_root_data_palette_ref() for data_ref in palette['data']: data_ref['ref'] = dp_ref + ';' + data_ref['ref'] def _save_data_palette(self, palette): obj_info = self.ws.save_objects({ 'id': self.ws_info.id, 'objects': [{ 'type': self.PALETTE_OBJ_WS_TYPE, 'objid': self._get_root_data_palette_objid(), 'data': palette, 'provenance': self.PROVENANCE, 'hidden': 1 }] })[0] return obj_info def _get_data_palette(self): palette_ref = self._get_root_data_palette_ref() if palette_ref is None: return self._create_data_palette() data = self.ws.get_objects2({ 'objects': [{'ref': palette_ref}] }) return data['data'][0]['data'] def _create_data_palette(self): # 1) save the data_palette object palette = {'data': []} new_palette_info = self.ws.save_objects({ 'id': self.ws_info.id, 'objects': [{ 'type': self.PALETTE_OBJ_WS_TYPE, 'name': self.DEFAULT_PALETTE_OBJ_NAME, 'data': palette, 'provenance': self.PROVENANCE, 'hidden': 1 }] })[0] # 2) update ws metadata self._update_ws_palette_metadata(new_palette_info) return palette def create_from_existing_palette(self, existing_data_palette): # 1) make sure we can actually do it dp_target_ref = self._get_root_data_palette_ref() if dp_target_ref is not None: raise ValueError('Cannot copy data_palette- a data palette already exists in that workspace.') dp_source_ref = existing_data_palette._get_root_data_palette_ref() if dp_source_ref is None: # data palette did not exist, so we don't have to copy over anything return {} # 2) make the copy new_palette_info = self.ws.copy_object({ 'from': {'ref': dp_source_ref}, 'to': {'wsid': self.ws_info.id, 'name': self.DEFAULT_PALETTE_OBJ_NAME} }) # 3) update ws metadata self._update_ws_palette_metadata(new_palette_info) return {} def set_palette_to_obj(self, new_data_palette_name_or_id): if new_data_palette_name_or_id is None: new_data_palette_name_or_id = self.DEFAULT_PALETTE_OBJ_NAME new_palette_ref = str(self.ws_info.id) + '/' + str(new_data_palette_name_or_id) new_palette_info = self._get_object_info([{'ref': new_palette_ref}])[0] if not str(new_palette_info[2]).startswith(self.PALETTE_OBJ_WS_TYPE): raise ValueError('Cannot set data palette for workspace to non-palette type. Type of (' + new_palette_ref + ') was: ' + str(new_palette_info[1])) self._update_ws_palette_metadata(new_palette_info) return {} def _get_root_data_palette_objid(self): ref = self._get_root_data_palette_ref() if ref is None: return None return self._get_root_data_palette_ref().split('/')[1] def _get_root_data_palette_ref(self): if self.palette_ref is not None: return self.palette_ref if self.DATA_PALETTE_WS_METADATA_KEY not in self.ws_info.metadata: return None dp_id = self.ws_info.metadata[self.DATA_PALETTE_WS_METADATA_KEY] if not str(dp_id).isdigit(): raise ValueError('Warning: WS metadata for ' + str(self.ws_info.id) + 'was corrupted. It is not set to an object ID. It was: ' + str(dp_id)) self.palette_ref = str(self.ws_info.id) + '/' + str(dp_id) return self.palette_ref def _update_ws_palette_metadata(self, palette_obj_info): self.ws.alter_workspace_metadata({ 'wsi': { 'id': self.ws_info.id }, 'new': { self.DATA_PALETTE_WS_METADATA_KEY: str(palette_obj_info[0]) } }) # refresh local ws info self.ws_info = WorkspaceInfo(self.ws.get_workspace_info({'id': self.ws_info.id}))