def check_workspace_name(ws_name, ws_url):
"""
Tests whether the given workspace name actually exists, is accessible by the current user,
and is an appropriately formatted name.
Really, it just pokes the Workspace with the name and returns True if it can.
"""
ws = Workspace(ws_url)
try:
# let the Workspace do the work - if this is NOT a real name, it will raise an exception.
ws.get_workspace_info({"workspace": ws_name})
return True
except:
return False
def list_data(self, ctx, params):
'''
'''
token = self._extract_token(ctx)
if 'workspaces' not in params:
raise ValueError(
'missing required field "workspaces" in parameters to list_data'
)
if not isinstance(params['workspaces'], list):
raise ValueError('"workspaces" field must be a list')
workspaces = params['workspaces']
include_metadata = params.get('include_metadata', 0)
ws = Workspace(self.ws_url, token=token)
ws_info_list = []
if len(workspaces) == 1:
workspace = workspaces[0]
list_params = {}
if str(workspace).isdigit():
list_params['id'] = int(workspace)
else:
list_params['workspace'] = str(workspace)
ws_info_list.append(ws.get_workspace_info(list_params))
else:
ws_map = {key: True for key in workspaces}
for ws_info in ws.list_workspace_info({'perm': 'r'}):
if ws_info[1] in ws_map or str(ws_info[0]) in ws_map:
ws_info_list.append(ws_info)
data = []
dp_list_filter = {'include_metadata': include_metadata}
data_palette_refs = {}
for ws_info in ws_info_list:
dp = DataPalette(None, ws_info=ws_info, ws=ws)
data = data + dp.list(dp_list_filter)
dp_ref = dp._get_root_data_palette_ref()
if dp_ref:
data_palette_refs[str(ws_info[0])] = dp_ref
data = self._remove_duplicate_data(data)
return {'data': data, 'data_palette_refs': data_palette_refs}
class WorkspaceAdminUtil:
def __init__(self, config):
wsurl = config.get('workspace-url')
self.atoken = config.get('ws-admin-token')
self.noadmin = False
if self.atoken is None or self.atoken == '':
self.noadmin = True
self.atoken = config['token']
self.ws = Workspace(wsurl, token=self.atoken)
def list_objects(self, params):
"""
Provide something that acts like a standard listObjects
"""
if self.noadmin:
return self.ws.list_objects(params)
return self.ws.administer({'command': 'listObjects', 'params': params})
def get_objects2(self, params):
"""
Provide something that acts like a standard getObjects
"""
if self.noadmin:
return self.ws.get_objects2(params)
return self.ws.administer({'command': 'getObjects', 'params': params})
def get_workspace_info(self, params):
"""
Provide something that acts like a standard getObjects
"""
if self.noadmin:
return self.ws.get_workspace_info(params)
return self.ws.administer({
'command': 'getWorkspaceInfo',
'params': params
})
class NarrativeManager:
KB_CELL = 'kb-cell'
KB_TYPE = 'type'
KB_APP_CELL = 'kb_app'
KB_FUNCTION_CELL = 'function_input'
KB_OUTPUT_CELL = 'function_output'
KB_ERROR_CELL = 'kb_error'
KB_CODE_CELL = 'kb_code'
KB_STATE = 'widget_state'
DEBUG = False
DATA_PALETTES_TYPES = DataPaletteTypes(False)
def __init__(self, config, ctx, set_api_cache, dps_cache):
self.narrativeMethodStoreURL = config['narrative-method-store']
self.set_api_cache = set_api_cache # DynamicServiceCache type
self.dps_cache = dps_cache # DynamicServiceCache type
self.token = ctx["token"]
self.user_id = ctx["user_id"]
self.ws = Workspace(config['workspace-url'], token=self.token)
self.intro_md_file = config['intro-markdown-file']
# We switch DPs on only for internal Continuous Integration environment for now:
if config['kbase-endpoint'].startswith("https://ci.kbase.us/"):
self.DATA_PALETTES_TYPES = DataPaletteTypes(True)
def list_objects_with_sets(self,
ws_id=None,
ws_name=None,
workspaces=None,
types=None,
include_metadata=0):
if not workspaces:
if (not ws_id) and (not ws_name):
raise ValueError(
"One and only one of 'ws_id', 'ws_name', 'workspaces' " +
"parameters should be set")
workspaces = [self._get_workspace_name_or_id(ws_id, ws_name)]
return self._list_objects_with_sets(workspaces, types,
include_metadata)
def _list_objects_with_sets(self, workspaces, types, include_metadata):
type_map = None
if types is not None:
type_map = {key: True for key in types}
processed_refs = {}
data = []
if self.DEBUG:
print("NarrativeManager._list_objects_with_sets: processing sets")
t1 = time.time()
set_ret = self.set_api_cache.call_method(
"list_sets", [{
'workspaces': workspaces,
'include_set_item_info': 1,
'include_raw_data_palettes': 1,
'include_metadata': include_metadata
}], self.token)
sets = set_ret['sets']
dp_data = set_ret.get('raw_data_palettes')
dp_refs = set_ret.get('raw_data_palette_refs')
for set_info in sets:
# Process
target_set_items = []
for set_item in set_info['items']:
target_set_items.append(set_item['info'])
if self._check_info_type(set_info['info'], type_map):
data_item = {
'object_info': set_info['info'],
'set_items': {
'set_items_info': target_set_items
}
}
data.append(data_item)
processed_refs[set_info['ref']] = data_item
if self.DEBUG:
print(" (time=" + str(time.time() - t1) + ")")
if self.DEBUG:
print("NarrativeManager._list_objects_with_sets: loading ws_info")
t2 = time.time()
ws_info_list = []
#for ws in workspaces:
if len(workspaces) == 1:
ws = workspaces[0]
ws_id = None
ws_name = None
if str(ws).isdigit():
ws_id = int(ws)
else:
ws_name = str(ws)
ws_info_list.append(
self.ws.get_workspace_info({
"id": ws_id,
"workspace": ws_name
}))
else:
ws_map = {key: True for key in workspaces}
for ws_info in self.ws.list_workspace_info({'perm': 'r'}):
if ws_info[1] in ws_map or str(ws_info[0]) in ws_map:
ws_info_list.append(ws_info)
if self.DEBUG:
print(" (time=" + str(time.time() - t2) + ")")
if self.DEBUG:
print(
"NarrativeManager._list_objects_with_sets: loading workspace objects"
)
t3 = time.time()
for info in WorkspaceListObjectsIterator(
self.ws,
ws_info_list=ws_info_list,
list_objects_params={'includeMetadata': include_metadata}):
item_ref = str(info[6]) + '/' + str(info[0]) + '/' + str(info[4])
if item_ref not in processed_refs and self._check_info_type(
info, type_map):
data_item = {'object_info': info}
data.append(data_item)
processed_refs[item_ref] = data_item
if self.DEBUG:
print(" (time=" + str(time.time() - t3) + ")")
if self.DEBUG:
print(
"NarrativeManager._list_objects_with_sets: processing DataPalettes"
)
t5 = time.time()
if dp_data is None or dp_refs is None:
dps = self.dps_cache
dp_ret = dps.call_method("list_data",
[{
'workspaces': workspaces,
'include_metadata': include_metadata
}], self.token)
dp_data = dp_ret['data']
dp_refs = dp_ret['data_palette_refs']
for item in dp_data:
ref = item['ref']
if self._check_info_type(item['info'], type_map):
data_item = None
if ref in processed_refs:
data_item = processed_refs[ref]
else:
data_item = {'object_info': item['info']}
processed_refs[ref] = data_item
data.append(data_item)
dp_info = {}
if 'dp_ref' in item:
dp_info['ref'] = item['dp_ref']
if 'dp_refs' in item:
dp_info['refs'] = item['dp_refs']
data_item['dp_info'] = dp_info
if self.DEBUG:
print(" (time=" + str(time.time() - t5) + ")")
return {"data": data, 'data_palette_refs': dp_refs}
def _check_info_type(self, info, type_map):
if type_map is None:
return True
obj_type = info[2].split('-')[0]
return type_map.get(obj_type, False)
def copy_narrative(self, newName, workspaceRef, workspaceId):
time_ms = int(round(time.time() * 1000))
newWsName = self.user_id + ':narrative_' + str(time_ms)
# add the 'narrative' field to newWsMeta later.
newWsMeta = {"is_temporary": "false", "narrative_nice_name": newName}
# start with getting the existing narrative object.
currentNarrative = self.ws.get_objects([{'ref': workspaceRef}])[0]
if not workspaceId:
workspaceId = currentNarrative['info'][6]
# Let's prepare exceptions for clone the workspace.
# 1) currentNarrative object:
excluded_list = [{'objid': currentNarrative['info'][0]}]
# 2) let's exclude objects of types under DataPalette handling:
data_palette_type = "DataPalette.DataPalette"
excluded_types = [data_palette_type]
excluded_types.extend(self.DATA_PALETTES_TYPES.keys())
add_to_palette_list = []
dp_detected = False
for obj_type in excluded_types:
list_objects_params = {'type': obj_type}
if obj_type == data_palette_type:
list_objects_params['showHidden'] = 1
for info in WorkspaceListObjectsIterator(
self.ws,
ws_id=workspaceId,
list_objects_params=list_objects_params):
if obj_type == data_palette_type:
dp_detected = True
else:
add_to_palette_list.append({
'ref':
str(info[6]) + '/' + str(info[0]) + '/' + str(info[4])
})
excluded_list.append({'objid': info[0]})
# clone the workspace EXCEPT for currentNarrative object + obejcts of DataPalette types:
newWsId = self.ws.clone_workspace({
'wsi': {
'id': workspaceId
},
'workspace': newWsName,
'meta': newWsMeta,
'exclude': excluded_list
})[0]
try:
if dp_detected:
self.dps_cache.call_method(
"copy_palette", [{
'from_workspace': str(workspaceId),
'to_workspace': str(newWsId)
}], self.token)
if len(add_to_palette_list) > 0:
# There are objects in source workspace that have type under DataPalette handling
# but these objects are physically stored in source workspace rather that saved
# in DataPalette object. So they weren't copied by "dps.copy_palette".
self.dps_cache.call_method("add_to_palette",
[{
'workspace': str(newWsId),
'new_refs': add_to_palette_list
}], self.token)
# update the ref inside the narrative object and the new workspace metadata.
newNarMetadata = currentNarrative['info'][10]
newNarMetadata['name'] = newName
newNarMetadata['ws_name'] = newWsName
newNarMetadata['job_info'] = json.dumps({
'queue_time': 0,
'running': 0,
'completed': 0,
'run_time': 0,
'error': 0
})
currentNarrative['data']['metadata']['name'] = newName
currentNarrative['data']['metadata']['ws_name'] = newWsName
currentNarrative['data']['metadata']['job_ids'] = {
'apps': [],
'methods': [],
'job_usage': {
'queue_time': 0,
'run_time': 0
}
}
# save the shiny new Narrative so it's at version 1
newNarInfo = self.ws.save_objects({
'id':
newWsId,
'objects': [{
'type': currentNarrative['info'][2],
'data': currentNarrative['data'],
'provenance': currentNarrative['provenance'],
'name': currentNarrative['info'][1],
'meta': newNarMetadata
}]
})
# now, just update the workspace metadata to point
# to the new narrative object
newNarId = newNarInfo[0][0]
self.ws.alter_workspace_metadata({
'wsi': {
'id': newWsId
},
'new': {
'narrative': str(newNarId)
}
})
return {'newWsId': newWsId, 'newNarId': newNarId}
except:
# let's delete copy of workspace so it's out of the way - it's broken
self.ws.delete_workspace({'id': newWsId})
raise # continue raising previous exception
def create_new_narrative(self, app, method, appparam, appData, markdown,
copydata, importData, includeIntroCell):
if app and method:
raise ValueError(
"Must provide no more than one of the app or method params")
if (not importData) and copydata:
importData = copydata.split(';')
if (not appData) and appparam:
appData = []
for tmp_item in appparam.split(';'):
tmp_tuple = tmp_item.split(',')
step_pos = None
if tmp_tuple[0]:
try:
step_pos = int(tmp_tuple[0])
except ValueError:
pass
appData.append([step_pos, tmp_tuple[1], tmp_tuple[2]])
cells = None
if app:
cells = [{"app": app}]
elif method:
cells = [{"method": method}]
elif markdown:
cells = [{"markdown": markdown}]
return self._create_temp_narrative(cells, appData, importData,
includeIntroCell)
def _get_intro_markdown(self):
"""
Creates and returns a cell with the introductory text included.
"""
# Load introductory markdown text
with open(self.intro_md_file) as intro_file:
intro_md = intro_file.read()
return intro_md
def _create_temp_narrative(self, cells, parameters, importData,
includeIntroCell):
# Migration to python of JavaScript class from https://github.com/kbase/kbase-ui/blob/4d31151d13de0278765a69b2b09f3bcf0e832409/src/client/modules/plugins/narrativemanager/modules/narrativeManager.js#L414
narr_id = int(round(time.time() * 1000))
workspaceName = self.user_id + ':narrative_' + str(narr_id)
narrativeName = "Narrative." + str(narr_id)
ws = self.ws
ws_info = ws.create_workspace({
'workspace': workspaceName,
'description': ''
})
newWorkspaceInfo = ServiceUtils.workspaceInfoToObject(ws_info)
[narrativeObject, metadataExternal
] = self._fetchNarrativeObjects(workspaceName, cells, parameters,
includeIntroCell)
objectInfo = ws.save_objects({
'workspace':
workspaceName,
'objects': [{
'type':
'KBaseNarrative.Narrative',
'data':
narrativeObject,
'name':
narrativeName,
'meta':
metadataExternal,
'provenance': [{
'script':
'NarrativeManager.py',
'description':
'Created new ' + 'Workspace/Narrative bundle.'
}],
'hidden':
0
}]
})[0]
objectInfo = ServiceUtils.objectInfoToObject(objectInfo)
self._completeNewNarrative(newWorkspaceInfo['id'], objectInfo['id'],
importData)
return {'workspaceInfo': newWorkspaceInfo, 'narrativeInfo': objectInfo}
def _fetchNarrativeObjects(self, workspaceName, cells, parameters,
includeIntroCell):
if not cells:
cells = []
# fetchSpecs
appSpecIds = []
methodSpecIds = []
specMapping = {'apps': {}, 'methods': {}}
for cell in cells:
if 'app' in cell:
appSpecIds.append(cell['app'])
elif 'method' in cell:
methodSpecIds.append(cell['method'])
nms = NarrativeMethodStore(self.narrativeMethodStoreURL,
token=self.token)
if len(appSpecIds) > 0:
appSpecs = nms.get_app_spec({'ids': appSpecIds})
for spec in appSpecs:
spec_id = spec['info']['id']
specMapping['apps'][spec_id] = spec
if len(methodSpecIds) > 0:
methodSpecs = nms.get_method_spec({'ids': methodSpecIds})
for spec in methodSpecs:
spec_id = spec['info']['id']
specMapping['methods'][spec_id] = spec
# end of fetchSpecs
metadata = {
'job_ids': {
'methods': [],
'apps': [],
'job_usage': {
'queue_time': 0,
'run_time': 0
}
},
'format': 'ipynb',
'creator': self.user_id,
'ws_name': workspaceName,
'name': 'Untitled',
'type': 'KBaseNarrative.Narrative',
'description': '',
'data_dependencies': []
}
cellData = self._gatherCellData(cells, specMapping, parameters,
includeIntroCell)
narrativeObject = {
'nbformat_minor': 0,
'cells': cellData,
'metadata': metadata,
'nbformat': 4
}
metadataExternal = {}
for key in metadata:
value = metadata[key]
if isinstance(value, basestring):
metadataExternal[key] = value
else:
metadataExternal[key] = json.dumps(value)
return [narrativeObject, metadataExternal]
def _gatherCellData(self, cells, specMapping, parameters,
includeIntroCell):
cell_data = []
if includeIntroCell == 1:
cell_data.append({
'cell_type': 'markdown',
'source': self._get_intro_markdown(),
'metadata': {}
})
for cell_pos, cell in enumerate(cells):
if 'app' in cell:
cell_data.append(
self._buildAppCell(len(cell_data),
specMapping['apps'][cell['app']],
parameters))
elif 'method' in cell:
cell_data.append(
self._buildMethodCell(
len(cell_data), specMapping['methods'][cell['method']],
parameters))
elif 'markdown' in cell:
cell_data.append({
'cell_type': 'markdown',
'source': cell['markdown'],
'metadata': {}
})
else:
raise ValueError("cannot add cell #" + str(cell_pos) +
", unrecognized cell content")
return cell_data
def _buildAppCell(self, pos, spec, params):
cellId = 'kb-cell-' + str(pos) + '-' + str(uuid.uuid4())
cell = {
'cell_type':
'markdown',
'source':
"<div id='" + cellId + "'></div>" + "\n<script>" + "$('#" +
cellId + "').kbaseNarrativeAppCell({'appSpec' : '" +
self._safeJSONStringify(spec) + "', 'cellId' : '" + cellId +
"'});" + "</script>",
'metadata': {}
}
cellInfo = {}
widgetState = []
cellInfo[self.KB_TYPE] = self.KB_APP_CELL
cellInfo['app'] = spec
if params:
steps = {}
for param in params:
stepid = 'step_' + str(param[0])
if stepid not in steps:
steps[stepid] = {}
steps[stepid]['inputState'] = {}
steps[stepid]['inputState'][param[1]] = param[2]
state = {
'state': {
'step': steps
}
}
widgetState.append(state)
cellInfo[self.KB_STATE] = widgetState
cell['metadata'][self.KB_CELL] = cellInfo
return cell
def _buildMethodCell(self, pos, spec, params):
cellId = 'kb-cell-' + str(pos) + '-' + str(uuid.uuid4())
cell = {
'cell_type':
'markdown',
'source':
"<div id='" + cellId + "'></div>" + "\n<script>" + "$('#" +
cellId + "').kbaseNarrativeMethodCell({'method' : '" +
self._safeJSONStringify(spec) + "'});" + "</script>",
'metadata': {}
}
cellInfo = {'method': spec, 'widget': spec['widgets']['input']}
cellInfo[self.KB_TYPE] = self.KB_FUNCTION_CELL
widgetState = []
if params:
wparams = {}
for param in params:
wparams[param[1]] = param[2]
widgetState.append({'state': wparams})
cellInfo[self.KB_STATE] = widgetState
cell['metadata'][self.KB_CELL] = cellInfo
return cell
def _completeNewNarrative(self, workspaceId, objectId, importData):
self.ws.alter_workspace_metadata({
'wsi': {
'id': workspaceId
},
'new': {
'narrative': str(objectId),
'is_temporary': 'true'
}
})
# copy_to_narrative:
if not importData:
return
objectsToCopy = [{'ref': x} for x in importData]
infoList = self.ws.get_object_info_new({
'objects': objectsToCopy,
'includeMetadata': 0
})
for item in infoList:
objectInfo = ServiceUtils.objectInfoToObject(item)
self.copy_object(objectInfo['ref'], workspaceId, None, None,
objectInfo)
def _safeJSONStringify(self, obj):
return json.dumps(self._safeJSONStringifyPrepare(obj))
def _safeJSONStringifyPrepare(self, obj):
if isinstance(obj, basestring):
return obj.replace("'", "'").replace('"', """)
elif isinstance(obj, list):
for pos in range(len(obj)):
obj[pos] = self._safeJSONStringifyPrepare(obj[pos])
elif isinstance(obj, dict):
obj_keys = list(obj.keys())
for key in obj_keys:
obj[key] = self._safeJSONStringifyPrepare(obj[key])
else:
pass # it's boolean/int/float/None
return obj
def _get_workspace_name_or_id(self, ws_id, ws_name):
ret = ws_name
if not ret:
ret = str(ws_id)
return ret
def copy_object(self, ref, target_ws_id, target_ws_name, target_name,
src_info):
# There should be some logic related to DataPalettes
if (not target_ws_id) and (not target_ws_name):
raise ValueError("Neither target workspace ID nor name is defined")
if not src_info:
src_info_tuple = self.ws.get_object_info_new({
'objects': [{
'ref': ref
}],
'includeMetadata':
0
})[0]
src_info = ServiceUtils.objectInfoToObject(src_info_tuple)
type_name = src_info['typeModule'] + '.' + src_info['typeName']
type_config = self.DATA_PALETTES_TYPES.get(type_name)
if type_config is not None:
# Copy with DataPaletteService
if target_name:
raise ValueError(
"'target_name' cannot be defined for DataPalette copy")
target_ws_name_or_id = self._get_workspace_name_or_id(
target_ws_id, target_ws_name)
self.dps_cache.call_method("add_to_palette",
[{
'workspace': target_ws_name_or_id,
'new_refs': [{
'ref': ref
}]
}], self.token)
return {'info': src_info}
else:
if not target_name:
target_name = src_info['name']
obj_info_tuple = self.ws.copy_object({
'from': {
'ref': ref
},
'to': {
'wsid': target_ws_id,
'workspace': target_ws_name,
'name': target_name
}
})
obj_info = ServiceUtils.objectInfoToObject(obj_info_tuple)
return {'info': obj_info}
def list_available_types(self, workspaces):
data = self.list_objects_with_sets(workspaces=workspaces)['data']
type_stat = {}
for item in data:
info = item['object_info']
obj_type = info[2].split('-')[0]
if obj_type in type_stat:
type_stat[obj_type] += 1
else:
type_stat[obj_type] = 1
return {'type_stat': type_stat}
from Workspace.WorkspaceClient import Workspace
import json
wsid = 16962
upa = '16962/3'
upa2 = '16962/23'
ws = Workspace('https://ci.kbase.us/services/ws')
d = ws.get_workspace_info({'id': wsid})
with open('get_workspace_info.json', 'w') as f:
f.write(json.dumps(d))
d = ws.list_objects({'ids': [wsid]})
with open('list_objects.json', 'w') as f:
f.write(json.dumps(d))
d = ws.get_objects2({'objects': [{'ref': upa}]})
with open('get_objects.json', 'w') as f:
f.write(json.dumps(d))
d = ws.get_objects2({'objects': [{'ref': upa2}]})
with open('narrative_object.json', 'w') as f:
f.write(json.dumps(d))
class DataPalette():
PROVENANCE = [{'service': 'DataPaletteService'}]
DATA_PALETTE_WS_METADATA_KEY = 'data_palette_id'
DEFAULT_PALETTE_OBJ_NAME = 'data_palette'
PALETTE_OBJ_WS_TYPE = 'DataPalette.DataPalette'
# set of types that cannot be added to a data palette, add to configuration
PROHIBITED_DATA_TYPES = ['KBaseReport.Report',
'KBaseNarrative.Narrative',
'DataPalette.DataPalette']
def __init__(self, ws_name_or_id, ws_url=None, token=None, ws_info=None, ws=None):
if ws:
self.ws = ws
else:
if ws_url is None:
raise ValueError('ws_url was not defined')
if token is None:
print('DataPalette warning: token was not set')
self.ws = Workspace(ws_url, token=token)
if ws_info:
if ws_name_or_id:
raise ValueError("Either ws_name_or_id or ws_info should be set")
self.ws_info = WorkspaceInfo(ws_info)
else:
if str(ws_name_or_id).isdigit():
self.ws_info = WorkspaceInfo(self.ws.get_workspace_info({'id': int(ws_name_or_id)}))
else:
self.ws_info = WorkspaceInfo(self.ws.get_workspace_info({
'workspace': str(ws_name_or_id)
}))
self.palette_ref = None
def list(self, options):
# if there is no data palette, return nothing
dp_ref = self._get_root_data_palette_ref()
if dp_ref is None:
return []
palette = self._get_data_palette()
include_metadata = options.get('include_metadata', 0)
palette = self._attach_palette_data_info(palette, include_metadata)
return palette['data']
def add(self, refs=None):
'''
Adds the provided references to the data palette.
'''
if len(refs) == 0:
return {}
# make sure the references to add are visible and valid
objs = self._get_object_info(refs)
self._validate_objects_to_add(objs)
# get the existing palette and build an index
palette = self._get_data_palette()
data_index = self._build_palette_data_index(palette['data'])
# changing refs in DataPalette so that they are pointed through
# DataPalette object ref as ref-path
self._extend_ref_paths_before_saving(palette)
# perform the actual update palette update
for obj_pos in range(0, len(objs)):
o = objs[obj_pos]
ws = str(o[6])
obj = str(o[0])
ver = str(o[4])
ref = refs[obj_pos]['ref'] #ws + '/' + obj + '/' + ver
if ws + '/' + obj in data_index:
# the object is in the palette, so check versions
index = data_index[ws + '/' + obj]
if index['ver'] == ver:
# the version didn't change, so continue
continue
# the version is different, so update it
data_index[ws + '/' + obj]['ver'] = ver
palette['data'][index['idx']]['ref'] = ref
else:
# the object wasn't in the palette, so add it
idx = len(palette['data'])
palette['data'].append({'ref': ref})
data_index[ws + '/' + obj] = {'ver': ver, 'idx': idx}
# save the updated palette and return
self._save_data_palette(palette)
return {}
def remove(self, refs=None):
dp_ref = self._get_root_data_palette_ref()
if dp_ref is None:
raise ValueError('Cannot remove from data_palette- data palette ' +
'for Workspace does not exist')
if len(refs) == 0:
return {}
# right now, we only match on exact refs, so this works
palette = self._get_data_palette()
data_index = self._build_palette_data_index(palette['data'])
index_to_delete = []
for r in range(0, len(refs)):
ref = refs[r]['ref']
tokens = ref.split('/')
if len(tokens) != 3:
raise ValueError('Invalid absolute reference: ' + str(ref) + ' at position ' + str(r) +
' of removal list. References must be full, absolute numerical WS refs.')
is_digits = map(lambda x: x.isdigit(), tokens)
if False in is_digits:
raise ValueError('Invalid absolute reference: ' + str(ref) + ' at position ' + str(r) +
' of removal list. References must be full, absolute numerical WS refs.')
ws_slash_id = tokens[0] + '/' + tokens[1]
if ws_slash_id in data_index:
if data_index[ws_slash_id]['ver'] == tokens[2]:
index_to_delete.append(data_index[ws_slash_id]['idx'])
else:
raise ValueError('Reference: ' + str(ref) + ' at position ' + str(r) +
' of removal list was not found in palette. Object exists, but ' +
'version was not correct.')
else:
raise ValueError('Reference: ' + str(ref) + ' at position ' + str(r) +
' of removal list was not found in palette.')
index_to_delete = set(index_to_delete)
for i in sorted(index_to_delete, reverse=True):
del palette['data'][i]
self._extend_ref_paths_before_saving(palette)
self._save_data_palette(palette)
return {}
def _build_palette_data_index(self, palette_data):
data_index = {}
for k in range(0, len(palette_data)):
tokens = palette_data[k]['ref'].split('/')
key = tokens[0] + '/' + tokens[1]
value = {'ver': tokens[2], 'idx': k}
data_index[key] = value
return data_index
def _get_object_info(self, objects):
return self.ws.get_object_info_new({'objects': objects})
def _validate_objects_to_add(self, object_info_list):
for info in object_info_list:
# validate type, split and ignore the type version
full_type_name = info[2].split('-')[0]
if full_type_name in self.PROHIBITED_DATA_TYPES:
raise ValueError('Object ' + str(info[1]) + ' (id=' + str(info[6]) + '/' +
str(info[0]) + '/' + str(info[4]) + ') is a type (' + full_type_name +
') that cannot be added to a data palette.')
def _attach_palette_data_info(self, palette, include_metadata = 0):
# TODO: make sure we get object info via reference chain
if len(palette['data']) == 0:
return palette
palette_ref = self._get_root_data_palette_ref()
info_input = [{'ref': palette_ref + ';' + obj['ref']} for obj in palette['data']]
all_info = self.ws.get_object_info_new({
'objects': info_input,
'includeMetadata': include_metadata
})
dp_ref = self._get_root_data_palette_ref()
for k in range(0, len(all_info)):
palette['data'][k]['info'] = all_info[k]
palette['data'][k]['dp_ref'] = dp_ref
palette['data'][k]['dp_refs'] = [dp_ref]
return palette
def _extend_ref_paths_before_saving(self, palette):
dp_ref = self._get_root_data_palette_ref()
for data_ref in palette['data']:
data_ref['ref'] = dp_ref + ';' + data_ref['ref']
def _save_data_palette(self, palette):
obj_info = self.ws.save_objects({
'id': self.ws_info.id,
'objects': [{
'type': self.PALETTE_OBJ_WS_TYPE,
'objid': self._get_root_data_palette_objid(),
'data': palette,
'provenance': self.PROVENANCE,
'hidden': 1
}]
})[0]
return obj_info
def _get_data_palette(self):
palette_ref = self._get_root_data_palette_ref()
if palette_ref is None:
return self._create_data_palette()
data = self.ws.get_objects2({
'objects': [{'ref': palette_ref}]
})
return data['data'][0]['data']
def _create_data_palette(self):
# 1) save the data_palette object
palette = {'data': []}
new_palette_info = self.ws.save_objects({
'id': self.ws_info.id,
'objects': [{
'type': self.PALETTE_OBJ_WS_TYPE,
'name': self.DEFAULT_PALETTE_OBJ_NAME,
'data': palette,
'provenance': self.PROVENANCE,
'hidden': 1
}]
})[0]
# 2) update ws metadata
self._update_ws_palette_metadata(new_palette_info)
return palette
def create_from_existing_palette(self, existing_data_palette):
# 1) make sure we can actually do it
dp_target_ref = self._get_root_data_palette_ref()
if dp_target_ref is not None:
raise ValueError('Cannot copy data_palette- a data palette already exists in that workspace.')
dp_source_ref = existing_data_palette._get_root_data_palette_ref()
if dp_source_ref is None:
# data palette did not exist, so we don't have to copy over anything
return {}
# 2) make the copy
new_palette_info = self.ws.copy_object({
'from': {'ref': dp_source_ref},
'to': {'wsid': self.ws_info.id, 'name': self.DEFAULT_PALETTE_OBJ_NAME}
})
# 3) update ws metadata
self._update_ws_palette_metadata(new_palette_info)
return {}
def set_palette_to_obj(self, new_data_palette_name_or_id):
if new_data_palette_name_or_id is None:
new_data_palette_name_or_id = self.DEFAULT_PALETTE_OBJ_NAME
new_palette_ref = str(self.ws_info.id) + '/' + str(new_data_palette_name_or_id)
new_palette_info = self._get_object_info([{'ref': new_palette_ref}])[0]
if not str(new_palette_info[2]).startswith(self.PALETTE_OBJ_WS_TYPE):
raise ValueError('Cannot set data palette for workspace to non-palette type. Type of (' +
new_palette_ref + ') was: ' + str(new_palette_info[1]))
self._update_ws_palette_metadata(new_palette_info)
return {}
def _get_root_data_palette_objid(self):
ref = self._get_root_data_palette_ref()
if ref is None:
return None
return self._get_root_data_palette_ref().split('/')[1]
def _get_root_data_palette_ref(self):
if self.palette_ref is not None:
return self.palette_ref
if self.DATA_PALETTE_WS_METADATA_KEY not in self.ws_info.metadata:
return None
dp_id = self.ws_info.metadata[self.DATA_PALETTE_WS_METADATA_KEY]
if not str(dp_id).isdigit():
raise ValueError('Warning: WS metadata for ' + str(self.ws_info.id) +
'was corrupted. It is not set to an object ID. It was: ' + str(dp_id))
self.palette_ref = str(self.ws_info.id) + '/' + str(dp_id)
return self.palette_ref
def _update_ws_palette_metadata(self, palette_obj_info):
self.ws.alter_workspace_metadata({
'wsi': {
'id': self.ws_info.id
},
'new': {
self.DATA_PALETTE_WS_METADATA_KEY: str(palette_obj_info[0])
}
})
# refresh local ws info
self.ws_info = WorkspaceInfo(self.ws.get_workspace_info({'id': self.ws_info.id}))
