def test_workspace_list_objects_iterator(self):
ws_name = self.__class__.example_ws_name
part_size = 10
ws_info = self.getWsClient().get_workspace_info({'workspace': ws_name})
max_obj_count = ws_info[4]
min_obj_id = 1
obj_count = 0
while min_obj_id <= max_obj_count:
max_obj_id = min_obj_id + 10000 - 1
part = self.getWsClient().list_objects({'workspaces': [ws_name],
'minObjectID': min_obj_id,
'maxObjectID': max_obj_id})
obj_count += len(part)
min_obj_id += 10000
obj_count2 = 0
for info in WorkspaceListObjectsIterator(self.getWsClient(), ws_info_list=[ws_info],
part_size=part_size):
self.assertEqual(11, len(info))
obj_count2 += 1
self.assertEqual(obj_count, obj_count2)
def _list_objects_with_sets(self, workspaces, types, include_metadata):
type_map = None
if types is not None:
type_map = {key: True for key in types}
processed_refs = {}
data = []
if self.DEBUG:
print("NarrativeManager._list_objects_with_sets: processing sets")
t1 = time.time()
set_ret = self.set_api_cache.call_method(
"list_sets", [{
'workspaces': workspaces,
'include_set_item_info': 1,
'include_raw_data_palettes': 1,
'include_metadata': include_metadata
}], self.token)
sets = set_ret['sets']
dp_data = set_ret.get('raw_data_palettes')
dp_refs = set_ret.get('raw_data_palette_refs')
for set_info in sets:
# Process
target_set_items = []
for set_item in set_info['items']:
target_set_items.append(set_item['info'])
if self._check_info_type(set_info['info'], type_map):
data_item = {
'object_info': set_info['info'],
'set_items': {
'set_items_info': target_set_items
}
}
data.append(data_item)
processed_refs[set_info['ref']] = data_item
if self.DEBUG:
print(" (time=" + str(time.time() - t1) + ")")
if self.DEBUG:
print("NarrativeManager._list_objects_with_sets: loading ws_info")
t2 = time.time()
ws_info_list = []
#for ws in workspaces:
if len(workspaces) == 1:
ws = workspaces[0]
ws_id = None
ws_name = None
if str(ws).isdigit():
ws_id = int(ws)
else:
ws_name = str(ws)
ws_info_list.append(
self.ws.get_workspace_info({
"id": ws_id,
"workspace": ws_name
}))
else:
ws_map = {key: True for key in workspaces}
for ws_info in self.ws.list_workspace_info({'perm': 'r'}):
if ws_info[1] in ws_map or str(ws_info[0]) in ws_map:
ws_info_list.append(ws_info)
if self.DEBUG:
print(" (time=" + str(time.time() - t2) + ")")
if self.DEBUG:
print(
"NarrativeManager._list_objects_with_sets: loading workspace objects"
)
t3 = time.time()
for info in WorkspaceListObjectsIterator(
self.ws,
ws_info_list=ws_info_list,
list_objects_params={'includeMetadata': include_metadata}):
item_ref = str(info[6]) + '/' + str(info[0]) + '/' + str(info[4])
if item_ref not in processed_refs and self._check_info_type(
info, type_map):
data_item = {'object_info': info}
data.append(data_item)
processed_refs[item_ref] = data_item
if self.DEBUG:
print(" (time=" + str(time.time() - t3) + ")")
if self.DEBUG:
print(
"NarrativeManager._list_objects_with_sets: processing DataPalettes"
)
t5 = time.time()
if dp_data is None or dp_refs is None:
dps = self.dps_cache
dp_ret = dps.call_method("list_data",
[{
'workspaces': workspaces,
'include_metadata': include_metadata
}], self.token)
dp_data = dp_ret['data']
dp_refs = dp_ret['data_palette_refs']
for item in dp_data:
ref = item['ref']
if self._check_info_type(item['info'], type_map):
data_item = None
if ref in processed_refs:
data_item = processed_refs[ref]
else:
data_item = {'object_info': item['info']}
processed_refs[ref] = data_item
data.append(data_item)
dp_info = {}
if 'dp_ref' in item:
dp_info['ref'] = item['dp_ref']
if 'dp_refs' in item:
dp_info['refs'] = item['dp_refs']
data_item['dp_info'] = dp_info
if self.DEBUG:
print(" (time=" + str(time.time() - t5) + ")")
return {"data": data, 'data_palette_refs': dp_refs}
def copy_narrative(self, newName, workspaceRef, workspaceId):
time_ms = int(round(time.time() * 1000))
newWsName = self.user_id + ':narrative_' + str(time_ms)
# add the 'narrative' field to newWsMeta later.
newWsMeta = {"is_temporary": "false", "narrative_nice_name": newName}
# start with getting the existing narrative object.
currentNarrative = self.ws.get_objects([{'ref': workspaceRef}])[0]
if not workspaceId:
workspaceId = currentNarrative['info'][6]
# Let's prepare exceptions for clone the workspace.
# 1) currentNarrative object:
excluded_list = [{'objid': currentNarrative['info'][0]}]
# 2) let's exclude objects of types under DataPalette handling:
data_palette_type = "DataPalette.DataPalette"
excluded_types = [data_palette_type]
excluded_types.extend(self.DATA_PALETTES_TYPES.keys())
add_to_palette_list = []
dp_detected = False
for obj_type in excluded_types:
list_objects_params = {'type': obj_type}
if obj_type == data_palette_type:
list_objects_params['showHidden'] = 1
for info in WorkspaceListObjectsIterator(
self.ws,
ws_id=workspaceId,
list_objects_params=list_objects_params):
if obj_type == data_palette_type:
dp_detected = True
else:
add_to_palette_list.append({
'ref':
str(info[6]) + '/' + str(info[0]) + '/' + str(info[4])
})
excluded_list.append({'objid': info[0]})
# clone the workspace EXCEPT for currentNarrative object + obejcts of DataPalette types:
newWsId = self.ws.clone_workspace({
'wsi': {
'id': workspaceId
},
'workspace': newWsName,
'meta': newWsMeta,
'exclude': excluded_list
})[0]
try:
if dp_detected:
self.dps_cache.call_method(
"copy_palette", [{
'from_workspace': str(workspaceId),
'to_workspace': str(newWsId)
}], self.token)
if len(add_to_palette_list) > 0:
# There are objects in source workspace that have type under DataPalette handling
# but these objects are physically stored in source workspace rather that saved
# in DataPalette object. So they weren't copied by "dps.copy_palette".
self.dps_cache.call_method("add_to_palette",
[{
'workspace': str(newWsId),
'new_refs': add_to_palette_list
}], self.token)
# update the ref inside the narrative object and the new workspace metadata.
newNarMetadata = currentNarrative['info'][10]
newNarMetadata['name'] = newName
newNarMetadata['ws_name'] = newWsName
newNarMetadata['job_info'] = json.dumps({
'queue_time': 0,
'running': 0,
'completed': 0,
'run_time': 0,
'error': 0
})
currentNarrative['data']['metadata']['name'] = newName
currentNarrative['data']['metadata']['ws_name'] = newWsName
currentNarrative['data']['metadata']['job_ids'] = {
'apps': [],
'methods': [],
'job_usage': {
'queue_time': 0,
'run_time': 0
}
}
# save the shiny new Narrative so it's at version 1
newNarInfo = self.ws.save_objects({
'id':
newWsId,
'objects': [{
'type': currentNarrative['info'][2],
'data': currentNarrative['data'],
'provenance': currentNarrative['provenance'],
'name': currentNarrative['info'][1],
'meta': newNarMetadata
}]
})
# now, just update the workspace metadata to point
# to the new narrative object
newNarId = newNarInfo[0][0]
self.ws.alter_workspace_metadata({
'wsi': {
'id': newWsId
},
'new': {
'narrative': str(newNarId)
}
})
return {'newWsId': newWsId, 'newNarId': newNarId}
except:
# let's delete copy of workspace so it's out of the way - it's broken
self.ws.delete_workspace({'id': newWsId})
raise # continue raising previous exception