def convert_items_from_db(items):
"""
Helper function.
(bundle_uuid, subworksheet_uuid, value, type) -> (bundle_info, subworksheet_info, value_obj, type)
"""
# Database only contains the uuid; need to expand to info.
# We need to do to convert the bundle_uuids into bundle_info dicts.
# However, we still make O(1) database calls because we use the
# optimized batch_get_bundles multiget method.
bundle_uuids = set(bundle_uuid for (bundle_uuid, subworksheet_uuid, value,
type) in items
if bundle_uuid is not None)
bundle_dict = get_bundle_infos(bundle_uuids)
# Go through the items and substitute the components
new_items = []
for (bundle_uuid, subworksheet_uuid, value, type) in items:
bundle_info = bundle_dict.get(
bundle_uuid, {'uuid': bundle_uuid}) if bundle_uuid else None
if subworksheet_uuid:
try:
subworksheet_info = local.model.get_worksheet(
subworksheet_uuid, fetch_items=False).to_dict(legacy=True)
except UsageError, e:
# If can't get the subworksheet, it's probably invalid, so just replace it with an error
# type = worksheet_util.TYPE_MARKUP
subworksheet_info = {'uuid': subworksheet_uuid}
# value = 'ERROR: non-existent worksheet %s' % subworksheet_uuid
else:
subworksheet_info = None
value_obj = formatting.string_to_tokens(
value) if type == worksheet_util.TYPE_DIRECTIVE else value
new_items.append((bundle_info, subworksheet_info, value_obj, type))
def interpret_search(query):
"""
Input: specification of a bundle search query.
Output: worksheet items based on the result of issuing the search query.
"""
# Perform search
keywords = rest_util.resolve_owner_in_keywords(query['keywords'])
bundle_uuids = local.model.search_bundle_uuids(request.user.user_id,
keywords)
# Single number, just print it out...
if not isinstance(bundle_uuids, list):
return interpret_items(query['schemas'],
[markup_item(str(bundle_uuids))])
# Set display
items = [directive_item(('display', ) + tuple(query['display']))]
# Show bundles
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
items.append(bundle_item(bundle_infos[bundle_uuid]))
# Finally, interpret the items
return interpret_items(query['schemas'], items)
def _update_bundles():
"""
Bulk update bundles.
"""
bundle_updates = BundleSchema(
strict=True, many=True,
dump_only=BUNDLE_UPDATE_RESTRICTED_FIELDS,
).load(request.json, partial=True).data
# Check permissions
bundle_uuids = [b.pop('uuid') for b in bundle_updates]
check_bundles_have_all_permission(local.model, request.user, bundle_uuids)
bundles = local.model.batch_get_bundles(uuid=bundle_uuids)
# Update bundles
for bundle, update in izip(bundles, bundle_updates):
local.model.update_bundle(bundle, update)
# Get updated bundles
bundles_dict = get_bundle_infos(bundle_uuids)
# Create list of bundles in original order
updated_bundles = [bundles_dict[uuid] for uuid in bundle_uuids]
return BundleSchema(many=True).dump(updated_bundles).data
def _update_bundles():
"""
Bulk update bundles.
"""
bundle_updates = (BundleSchema(
strict=True, many=True,
dump_only=BUNDLE_UPDATE_RESTRICTED_FIELDS).load(request.json,
partial=True).data)
# Check permissions
bundle_uuids = [b.pop('uuid') for b in bundle_updates]
check_bundles_have_all_permission(local.model, request.user, bundle_uuids)
bundles = local.model.batch_get_bundles(uuid=bundle_uuids)
# Update bundles
for bundle, update in zip(bundles, bundle_updates):
local.model.update_bundle(bundle, update)
# Get updated bundles
bundles_dict = get_bundle_infos(bundle_uuids)
# Create list of bundles in original order
# Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted
# right after being updated.
updated_bundles = [
bundles_dict[uuid] for uuid in bundle_uuids if uuid in bundles_dict
]
return BundleSchema(many=True).dump(updated_bundles).data
def resolve_items_into_infos(items):
"""
Helper function.
{'bundle_uuid': '...', 'subworksheet_uuid': '...', 'value': '...', 'type': '...')
-> (bundle_info, subworksheet_info, value_obj, type)
"""
# Database only contains the uuid; need to expand to info.
# We need to do to convert the bundle_uuids into bundle_info dicts.
# However, we still make O(1) database calls because we use the
# optimized batch_get_bundles multiget method.
bundle_uuids = set(
i['bundle_uuid'] for i in items
if i['bundle_uuid'] is not None
)
bundle_dict = rest_util.get_bundle_infos(bundle_uuids)
# Go through the items and substitute the components
new_items = []
for i in items:
bundle_info = bundle_dict.get(i['bundle_uuid'], {'uuid': i['bundle_uuid']}) if i['bundle_uuid'] else None
if i['subworksheet_uuid']:
try:
subworksheet_info = local.model.get_worksheet(i['subworksheet_uuid'], fetch_items=False).to_dict()
except UsageError, e:
# If can't get the subworksheet, it's probably invalid, so just replace it with an error
# type = worksheet_util.TYPE_MARKUP
subworksheet_info = {'uuid': i['subworksheet_uuid']}
# value = 'ERROR: non-existent worksheet %s' % subworksheet_uuid
else:
subworksheet_info = None
value_obj = formatting.string_to_tokens(i['value']) if i['type'] == TYPE_DIRECTIVE else i['value']
new_items.append((bundle_info, subworksheet_info, value_obj, i['type']))
def build_bundles_document(bundle_uuids):
include_set = query_get_json_api_include_set(supported={
'owner', 'group_permissions', 'children', 'host_worksheets'
})
bundles_dict = get_bundle_infos(
bundle_uuids,
get_children='children' in include_set,
get_permissions='group_permissions' in include_set,
get_host_worksheets='host_worksheets' in include_set,
ignore_not_found=False,
)
# Create list of bundles in original order
bundles = [bundles_dict[uuid] for uuid in bundle_uuids]
# Build response document
document = BundleSchema(many=True).dump(bundles).data
# Shim in display metadata used by the front-end application
if query_get_bool('include_display_metadata', default=False):
for bundle, data in zip(bundles, document['data']):
bundle_class = get_bundle_subclass(bundle['bundle_type'])
json_api_meta(
data,
{
'editable_metadata_keys':
worksheet_util.get_editable_metadata_fields(bundle_class),
'metadata_type':
worksheet_util.get_metadata_types(bundle_class),
},
)
if 'owner' in include_set:
owner_ids = set(b['owner_id'] for b in bundles
if b['owner_id'] is not None)
json_api_include(
document,
UserSchema(),
local.model.get_users(user_ids=owner_ids,
limit=len(owner_ids))['results'],
)
if 'group_permissions' in include_set:
for bundle in bundles:
json_api_include(document, BundlePermissionSchema(),
bundle.get('group_permissions', []))
if 'children' in include_set:
for bundle in bundles:
json_api_include(document, BundleSchema(),
bundle.get('children', []))
if 'host_worksheets' in include_set:
for bundle in bundles:
json_api_include(document, WorksheetSchema(),
bundle.get('host_worksheets', []))
return document
def get_bundle_infos(uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
"""
get_children, get_host_worksheets, get_permissions: whether we want to return more detailed information.
Return map from bundle uuid to info.
"""
return get_bundle_infos(
uuids,
get_children=get_children,
get_host_worksheets=get_host_worksheets,
get_permissions=get_permissions)
def build_bundles_document(bundle_uuids):
bundles_dict = get_bundle_infos(
bundle_uuids,
get_children=True,
get_permissions=True,
get_host_worksheets=True,
)
# Create list of bundles in original order
try:
bundles = [bundles_dict[uuid] for uuid in bundle_uuids]
except KeyError as e:
abort(httplib.NOT_FOUND, "Bundle %s not found" % e.args[0])
# Build response document
document = BundleSchema(many=True).dump(bundles).data
# Shim in editable metadata keys
# Used by the front-end application
for bundle, data in izip(bundles, document['data']):
json_api_meta(
data, {
'editable_metadata_keys':
worksheet_util.get_editable_metadata_fields(
get_bundle_subclass(bundle['bundle_type']))
})
# Include users
owner_ids = set(b['owner_id'] for b in bundles)
json_api_include(document, UserSchema(), local.model.get_users(owner_ids))
# Include permissions
for bundle in bundles:
json_api_include(document, BundlePermissionSchema(),
bundle['group_permissions'])
# Include child bundles
children_uuids = set(c['uuid'] for bundle in bundles
for c in bundle['children'])
json_api_include(document, BundleSchema(),
get_bundle_infos(children_uuids).values())
return document
def test_get_bundle_infos_single_worksheet(self):
bundle_uuids = {'0x123', '0x234', '0x345'}
worksheet_uuids = {'0x111', '0x222', '0x333', '0x444'}
model = Mock()
model.batch_get_bundles.side_effect = self._mock_model_call(
bundle_uuids, [self._mock_bundle(uuid) for uuid in bundle_uuids])
bundle_permissions = {
'0x123': GROUP_OBJECT_PERMISSION_ALL,
'0x234': GROUP_OBJECT_PERMISSION_ALL,
'0x345': GROUP_OBJECT_PERMISSION_ALL,
}
model.get_user_bundle_permissions.side_effect = self._mock_model_call(
bundle_uuids, bundle_permissions, 1)
bundle_worksheet_uuids = {
'0x123': ['0x111', '0x444'],
'0x234': ['0x222'],
'0x345': ['0x333'],
}
model.get_host_worksheet_uuids.side_effect = self._mock_model_call(
bundle_uuids, bundle_worksheet_uuids)
worksheet_permissions = {
'0x111': GROUP_OBJECT_PERMISSION_ALL,
'0x222': GROUP_OBJECT_PERMISSION_ALL,
'0x333': GROUP_OBJECT_PERMISSION_NONE,
'0x444': GROUP_OBJECT_PERMISSION_NONE,
}
model.get_user_worksheet_permissions.side_effect = self._mock_model_call(
worksheet_uuids, worksheet_permissions, 1)
worksheets = [Mock(uuid='0x111'), Mock(uuid='0x222')]
worksheets[0].configure_mock(name='ws1')
worksheets[1].configure_mock(name='ws2')
model.batch_get_worksheets.side_effect = self._mock_model_call(
{'0x111', '0x222'}, worksheets)
infos = util.get_bundle_infos(
bundle_uuids,
get_children=False,
get_single_host_worksheet=True,
get_host_worksheets=False,
get_permissions=False,
ignore_not_found=True,
model=model,
)
# Bundles 0x123 and 0x234 had a worksheet with correct permissions. 0x345 had a worksheet that can't be read.
self.assertDictEqual(infos['0x123']['host_worksheet'], {
'uuid': '0x111',
'name': 'ws1'
})
self.assertDictEqual(infos['0x234']['host_worksheet'], {
'uuid': '0x222',
'name': 'ws2'
})
self.assertTrue('host_worksheet' not in infos['0x345'])
def expand_raw_item(raw_item):
"""
Raw items that include searches must be expanded into more raw items.
Input: Raw item.
Output: Array of raw items. If raw item does not need expanding,
this returns an 1-length array that contains original raw item,
otherwise it contains the search result. You do not need to call
resolve_items_into_infos on the returned raw_items.
"""
(bundle_info, subworksheet_info, value_obj, item_type, id,
sort_key) = raw_item
is_search = item_type == TYPE_DIRECTIVE and get_command(
value_obj) == 'search'
is_wsearch = item_type == TYPE_DIRECTIVE and get_command(
value_obj) == 'wsearch'
if is_search or is_wsearch:
command = get_command(value_obj)
keywords = value_obj[1:]
raw_items = []
if is_search:
keywords = rest_util.resolve_owner_in_keywords(keywords)
search_result = local.model.search_bundles(request.user.user_id,
keywords)
if search_result['is_aggregate']:
# Add None's for the 'id' and 'sort_key' of the tuple, since these
# items are not really worksheet items.
raw_items.append(
markup_item(str(search_result['result'])) + (None, None))
else:
bundle_uuids = search_result['result']
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
# Since bundle UUID's are queried first, we can't assume a UUID exists in
# the subsequent bundle info query.
if bundle_uuid in bundle_infos:
raw_items.append(
bundle_item(bundle_infos[bundle_uuid]) +
(None, None))
elif is_wsearch:
worksheet_infos = search_worksheets(keywords)
for worksheet_info in worksheet_infos:
raw_items.append(
subworksheet_item(worksheet_info) + (None, None))
return raw_items
else:
return [raw_item]
def fetch_worksheet(uuid):
worksheet = get_worksheet_info(
uuid,
fetch_items=True,
fetch_permission=True,
)
# Build response document
document = WorksheetSchema().dump(worksheet).data
# Include items
json_api_include(document, WorksheetItemSchema(), worksheet['items'])
# Include bundles
bundle_uuids = {
item['bundle_uuid']
for item in worksheet['items']
if item['type'] == worksheet_util.TYPE_BUNDLE
and item['bundle_uuid'] is not None
}
bundle_infos = get_bundle_infos(bundle_uuids).values()
json_api_include(document, BundleSchema(), bundle_infos)
# Include users
user_ids = {b['owner_id'] for b in bundle_infos}
user_ids.add(worksheet['owner_id'])
if user_ids:
json_api_include(document, UserSchema(),
local.model.get_users(user_ids))
# Include subworksheets
subworksheet_uuids = {
item['subworksheet_uuid']
for item in worksheet['items']
if item['type'] == worksheet_util.TYPE_WORKSHEET
and item['subworksheet_uuid'] is not None
}
json_api_include(
document, WorksheetSchema(),
local.model.batch_get_worksheets(fetch_items=False,
uuid=subworksheet_uuids))
# FIXME: tokenizing directive args
# value_obj = formatting.string_to_tokens(value) if type == worksheet_util.TYPE_DIRECTIVE else value
# Include permissions
json_api_include(document, WorksheetPermissionSchema(),
worksheet['group_permissions'])
return document
def expand_raw_item(raw_item):
"""
Raw items that include searches must be expanded into more raw items.
Input: Raw item.
Output: Array of raw items. If raw item does not need expanding,
this returns an 1-length array that contains original raw item,
otherwise it contains the search result. You do not need to call
resolve_items_into_infos on the returned raw_items.
"""
(bundle_info, subworksheet_info, value_obj, item_type) = raw_item
is_search = item_type == TYPE_DIRECTIVE and get_command(
value_obj) == 'search'
is_wsearch = item_type == TYPE_DIRECTIVE and get_command(
value_obj) == 'wsearch'
if is_search or is_wsearch:
command = get_command(value_obj)
keywords = value_obj[1:]
raw_items = []
if is_search:
keywords = rest_util.resolve_owner_in_keywords(keywords)
search_result = local.model.search_bundle_uuids(
request.user.user_id, keywords)
if not isinstance(search_result, list):
raw_items.append(markup_item(str(search_result)))
else:
bundle_uuids = search_result
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
raw_items.append(bundle_item(bundle_infos[bundle_uuid]))
elif is_wsearch:
worksheet_infos = search_worksheets(keywords)
for worksheet_info in worksheet_infos:
raw_items.append(subworksheet_item(worksheet_info))
return raw_items
else:
return [raw_item]
def perform_search_query(value_obj):
"""
Perform a search query and return the resulting raw items.
Input: directive that is tokenized by formatting.string_to_tokens(),
such as formatting.string_to_tokens("search 0x .limit=100")
"""
command = get_command(value_obj)
is_search = command == 'search'
is_wsearch = command == 'wsearch'
if is_search or is_wsearch:
keywords = value_obj[1:]
raw_items = []
if is_search:
keywords = rest_util.resolve_owner_in_keywords(keywords)
search_result = local.model.search_bundles(request.user.user_id,
keywords)
if search_result['is_aggregate']:
# Add None's for the 'id' and 'sort_key' of the tuple, since these
# items are not really worksheet items.
raw_items.append(
markup_item(str(search_result['result'])) + (None, None))
else:
bundle_uuids = search_result['result']
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
# Since bundle UUID's are queried first, we can't assume a UUID exists in
# the subsequent bundle info query.
if bundle_uuid in bundle_infos:
raw_items.append(
bundle_item(bundle_infos[bundle_uuid]) +
(None, None))
elif is_wsearch:
worksheet_infos = search_worksheets(keywords)
for worksheet_info in worksheet_infos:
raw_items.append(
subworksheet_item(worksheet_info) + (None, None))
return raw_items
else:
# Not a search query
return []
def resolve_items_into_infos(items):
"""
Helper function.
{'bundle_uuid': '...', 'subworksheet_uuid': '...', 'value': '...', 'type': '...')
-> (bundle_info, subworksheet_info, value_obj, type)
"""
# Database only contains the uuid; need to expand to info.
# We need to do to convert the bundle_uuids into bundle_info dicts.
# However, we still make O(1) database calls because we use the
# optimized batch_get_bundles multiget method.
bundle_uuids = set(i['bundle_uuid'] for i in items if i['bundle_uuid'] is not None)
bundle_dict = rest_util.get_bundle_infos(bundle_uuids)
# Go through the items and substitute the components
new_items = []
for i in items:
bundle_info = (
bundle_dict.get(i['bundle_uuid'], {'uuid': i['bundle_uuid']})
if i['bundle_uuid']
else None
)
if i['subworksheet_uuid']:
try:
subworksheet_info = local.model.get_worksheet(
i['subworksheet_uuid'], fetch_items=False
).to_dict()
except UsageError as e:
# If can't get the subworksheet, it's probably invalid, so just replace it with an error
# type = worksheet_util.TYPE_MARKUP
subworksheet_info = {'uuid': i['subworksheet_uuid']}
# value = 'ERROR: non-existent worksheet %s' % subworksheet_uuid
else:
subworksheet_info = None
value_obj = (
formatting.string_to_tokens(i['value']) if i['type'] == TYPE_DIRECTIVE else i['value']
)
new_items.append((bundle_info, subworksheet_info, value_obj, i['type']))
return new_items
def expand_raw_item(raw_item):
"""
Raw items that include searches must be expanded into more raw items.
Input: Raw item.
Output: Array of raw items. If raw item does not need expanding,
this returns an 1-length array that contains original raw item,
otherwise it contains the search result. You do not need to call
resolve_items_into_infos on the returned raw_items.
"""
(bundle_info, subworksheet_info, value_obj, item_type) = raw_item
is_search = item_type == TYPE_DIRECTIVE and get_command(value_obj) == 'search'
is_wsearch = item_type == TYPE_DIRECTIVE and get_command(value_obj) == 'wsearch'
if is_search or is_wsearch:
command = get_command(value_obj)
keywords = value_obj[1:]
raw_items = []
if is_search:
keywords = rest_util.resolve_owner_in_keywords(keywords)
search_result = local.model.search_bundles(request.user.user_id, keywords)
if search_result['is_aggregate']:
raw_items.append(markup_item(str(search_result['result'])))
else:
bundle_uuids = search_result['result']
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
raw_items.append(bundle_item(bundle_infos[bundle_uuid]))
elif is_wsearch:
worksheet_infos = search_worksheets(keywords)
for worksheet_info in worksheet_infos:
raw_items.append(subworksheet_item(worksheet_info))
return raw_items
else:
return [raw_item]
def _update_bundles():
"""
Bulk update bundles.
"""
bundle_updates = (BundleSchema(
strict=True, many=True,
dump_only=BUNDLE_UPDATE_RESTRICTED_FIELDS).load(request.json,
partial=True).data)
# Check permissions
bundle_uuids = [b.pop('uuid') for b in bundle_updates]
check_bundles_have_all_permission(local.model, request.user, bundle_uuids)
bundles = local.model.batch_get_bundles(uuid=bundle_uuids)
for bundle, update in zip(bundles, bundle_updates):
if "frozen" not in update:
bundle_util.check_bundle_not_frozen(bundle)
else:
# If we're freezing or unfreezing the bundle, check that
# the bundle is in a final state.
# If we're freezing, additionally check that the bundle is not already frozen.
bundle_util.check_bundle_freezable(bundle)
if update["frozen"]:
bundle_util.check_bundle_not_frozen(bundle)
# Update bundles
for bundle, update in zip(bundles, bundle_updates):
local.model.update_bundle(bundle, update)
# Get updated bundles
bundles_dict = get_bundle_infos(bundle_uuids)
# Create list of bundles in original order
# Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted
# right after being updated.
updated_bundles = [
bundles_dict[uuid] for uuid in bundle_uuids if uuid in bundles_dict
]
return BundleSchema(many=True).dump(updated_bundles).data
def fetch_worksheet(uuid):
"""
Fetch a single worksheet by UUID.
Query parameters:
- `include`: comma-separated list of related resources to include, such as "owner"
"""
include_set = query_get_json_api_include_set(
supported={
'owner',
'group_permissions',
'items',
'items.bundle',
'items.bundle.owner',
'items.subworksheet',
}
)
worksheet = get_worksheet_info(
uuid,
fetch_items='items' in include_set,
fetch_permissions='group_permissions' in include_set,
)
# Build response document
document = WorksheetSchema().dump(worksheet).data
# Include items
if 'items' in include_set:
json_api_include(document, WorksheetItemSchema(), worksheet['items'])
user_ids = set()
# Include bundles
if 'items.bundle' in include_set:
bundle_uuids = {
item['bundle_uuid']
for item in worksheet['items']
if item['type'] == worksheet_util.TYPE_BUNDLE and item['bundle_uuid'] is not None
}
bundle_infos = get_bundle_infos(bundle_uuids).values()
json_api_include(document, BundleSchema(), bundle_infos)
if 'items.bundle.owner' in include_set:
user_ids.update({b['owner_id'] for b in bundle_infos})
# Include users
if 'owner' in include_set:
user_ids.add(worksheet['owner_id'])
if user_ids:
json_api_include(document, UserSchema(), local.model.get_users(user_ids))
# Include subworksheets
if 'items.subworksheets' in include_set:
subworksheet_uuids = {
item['subworksheet_uuid']
for item in worksheet['items']
if item['type'] == worksheet_util.TYPE_WORKSHEET
and item['subworksheet_uuid'] is not None
}
json_api_include(
document,
WorksheetSchema(),
local.model.batch_get_worksheets(fetch_items=False, uuid=subworksheet_uuids),
)
# Include permissions
if 'group_permissions' in include_set:
json_api_include(document, WorksheetPermissionSchema(), worksheet['group_permissions'])
return document
def _create_bundles():
"""
Bulk create bundles.
Query parameters:
- `worksheet`: UUID of the parent worksheet of the new bundle, add to
this worksheet if not detached or shadowing another bundle. The new
bundle also inherits permissions from this worksheet.
- `shadow`: UUID of the bundle to "shadow" (the new bundle will be added
as an item immediately after this bundle in its parent worksheet).
- `detached`: 1 if should not add new bundle to any worksheet,
or 0 otherwise. Default is 0.
- `wait_for_upload`: 1 if the bundle state should be initialized to
"uploading" regardless of the bundle type, or 0 otherwise. Used when
copying bundles from another CodaLab instance, this prevents these new
bundles from being executed by the BundleManager. Default is 0.
"""
worksheet_uuid = request.query.get('worksheet')
shadow_parent_uuid = request.query.get('shadow')
after_sort_key = request.query.get('after_sort_key')
detached = query_get_bool('detached', default=False)
if worksheet_uuid is None:
abort(
http.client.BAD_REQUEST,
"Parent worksheet id must be specified as"
"'worksheet' query parameter",
)
# Deserialize bundle fields
bundles = (BundleSchema(strict=True,
many=True,
dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS).load(
request.json).data)
# Check for all necessary permissions
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
worksheet_util.check_worksheet_not_frozen(worksheet)
request.user.check_quota(need_time=True, need_disk=True)
created_uuids = []
for bundle in bundles:
# Prep bundle info for saving into database
# Unfortunately cannot use the `construct` methods because they don't
# provide a uniform interface for constructing bundles for all types
# Hopefully this can all be unified after REST migration is complete
bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
created_uuids.append(bundle_uuid)
bundle_class = get_bundle_subclass(bundle['bundle_type'])
bundle['owner_id'] = request.user.user_id
metadata = bundle.get("metadata", {})
if metadata.get("link_url"):
bundle['state'] = State.READY
elif issubclass(bundle_class, UploadedBundle) or query_get_bool(
'wait_for_upload', False):
bundle['state'] = State.UPLOADING
else:
bundle['state'] = State.CREATED
bundle[
'is_anonymous'] = worksheet.is_anonymous # inherit worksheet anonymity
bundle.setdefault('metadata', {})['created'] = int(time.time())
for dep in bundle.setdefault('dependencies', []):
dep['child_uuid'] = bundle_uuid
# Create bundle object
bundle = bundle_class(bundle, strict=False)
# Save bundle into model
local.model.save_bundle(bundle)
# Inherit worksheet permissions
group_permissions = local.model.get_group_worksheet_permissions(
request.user.user_id, worksheet_uuid)
set_bundle_permissions([{
'object_uuid': bundle_uuid,
'group_uuid': p['group_uuid'],
'permission': p['permission'],
} for p in group_permissions])
# Add as item to worksheet
if not detached:
if shadow_parent_uuid is None:
local.model.add_worksheet_items(
worksheet_uuid, [worksheet_util.bundle_item(bundle_uuid)],
after_sort_key)
else:
local.model.add_shadow_worksheet_items(shadow_parent_uuid,
bundle_uuid)
# Get created bundles
bundles_dict = get_bundle_infos(created_uuids)
# Return bundles in original order
# Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted
# right after being created.
bundles = [
bundles_dict[uuid] for uuid in created_uuids if uuid in bundles_dict
]
return BundleSchema(many=True).dump(bundles).data
def _create_bundles():
"""
Bulk create bundles.
|worksheet_uuid| - The parent worksheet of the bundle, add to this worksheet
if not detached or shadowing another bundle. Also used
to inherit permissions.
|shadow| - the uuid of the bundle to shadow
|detached| - True ('1') if should not add new bundle to any worksheet,
or False ('0') otherwise. Default is False.
|wait_for_upload| - True ('1') if the bundle state should be initialized to
UPLOADING regardless of the bundle type, or False ('0')
otherwise. This prevents run bundles that are being
copied from another instance from being run by the
BundleManager. Default is False.
"""
worksheet_uuid = request.query.get('worksheet')
shadow_parent_uuid = request.query.get('shadow')
detached = query_get_bool('detached', default=False)
if worksheet_uuid is None:
abort(
httplib.BAD_REQUEST, "Parent worksheet id must be specified as"
"'worksheet' query parameter")
# Deserialize bundle fields
bundles = BundleSchema(
strict=True,
many=True,
dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS,
).load(request.json).data
# Check for all necessary permissions
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
worksheet_util.check_worksheet_not_frozen(worksheet)
request.user.check_quota(need_time=True, need_disk=True)
created_uuids = []
for bundle in bundles:
# Prep bundle info for saving into database
# Unfortunately cannot use the `construct` methods because they don't
# provide a uniform interface for constructing bundles for all types
# Hopefully this can all be unified after REST migration is complete
bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
created_uuids.append(bundle_uuid)
bundle_class = get_bundle_subclass(bundle['bundle_type'])
bundle['owner_id'] = request.user.user_id
bundle['state'] = (
State.UPLOADING if issubclass(bundle_class, UploadedBundle)
or query_get_bool('wait_for_upload', False) else State.CREATED)
bundle.setdefault('metadata', {})['created'] = int(time.time())
for dep in bundle.setdefault('dependencies', []):
dep['child_uuid'] = bundle_uuid
# Create bundle object
bundle = bundle_class(bundle, strict=False)
# Save bundle into model
local.model.save_bundle(bundle)
# Inherit worksheet permissions
group_permissions = local.model.get_group_worksheet_permissions(
request.user.user_id, worksheet_uuid)
set_bundle_permissions([{
'object_uuid': bundle_uuid,
'group_uuid': p['group_uuid'],
'permission': p['permission'],
} for p in group_permissions])
# Add as item to worksheet
if not detached:
if shadow_parent_uuid is None:
local.model.add_worksheet_item(
worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
else:
local.model.add_shadow_worksheet_items(shadow_parent_uuid,
bundle_uuid)
# Get created bundles
bundles_dict = get_bundle_infos(created_uuids)
# Return bundles in original order
bundles = [bundles_dict[uuid] for uuid in created_uuids]
return BundleSchema(many=True).dump(bundles).data
def fetch_worksheet(uuid):
"""
Fetch a single worksheet by UUID.
Query parameters:
- `include`: comma-separated list of related resources to include, such as "owner"
"""
include_set = query_get_json_api_include_set(
supported={
'owner',
'group_permissions',
'items',
'items.bundle',
'items.bundle.owner',
'items.subworksheet',
})
worksheet = get_worksheet_info(
uuid,
fetch_items='items' in include_set,
fetch_permissions='group_permissions' in include_set,
)
# Build response document
document = WorksheetSchema().dump(worksheet).data
# Include items
if 'items' in include_set:
json_api_include(document, WorksheetItemSchema(), worksheet['items'])
user_ids = set()
# Include bundles
if 'items.bundle' in include_set:
bundle_uuids = {
item['bundle_uuid']
for item in worksheet['items']
if item['type'] == worksheet_util.TYPE_BUNDLE
and item['bundle_uuid'] is not None
}
bundle_infos = list(get_bundle_infos(bundle_uuids).values())
json_api_include(document, BundleSchema(), bundle_infos)
if 'items.bundle.owner' in include_set:
user_ids.update({b['owner_id'] for b in bundle_infos})
# Include users
if 'owner' in include_set:
user_ids.add(worksheet['owner_id'])
if user_ids:
json_api_include(document, UserSchema(),
local.model.get_users(user_ids=user_ids)['results'])
# Include subworksheets
if 'items.subworksheets' in include_set:
subworksheet_uuids = {
item['subworksheet_uuid']
for item in worksheet['items']
if item['type'] == worksheet_util.TYPE_WORKSHEET
and item['subworksheet_uuid'] is not None
}
json_api_include(
document,
WorksheetSchema(),
local.model.batch_get_worksheets(fetch_items=False,
uuid=subworksheet_uuids),
)
# Include permissions
if 'group_permissions' in include_set:
json_api_include(document, WorksheetPermissionSchema(),
worksheet['group_permissions'])
return document