def _update_bundles():
"""
Bulk update bundles.
"""
bundle_updates = BundleSchema(
strict=True, many=True,
dump_only=BUNDLE_UPDATE_RESTRICTED_FIELDS,
).load(request.json, partial=True).data
# Check permissions
bundle_uuids = [b.pop('uuid') for b in bundle_updates]
check_bundles_have_all_permission(local.model, request.user, bundle_uuids)
bundles = local.model.batch_get_bundles(uuid=bundle_uuids)
# Update bundles
for bundle, update in izip(bundles, bundle_updates):
local.model.update_bundle(bundle, update)
# Get updated bundles
bundles_dict = get_bundle_infos(bundle_uuids)
# Create list of bundles in original order
updated_bundles = [bundles_dict[uuid] for uuid in bundle_uuids]
return BundleSchema(many=True).dump(updated_bundles).data
def _update_bundles():
"""
Bulk update bundles.
"""
bundle_updates = (BundleSchema(
strict=True, many=True,
dump_only=BUNDLE_UPDATE_RESTRICTED_FIELDS).load(request.json,
partial=True).data)
# Check permissions
bundle_uuids = [b.pop('uuid') for b in bundle_updates]
check_bundles_have_all_permission(local.model, request.user, bundle_uuids)
bundles = local.model.batch_get_bundles(uuid=bundle_uuids)
# Update bundles
for bundle, update in zip(bundles, bundle_updates):
local.model.update_bundle(bundle, update)
# Get updated bundles
bundles_dict = get_bundle_infos(bundle_uuids)
# Create list of bundles in original order
# Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted
# right after being updated.
updated_bundles = [
bundles_dict[uuid] for uuid in bundle_uuids if uuid in bundles_dict
]
return BundleSchema(many=True).dump(updated_bundles).data
def build_bundles_document(bundle_uuids):
include_set = query_get_json_api_include_set(supported={
'owner', 'group_permissions', 'children', 'host_worksheets'
})
bundles_dict = get_bundle_infos(
bundle_uuids,
get_children='children' in include_set,
get_permissions='group_permissions' in include_set,
get_host_worksheets='host_worksheets' in include_set,
ignore_not_found=False,
)
# Create list of bundles in original order
bundles = [bundles_dict[uuid] for uuid in bundle_uuids]
# Build response document
document = BundleSchema(many=True).dump(bundles).data
# Shim in display metadata used by the front-end application
if query_get_bool('include_display_metadata', default=False):
for bundle, data in zip(bundles, document['data']):
bundle_class = get_bundle_subclass(bundle['bundle_type'])
json_api_meta(
data,
{
'editable_metadata_keys':
worksheet_util.get_editable_metadata_fields(bundle_class),
'metadata_type':
worksheet_util.get_metadata_types(bundle_class),
},
)
if 'owner' in include_set:
owner_ids = set(b['owner_id'] for b in bundles
if b['owner_id'] is not None)
json_api_include(
document,
UserSchema(),
local.model.get_users(user_ids=owner_ids,
limit=len(owner_ids))['results'],
)
if 'group_permissions' in include_set:
for bundle in bundles:
json_api_include(document, BundlePermissionSchema(),
bundle.get('group_permissions', []))
if 'children' in include_set:
for bundle in bundles:
json_api_include(document, BundleSchema(),
bundle.get('children', []))
if 'host_worksheets' in include_set:
for bundle in bundles:
json_api_include(document, WorksheetSchema(),
bundle.get('host_worksheets', []))
return document
def build_bundles_document(bundle_uuids):
bundles_dict = get_bundle_infos(
bundle_uuids,
get_children=True,
get_permissions=True,
get_host_worksheets=True,
)
# Create list of bundles in original order
try:
bundles = [bundles_dict[uuid] for uuid in bundle_uuids]
except KeyError as e:
abort(httplib.NOT_FOUND, "Bundle %s not found" % e.args[0])
# Build response document
document = BundleSchema(many=True).dump(bundles).data
# Shim in editable metadata keys
# Used by the front-end application
for bundle, data in izip(bundles, document['data']):
json_api_meta(
data, {
'editable_metadata_keys':
worksheet_util.get_editable_metadata_fields(
get_bundle_subclass(bundle['bundle_type']))
})
# Include users
owner_ids = set(b['owner_id'] for b in bundles)
json_api_include(document, UserSchema(), local.model.get_users(owner_ids))
# Include permissions
for bundle in bundles:
json_api_include(document, BundlePermissionSchema(),
bundle['group_permissions'])
# Include child bundles
children_uuids = set(c['uuid'] for bundle in bundles
for c in bundle['children'])
json_api_include(document, BundleSchema(),
get_bundle_infos(children_uuids).values())
return document
def _update_bundles():
"""
Bulk update bundles.
"""
bundle_updates = (BundleSchema(
strict=True, many=True,
dump_only=BUNDLE_UPDATE_RESTRICTED_FIELDS).load(request.json,
partial=True).data)
# Check permissions
bundle_uuids = [b.pop('uuid') for b in bundle_updates]
check_bundles_have_all_permission(local.model, request.user, bundle_uuids)
bundles = local.model.batch_get_bundles(uuid=bundle_uuids)
for bundle, update in zip(bundles, bundle_updates):
if "frozen" not in update:
bundle_util.check_bundle_not_frozen(bundle)
else:
# If we're freezing or unfreezing the bundle, check that
# the bundle is in a final state.
# If we're freezing, additionally check that the bundle is not already frozen.
bundle_util.check_bundle_freezable(bundle)
if update["frozen"]:
bundle_util.check_bundle_not_frozen(bundle)
# Update bundles
for bundle, update in zip(bundles, bundle_updates):
local.model.update_bundle(bundle, update)
# Get updated bundles
bundles_dict = get_bundle_infos(bundle_uuids)
# Create list of bundles in original order
# Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted
# right after being updated.
updated_bundles = [
bundles_dict[uuid] for uuid in bundle_uuids if uuid in bundles_dict
]
return BundleSchema(many=True).dump(updated_bundles).data
def fetch_worksheet(uuid):
worksheet = get_worksheet_info(
uuid,
fetch_items=True,
fetch_permission=True,
)
# Build response document
document = WorksheetSchema().dump(worksheet).data
# Include items
json_api_include(document, WorksheetItemSchema(), worksheet['items'])
# Include bundles
bundle_uuids = {
item['bundle_uuid']
for item in worksheet['items']
if item['type'] == worksheet_util.TYPE_BUNDLE
and item['bundle_uuid'] is not None
}
bundle_infos = get_bundle_infos(bundle_uuids).values()
json_api_include(document, BundleSchema(), bundle_infos)
# Include users
user_ids = {b['owner_id'] for b in bundle_infos}
user_ids.add(worksheet['owner_id'])
if user_ids:
json_api_include(document, UserSchema(),
local.model.get_users(user_ids))
# Include subworksheets
subworksheet_uuids = {
item['subworksheet_uuid']
for item in worksheet['items']
if item['type'] == worksheet_util.TYPE_WORKSHEET
and item['subworksheet_uuid'] is not None
}
json_api_include(
document, WorksheetSchema(),
local.model.batch_get_worksheets(fetch_items=False,
uuid=subworksheet_uuids))
# FIXME: tokenizing directive args
# value_obj = formatting.string_to_tokens(value) if type == worksheet_util.TYPE_DIRECTIVE else value
# Include permissions
json_api_include(document, WorksheetPermissionSchema(),
worksheet['group_permissions'])
return document
def _create_bundles():
"""
Bulk create bundles.
Query parameters:
- `worksheet`: UUID of the parent worksheet of the new bundle, add to
this worksheet if not detached or shadowing another bundle. The new
bundle also inherits permissions from this worksheet.
- `shadow`: UUID of the bundle to "shadow" (the new bundle will be added
as an item immediately after this bundle in its parent worksheet).
- `detached`: 1 if should not add new bundle to any worksheet,
or 0 otherwise. Default is 0.
- `wait_for_upload`: 1 if the bundle state should be initialized to
"uploading" regardless of the bundle type, or 0 otherwise. Used when
copying bundles from another CodaLab instance, this prevents these new
bundles from being executed by the BundleManager. Default is 0.
"""
worksheet_uuid = request.query.get('worksheet')
shadow_parent_uuid = request.query.get('shadow')
after_sort_key = request.query.get('after_sort_key')
detached = query_get_bool('detached', default=False)
if worksheet_uuid is None:
abort(
http.client.BAD_REQUEST,
"Parent worksheet id must be specified as"
"'worksheet' query parameter",
)
# Deserialize bundle fields
bundles = (BundleSchema(strict=True,
many=True,
dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS).load(
request.json).data)
# Check for all necessary permissions
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
worksheet_util.check_worksheet_not_frozen(worksheet)
request.user.check_quota(need_time=True, need_disk=True)
created_uuids = []
for bundle in bundles:
# Prep bundle info for saving into database
# Unfortunately cannot use the `construct` methods because they don't
# provide a uniform interface for constructing bundles for all types
# Hopefully this can all be unified after REST migration is complete
bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
created_uuids.append(bundle_uuid)
bundle_class = get_bundle_subclass(bundle['bundle_type'])
bundle['owner_id'] = request.user.user_id
metadata = bundle.get("metadata", {})
if metadata.get("link_url"):
bundle['state'] = State.READY
elif issubclass(bundle_class, UploadedBundle) or query_get_bool(
'wait_for_upload', False):
bundle['state'] = State.UPLOADING
else:
bundle['state'] = State.CREATED
bundle[
'is_anonymous'] = worksheet.is_anonymous # inherit worksheet anonymity
bundle.setdefault('metadata', {})['created'] = int(time.time())
for dep in bundle.setdefault('dependencies', []):
dep['child_uuid'] = bundle_uuid
# Create bundle object
bundle = bundle_class(bundle, strict=False)
# Save bundle into model
local.model.save_bundle(bundle)
# Inherit worksheet permissions
group_permissions = local.model.get_group_worksheet_permissions(
request.user.user_id, worksheet_uuid)
set_bundle_permissions([{
'object_uuid': bundle_uuid,
'group_uuid': p['group_uuid'],
'permission': p['permission'],
} for p in group_permissions])
# Add as item to worksheet
if not detached:
if shadow_parent_uuid is None:
local.model.add_worksheet_items(
worksheet_uuid, [worksheet_util.bundle_item(bundle_uuid)],
after_sort_key)
else:
local.model.add_shadow_worksheet_items(shadow_parent_uuid,
bundle_uuid)
# Get created bundles
bundles_dict = get_bundle_infos(created_uuids)
# Return bundles in original order
# Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted
# right after being created.
bundles = [
bundles_dict[uuid] for uuid in created_uuids if uuid in bundles_dict
]
return BundleSchema(many=True).dump(bundles).data
def fetch_worksheet(uuid):
"""
Fetch a single worksheet by UUID.
Query parameters:
- `include`: comma-separated list of related resources to include, such as "owner"
"""
include_set = query_get_json_api_include_set(
supported={
'owner',
'group_permissions',
'items',
'items.bundle',
'items.bundle.owner',
'items.subworksheet',
})
worksheet = get_worksheet_info(
uuid,
fetch_items='items' in include_set,
fetch_permissions='group_permissions' in include_set,
)
# Build response document
document = WorksheetSchema().dump(worksheet).data
# Include items
if 'items' in include_set:
json_api_include(document, WorksheetItemSchema(), worksheet['items'])
user_ids = set()
# Include bundles
if 'items.bundle' in include_set:
bundle_uuids = {
item['bundle_uuid']
for item in worksheet['items']
if item['type'] == worksheet_util.TYPE_BUNDLE
and item['bundle_uuid'] is not None
}
bundle_infos = list(get_bundle_infos(bundle_uuids).values())
json_api_include(document, BundleSchema(), bundle_infos)
if 'items.bundle.owner' in include_set:
user_ids.update({b['owner_id'] for b in bundle_infos})
# Include users
if 'owner' in include_set:
user_ids.add(worksheet['owner_id'])
if user_ids:
json_api_include(document, UserSchema(),
local.model.get_users(user_ids=user_ids)['results'])
# Include subworksheets
if 'items.subworksheets' in include_set:
subworksheet_uuids = {
item['subworksheet_uuid']
for item in worksheet['items']
if item['type'] == worksheet_util.TYPE_WORKSHEET
and item['subworksheet_uuid'] is not None
}
json_api_include(
document,
WorksheetSchema(),
local.model.batch_get_worksheets(fetch_items=False,
uuid=subworksheet_uuids),
)
# Include permissions
if 'group_permissions' in include_set:
json_api_include(document, WorksheetPermissionSchema(),
worksheet['group_permissions'])
return document
def _create_bundles():
"""
Bulk create bundles.
|worksheet_uuid| - The parent worksheet of the bundle, add to this worksheet
if not detached or shadowing another bundle. Also used
to inherit permissions.
|shadow| - the uuid of the bundle to shadow
|detached| - True ('1') if should not add new bundle to any worksheet,
or False ('0') otherwise. Default is False.
|wait_for_upload| - True ('1') if the bundle state should be initialized to
UPLOADING regardless of the bundle type, or False ('0')
otherwise. This prevents run bundles that are being
copied from another instance from being run by the
BundleManager. Default is False.
"""
worksheet_uuid = request.query.get('worksheet')
shadow_parent_uuid = request.query.get('shadow')
detached = query_get_bool('detached', default=False)
if worksheet_uuid is None:
abort(
httplib.BAD_REQUEST, "Parent worksheet id must be specified as"
"'worksheet' query parameter")
# Deserialize bundle fields
bundles = BundleSchema(
strict=True,
many=True,
dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS,
).load(request.json).data
# Check for all necessary permissions
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
worksheet_util.check_worksheet_not_frozen(worksheet)
request.user.check_quota(need_time=True, need_disk=True)
created_uuids = []
for bundle in bundles:
# Prep bundle info for saving into database
# Unfortunately cannot use the `construct` methods because they don't
# provide a uniform interface for constructing bundles for all types
# Hopefully this can all be unified after REST migration is complete
bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
created_uuids.append(bundle_uuid)
bundle_class = get_bundle_subclass(bundle['bundle_type'])
bundle['owner_id'] = request.user.user_id
bundle['state'] = (
State.UPLOADING if issubclass(bundle_class, UploadedBundle)
or query_get_bool('wait_for_upload', False) else State.CREATED)
bundle.setdefault('metadata', {})['created'] = int(time.time())
for dep in bundle.setdefault('dependencies', []):
dep['child_uuid'] = bundle_uuid
# Create bundle object
bundle = bundle_class(bundle, strict=False)
# Save bundle into model
local.model.save_bundle(bundle)
# Inherit worksheet permissions
group_permissions = local.model.get_group_worksheet_permissions(
request.user.user_id, worksheet_uuid)
set_bundle_permissions([{
'object_uuid': bundle_uuid,
'group_uuid': p['group_uuid'],
'permission': p['permission'],
} for p in group_permissions])
# Add as item to worksheet
if not detached:
if shadow_parent_uuid is None:
local.model.add_worksheet_item(
worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
else:
local.model.add_shadow_worksheet_items(shadow_parent_uuid,
bundle_uuid)
# Get created bundles
bundles_dict = get_bundle_infos(created_uuids)
# Return bundles in original order
bundles = [bundles_dict[uuid] for uuid in created_uuids]
return BundleSchema(many=True).dump(bundles).data
