def build_bundles_document(bundle_uuids):
include_set = query_get_json_api_include_set(supported={
'owner', 'group_permissions', 'children', 'host_worksheets'
})
bundles_dict = get_bundle_infos(
bundle_uuids,
get_children='children' in include_set,
get_permissions='group_permissions' in include_set,
get_host_worksheets='host_worksheets' in include_set,
ignore_not_found=False,
)
# Create list of bundles in original order
bundles = [bundles_dict[uuid] for uuid in bundle_uuids]
# Build response document
document = BundleSchema(many=True).dump(bundles).data
# Shim in display metadata used by the front-end application
if query_get_bool('include_display_metadata', default=False):
for bundle, data in zip(bundles, document['data']):
bundle_class = get_bundle_subclass(bundle['bundle_type'])
json_api_meta(
data,
{
'editable_metadata_keys':
worksheet_util.get_editable_metadata_fields(bundle_class),
'metadata_type':
worksheet_util.get_metadata_types(bundle_class),
},
)
if 'owner' in include_set:
owner_ids = set(b['owner_id'] for b in bundles
if b['owner_id'] is not None)
json_api_include(
document,
UserSchema(),
local.model.get_users(user_ids=owner_ids,
limit=len(owner_ids))['results'],
)
if 'group_permissions' in include_set:
for bundle in bundles:
json_api_include(document, BundlePermissionSchema(),
bundle.get('group_permissions', []))
if 'children' in include_set:
for bundle in bundles:
json_api_include(document, BundleSchema(),
bundle.get('children', []))
if 'host_worksheets' in include_set:
for bundle in bundles:
json_api_include(document, WorksheetSchema(),
bundle.get('host_worksheets', []))
return document
def wrapper(*args, **kwargs):
try:
result = callback(*args, **kwargs)
# If response is JSON, add server version to meta
if isinstance(result, dict):
json_api_meta(result, {'version': CODALAB_VERSION})
return result
except ValidationError as err:
format_errors = query_get_bool('format_errors', default=False)
if format_errors:
msg = err.messages
else:
msg = '\n'.join([e['detail'] for e in err.messages['errors']])
abort(httplib.BAD_REQUEST, msg)
def wrapper(*args, **kwargs):
try:
result = callback(*args, **kwargs)
# If response is JSON, add server version to meta
if isinstance(result, dict):
json_api_meta(result, {'version': CODALAB_VERSION})
return result
except ValidationError as err:
format_errors = query_get_bool('format_errors', default=False)
if format_errors:
msg = err.messages
else:
msg = '\n'.join(
[e['detail'] for e in err.messages['errors']])
abort(httplib.BAD_REQUEST, msg)
def _delete_bundles():
"""
Delete the bundles specified.
Query parameters:
- `force`: 1 to allow deletion of bundles that have descendants or that
appear across multiple worksheets, or 0 to throw an error if any of the
specified bundles have multiple references. Default is 0.
- `recursive`: 1 to remove all bundles downstream too, or 0 otherwise.
Default is 0.
- `data-only`: 1 to only remove contents of the bundle(s) from the bundle
store and leave the bundle metadata intact, or 0 to remove both the
bundle contents and the bundle metadata. Default is 0.
- `dry-run`: 1 to just return list of bundles that would be deleted with
the given parameters without actually deleting them, or 0 to perform
the deletion. Default is 0.
"""
uuids = get_resource_ids(request.json, 'bundles')
force = query_get_bool('force', default=False)
recursive = query_get_bool('recursive', default=False)
data_only = query_get_bool('data-only', default=False)
dry_run = query_get_bool('dry-run', default=False)
deleted_uuids = delete_bundles(uuids,
force=force,
recursive=recursive,
data_only=data_only,
dry_run=dry_run)
# Return list of deleted ids as meta
return json_api_meta({}, {'ids': deleted_uuids})
def _fetch_bundles():
"""
Fetch bundles by bundle specs OR search keywords.
"""
keywords = query_get_list('keywords')
specs = query_get_list('specs')
worksheet_uuid = request.query.get('worksheet')
descendant_depth = query_get_type(int, 'depth', None)
if keywords:
# Handle search keywords
keywords = resolve_owner_in_keywords(keywords)
bundle_uuids = local.model.search_bundle_uuids(request.user.user_id,
worksheet_uuid,
keywords)
elif specs:
# Resolve bundle specs
bundle_uuids = canonicalize.get_bundle_uuids(local.model, request.user,
worksheet_uuid, specs)
else:
abort(
httplib.BAD_REQUEST, "Request must include either 'keywords' "
"or 'specs' query parameter")
# Find all descendants down to the provided depth
if descendant_depth is not None:
bundle_uuids = local.model.get_self_and_descendants(
bundle_uuids, depth=descendant_depth)
# Return simple dict if scalar result (e.g. .sum or .count queries)
if not isinstance(bundle_uuids, list):
return json_api_meta({}, {'result': bundle_uuids})
return build_bundles_document(bundle_uuids)
def build_bundles_document(bundle_uuids):
bundles_dict = get_bundle_infos(
bundle_uuids,
get_children=True,
get_permissions=True,
get_host_worksheets=True,
)
# Create list of bundles in original order
try:
bundles = [bundles_dict[uuid] for uuid in bundle_uuids]
except KeyError as e:
abort(httplib.NOT_FOUND, "Bundle %s not found" % e.args[0])
# Build response document
document = BundleSchema(many=True).dump(bundles).data
# Shim in editable metadata keys
# Used by the front-end application
for bundle, data in izip(bundles, document['data']):
json_api_meta(
data, {
'editable_metadata_keys':
worksheet_util.get_editable_metadata_fields(
get_bundle_subclass(bundle['bundle_type']))
})
# Include users
owner_ids = set(b['owner_id'] for b in bundles)
json_api_include(document, UserSchema(), local.model.get_users(owner_ids))
# Include permissions
for bundle in bundles:
json_api_include(document, BundlePermissionSchema(),
bundle['group_permissions'])
# Include child bundles
children_uuids = set(c['uuid'] for bundle in bundles
for c in bundle['children'])
json_api_include(document, BundleSchema(),
get_bundle_infos(children_uuids).values())
return document
def delete_user():
"""Fetch user ids"""
user_ids = get_resource_ids(request.json, 'users')
request_user_id = request.user.user_id
if request_user_id != local.model.root_user_id:
abort(http.client.UNAUTHORIZED, 'Only root user can delete other users.')
for user_id in user_ids:
if user_id == local.model.root_user_id:
abort(http.client.UNAUTHORIZED, 'Cannot delete root user.')
user = local.model.get_user(user_id=user_id)
if user is None:
abort(http.client.NOT_FOUND, 'User %s not found' % user_id)
'''
Check for owned bundles, worksheets, and groups.
If any are found, then do not allow user to be deleted.
'''
error_messages = []
bundles = local.model.batch_get_bundles(owner_id=user_id)
if bundles is not None and len(bundles) > 0:
bundle_uuids = [bundle.uuid for bundle in bundles]
error_messages.append(
'User %s owns bundles, can\'t delete user. UUIDs: %s\n'
% (user_id, ','.join(bundle_uuids))
)
worksheets = local.model.batch_get_worksheets(fetch_items=False, owner_id=user_id)
if worksheets is not None and len(worksheets) > 0:
worksheet_uuids = [worksheet.uuid for worksheet in worksheets]
error_messages.append(
'User %s owns worksheets, can\'t delete. UUIDs: %s\n'
% (user_id, ','.join(worksheet_uuids))
)
groups = local.model.batch_get_groups(owner_id=user_id)
if groups is not None and len(groups) > 0:
group_uuids = [group.uuid for group in groups]
error_messages.append(
'User %s owns groups, can\'t delete. UUIDs: %s\n' % (user_id, ','.join(group_uuids))
)
if error_messages:
abort(http.client.NOT_FOUND, '\n'.join(error_messages))
local.model.delete_user(user_id=user.user_id)
# Return list of deleted id as meta
return json_api_meta({}, {'ids': user_ids})
def delete_user():
"""Fetch user ids"""
user_ids = get_resource_ids(request.json, 'users')
request_user_id = request.user.user_id
if request_user_id != local.model.root_user_id:
abort(httplib.UNAUTHORIZED, 'Only root user can delete other users.')
for user_id in user_ids:
if user_id == local.model.root_user_id:
abort(httplib.UNAUTHORIZED, 'Cannot delete root user.')
user = local.model.get_user(user_id=user_id)
if user is None:
abort(httplib.NOT_FOUND, 'User %s not found' % user_id)
'''
Check for owned bundles, worksheets, and groups.
If any are found, then do not allow user to be deleted.
'''
error_messages = []
bundles = local.model.batch_get_bundles(owner_id=user_id)
if bundles is not None and len(bundles) > 0:
bundle_uuids = [bundle.uuid for bundle in bundles]
error_messages.append(
'User %s owns bundles, can\'t delete user. UUIDs: %s\n'
% (user_id, ','.join(bundle_uuids))
)
worksheets = local.model.batch_get_worksheets(fetch_items=False, owner_id=user_id)
if worksheets is not None and len(worksheets) > 0:
worksheet_uuids = [worksheet.uuid for worksheet in worksheets]
error_messages.append(
'User %s owns worksheets, can\'t delete. UUIDs: %s\n'
% (user_id, ','.join(worksheet_uuids))
)
groups = local.model.batch_get_groups(owner_id=user_id)
if groups is not None and len(groups) > 0:
group_uuids = [group.uuid for group in groups]
error_messages.append(
'User %s owns groups, can\'t delete. UUIDs: %s\n' % (user_id, ','.join(group_uuids))
)
if error_messages:
abort(httplib.NOT_FOUND, '\n'.join(error_messages))
local.model.delete_user(user_id=user.user_id)
# Return list of deleted id as meta
return json_api_meta({}, {'ids': user_ids})
def fetch_users():
"""
Fetch list of users, filterable by username and email.
Takes the following query parameters:
filter[user_name]=name1,name2,...
filter[email]=email1,email2,...
Query parameters:
- `keywords`: Search keyword. May be provided multiple times for multiple
keywords.
Examples of other special keyword forms:
- `name=<name> ` : More targeted search of using metadata fields.
- `date_joined=.sort ` : Sort by a particular field.
- `date_joined=.sort- ` : Sort by a particular field in reverse.
- `.count ` : Count the number of users.
- `.limit=10 ` : Limit the number of results to the top 10.
"""
# Combine username and email filters
usernames = set(request.query.get('filter[user_name]', '').split(','))
usernames |= set(request.query.get('filter[email]', '').split(','))
usernames.discard('') # str.split(',') will return '' on empty strings
keywords = query_get_list('keywords')
if usernames is None and keywords is None:
abort(http.client.BAD_REQUEST,
"Request must include 'keywords' query parameter or usernames")
if request.user.user_id != local.model.root_user_id:
for key in keywords:
if not all(accessed_field in key
for accessed_field in USER_ACCESSIBLE_KEYWORDS):
abort(http.client.FORBIDDEN,
"You don't have access to search for these fields")
# Handle search keywords
users = local.model.get_users(keywords=(keywords or None),
usernames=(usernames or None))
# Return simple dict if scalar result (e.g. .sum or .count queries)
if users.get('is_aggregate'):
return json_api_meta({}, {'results': users['results']})
else:
users = users['results']
return allowed_user_schema()(many=True).dump(users).data
def _delete_bundles():
"""
Delete the bundles specified.
If |force|, allow deletion of bundles that have descendants or that appear across multiple worksheets.
If |recursive|, add all bundles downstream too.
If |data-only|, only remove from the bundle store, not the bundle metadata.
If |dry-run|, just return list of bundles that would be deleted, but do not actually delete.
"""
uuids = get_resource_ids(request.json, 'bundles')
force = query_get_bool('force', default=False)
recursive = query_get_bool('recursive', default=False)
data_only = query_get_bool('data-only', default=False)
dry_run = query_get_bool('dry-run', default=False)
deleted_uuids = delete_bundles(uuids,
force=force,
recursive=recursive,
data_only=data_only,
dry_run=dry_run)
# Return list of deleted ids as meta
return json_api_meta({}, {'ids': deleted_uuids})
def _fetch_bundles():
"""
Fetch bundles in the following two ways:
1. By bundle `specs` OR search `keywords` . Behavior is undefined
when both `specs` and `keywords` are provided.
Query parameters:
- `worksheet`: UUID of the base worksheet. Required when fetching by specs.
- `specs`: Bundle spec of bundle to fetch. May be provided multiples times
to fetch multiple bundle specs. A bundle spec is either:
1. a UUID (8 or 32 hex characters with a preceding '0x')
2. a bundle name referring to the last bundle with that name on the
given base worksheet
3. or a reverse index of the form `^N` referring to the Nth-to-last
bundle on the given base worksheet.
- `keywords`: Search keyword. May be provided multiple times for multiple
keywords. Bare keywords match the names and descriptions of bundles.
Examples of other special keyword forms:
- `name=<name> ` : More targeted search of using metadata fields.
- `size=.sort ` : Sort by a particular field.
- `size=.sort- ` : Sort by a particular field in reverse.
- `size=.sum ` : Compute total of a particular field.
- `.mine ` : Match only bundles I own.
- `.floating ` : Match bundles that aren't on any worksheet.
- `.count ` : Count the number of bundles.
- `.limit=10 ` : Limit the number of results to the top 10.
- `include_display_metadata`: `1` to include additional metadata helpful
for displaying the bundle info, `0` to omit them. Default is `0`.
- `include`: comma-separated list of related resources to include, such as "owner"
When aggregation keywords such as `.count` are used, the resulting value
is returned as:
```
{
"meta": {
"results": <value>
}
}
```
2. By bundle `command` and/or `dependencies` (for `--memoized` option in cl [run/mimic] command).
When `dependencies` is not defined, the searching result will include bundles that match with command only.
Query parameters:
- `command` : the command of a bundle in string
- `dependencies` : the dependencies of a bundle in the format of
'[{"child_path":key1, "parent_uuid":UUID1},
{"child_path":key2, "parent_uuid":UUID2}]'
1. a UUID should be in the format of 32 hex characters with a preceding '0x' (partial UUID is not allowed).
2. the key should be able to uniquely identify a (child_path, parent_uuid) pair in the list.
The returning result will be aggregated in the same way as 1.
"""
keywords = query_get_list('keywords')
specs = query_get_list('specs')
worksheet_uuid = request.query.get('worksheet')
descendant_depth = query_get_type(int, 'depth', None)
command = query_get_type(str, 'command', '')
dependencies = query_get_type(str, 'dependencies', '[]')
if keywords:
# Handle search keywords
keywords = resolve_owner_in_keywords(keywords)
search_result = local.model.search_bundles(request.user.user_id,
keywords)
# Return simple dict if scalar result (e.g. .sum or .count queries)
if search_result['is_aggregate']:
return json_api_meta({}, {'result': search_result['result']})
# If not aggregate this is a list
bundle_uuids = search_result['result']
elif specs:
# Resolve bundle specs
bundle_uuids = canonicalize.get_bundle_uuids(local.model, request.user,
worksheet_uuid, specs)
elif command:
bundle_uuids = local.model.get_memoized_bundles(
request.user.user_id, command, dependencies)
else:
abort(
http.client.BAD_REQUEST,
"Request must include either 'keywords' "
"or 'specs' query parameter",
)
# Find all descendants down to the provided depth
if descendant_depth is not None:
bundle_uuids = local.model.get_self_and_descendants(
bundle_uuids, depth=descendant_depth)
return build_bundles_document(bundle_uuids)
def _fetch_bundles():
"""
Fetch bundles by bundle `specs` OR search `keywords`. Behavior is undefined
when both `specs` and `keywords` are provided.
Query parameters:
- `worksheet`: UUID of the base worksheet. Required when fetching by specs.
- `specs`: Bundle spec of bundle to fetch. May be provided multiples times
to fetch multiple bundle specs. A bundle spec is either:
1. a UUID (8 or 32 hex characters with a preceding '0x')
2. a bundle name referring to the last bundle with that name on the
given base worksheet
3. or a reverse index of the form `^N` referring to the Nth-to-last
bundle on the given base worksheet.
- `keywords`: Search keyword. May be provided multiples times for multiple
keywords. Bare keywords match the names and descriptions of bundles.
Examples of other special keyword forms:
- `name=<name> ` : More targeted search of using metadata fields.
- `size=.sort ` : Sort by a particular field.
- `size=.sort- ` : Sort by a particular field in reverse.
- `size=.sum ` : Compute total of a particular field.
- `.mine ` : Match only bundles I own.
- `.floating ` : Match bundles that aren't on any worksheet.
- `.count ` : Count the number of bundles.
- `.limit=10 ` : Limit the number of results to the top 10.
- `include_display_metadata`: `1` to include additional metadata helpful
for displaying the bundle info, `0` to omit them. Default is `0`.
- `include`: comma-separated list of related resources to include, such as "owner"
When aggregation keywords such as `.count` are used, the resulting value
is returned as:
```
{
"meta": {
"results": <value>
}
}
```
"""
keywords = query_get_list('keywords')
specs = query_get_list('specs')
worksheet_uuid = request.query.get('worksheet')
descendant_depth = query_get_type(int, 'depth', None)
if keywords:
# Handle search keywords
keywords = resolve_owner_in_keywords(keywords)
bundle_uuids = local.model.search_bundle_uuids(request.user.user_id, keywords)
elif specs:
# Resolve bundle specs
bundle_uuids = canonicalize.get_bundle_uuids(local.model, request.user, worksheet_uuid, specs)
else:
abort(httplib.BAD_REQUEST,
"Request must include either 'keywords' "
"or 'specs' query parameter")
# Find all descendants down to the provided depth
if descendant_depth is not None:
bundle_uuids = local.model.get_self_and_descendants(bundle_uuids, depth=descendant_depth)
# Return simple dict if scalar result (e.g. .sum or .count queries)
if not isinstance(bundle_uuids, list):
return json_api_meta({}, {'result': bundle_uuids})
return build_bundles_document(bundle_uuids)
