def build_bundles_document(bundle_uuids): include_set = query_get_json_api_include_set(supported={ 'owner', 'group_permissions', 'children', 'host_worksheets' }) bundles_dict = get_bundle_infos( bundle_uuids, get_children='children' in include_set, get_permissions='group_permissions' in include_set, get_host_worksheets='host_worksheets' in include_set, ignore_not_found=False, ) # Create list of bundles in original order bundles = [bundles_dict[uuid] for uuid in bundle_uuids] # Build response document document = BundleSchema(many=True).dump(bundles).data # Shim in display metadata used by the front-end application if query_get_bool('include_display_metadata', default=False): for bundle, data in zip(bundles, document['data']): bundle_class = get_bundle_subclass(bundle['bundle_type']) json_api_meta( data, { 'editable_metadata_keys': worksheet_util.get_editable_metadata_fields(bundle_class), 'metadata_type': worksheet_util.get_metadata_types(bundle_class), }, ) if 'owner' in include_set: owner_ids = set(b['owner_id'] for b in bundles if b['owner_id'] is not None) json_api_include( document, UserSchema(), local.model.get_users(user_ids=owner_ids, limit=len(owner_ids))['results'], ) if 'group_permissions' in include_set: for bundle in bundles: json_api_include(document, BundlePermissionSchema(), bundle.get('group_permissions', [])) if 'children' in include_set: for bundle in bundles: json_api_include(document, BundleSchema(), bundle.get('children', [])) if 'host_worksheets' in include_set: for bundle in bundles: json_api_include(document, WorksheetSchema(), bundle.get('host_worksheets', [])) return document
def wrapper(*args, **kwargs): try: result = callback(*args, **kwargs) # If response is JSON, add server version to meta if isinstance(result, dict): json_api_meta(result, {'version': CODALAB_VERSION}) return result except ValidationError as err: format_errors = query_get_bool('format_errors', default=False) if format_errors: msg = err.messages else: msg = '\n'.join([e['detail'] for e in err.messages['errors']]) abort(httplib.BAD_REQUEST, msg)
def wrapper(*args, **kwargs): try: result = callback(*args, **kwargs) # If response is JSON, add server version to meta if isinstance(result, dict): json_api_meta(result, {'version': CODALAB_VERSION}) return result except ValidationError as err: format_errors = query_get_bool('format_errors', default=False) if format_errors: msg = err.messages else: msg = '\n'.join( [e['detail'] for e in err.messages['errors']]) abort(httplib.BAD_REQUEST, msg)
def _delete_bundles(): """ Delete the bundles specified. Query parameters: - `force`: 1 to allow deletion of bundles that have descendants or that appear across multiple worksheets, or 0 to throw an error if any of the specified bundles have multiple references. Default is 0. - `recursive`: 1 to remove all bundles downstream too, or 0 otherwise. Default is 0. - `data-only`: 1 to only remove contents of the bundle(s) from the bundle store and leave the bundle metadata intact, or 0 to remove both the bundle contents and the bundle metadata. Default is 0. - `dry-run`: 1 to just return list of bundles that would be deleted with the given parameters without actually deleting them, or 0 to perform the deletion. Default is 0. """ uuids = get_resource_ids(request.json, 'bundles') force = query_get_bool('force', default=False) recursive = query_get_bool('recursive', default=False) data_only = query_get_bool('data-only', default=False) dry_run = query_get_bool('dry-run', default=False) deleted_uuids = delete_bundles(uuids, force=force, recursive=recursive, data_only=data_only, dry_run=dry_run) # Return list of deleted ids as meta return json_api_meta({}, {'ids': deleted_uuids})
def _fetch_bundles(): """ Fetch bundles by bundle specs OR search keywords. """ keywords = query_get_list('keywords') specs = query_get_list('specs') worksheet_uuid = request.query.get('worksheet') descendant_depth = query_get_type(int, 'depth', None) if keywords: # Handle search keywords keywords = resolve_owner_in_keywords(keywords) bundle_uuids = local.model.search_bundle_uuids(request.user.user_id, worksheet_uuid, keywords) elif specs: # Resolve bundle specs bundle_uuids = canonicalize.get_bundle_uuids(local.model, request.user, worksheet_uuid, specs) else: abort( httplib.BAD_REQUEST, "Request must include either 'keywords' " "or 'specs' query parameter") # Find all descendants down to the provided depth if descendant_depth is not None: bundle_uuids = local.model.get_self_and_descendants( bundle_uuids, depth=descendant_depth) # Return simple dict if scalar result (e.g. .sum or .count queries) if not isinstance(bundle_uuids, list): return json_api_meta({}, {'result': bundle_uuids}) return build_bundles_document(bundle_uuids)
def build_bundles_document(bundle_uuids): bundles_dict = get_bundle_infos( bundle_uuids, get_children=True, get_permissions=True, get_host_worksheets=True, ) # Create list of bundles in original order try: bundles = [bundles_dict[uuid] for uuid in bundle_uuids] except KeyError as e: abort(httplib.NOT_FOUND, "Bundle %s not found" % e.args[0]) # Build response document document = BundleSchema(many=True).dump(bundles).data # Shim in editable metadata keys # Used by the front-end application for bundle, data in izip(bundles, document['data']): json_api_meta( data, { 'editable_metadata_keys': worksheet_util.get_editable_metadata_fields( get_bundle_subclass(bundle['bundle_type'])) }) # Include users owner_ids = set(b['owner_id'] for b in bundles) json_api_include(document, UserSchema(), local.model.get_users(owner_ids)) # Include permissions for bundle in bundles: json_api_include(document, BundlePermissionSchema(), bundle['group_permissions']) # Include child bundles children_uuids = set(c['uuid'] for bundle in bundles for c in bundle['children']) json_api_include(document, BundleSchema(), get_bundle_infos(children_uuids).values()) return document
def delete_user(): """Fetch user ids""" user_ids = get_resource_ids(request.json, 'users') request_user_id = request.user.user_id if request_user_id != local.model.root_user_id: abort(http.client.UNAUTHORIZED, 'Only root user can delete other users.') for user_id in user_ids: if user_id == local.model.root_user_id: abort(http.client.UNAUTHORIZED, 'Cannot delete root user.') user = local.model.get_user(user_id=user_id) if user is None: abort(http.client.NOT_FOUND, 'User %s not found' % user_id) ''' Check for owned bundles, worksheets, and groups. If any are found, then do not allow user to be deleted. ''' error_messages = [] bundles = local.model.batch_get_bundles(owner_id=user_id) if bundles is not None and len(bundles) > 0: bundle_uuids = [bundle.uuid for bundle in bundles] error_messages.append( 'User %s owns bundles, can\'t delete user. UUIDs: %s\n' % (user_id, ','.join(bundle_uuids)) ) worksheets = local.model.batch_get_worksheets(fetch_items=False, owner_id=user_id) if worksheets is not None and len(worksheets) > 0: worksheet_uuids = [worksheet.uuid for worksheet in worksheets] error_messages.append( 'User %s owns worksheets, can\'t delete. UUIDs: %s\n' % (user_id, ','.join(worksheet_uuids)) ) groups = local.model.batch_get_groups(owner_id=user_id) if groups is not None and len(groups) > 0: group_uuids = [group.uuid for group in groups] error_messages.append( 'User %s owns groups, can\'t delete. UUIDs: %s\n' % (user_id, ','.join(group_uuids)) ) if error_messages: abort(http.client.NOT_FOUND, '\n'.join(error_messages)) local.model.delete_user(user_id=user.user_id) # Return list of deleted id as meta return json_api_meta({}, {'ids': user_ids})
def delete_user(): """Fetch user ids""" user_ids = get_resource_ids(request.json, 'users') request_user_id = request.user.user_id if request_user_id != local.model.root_user_id: abort(httplib.UNAUTHORIZED, 'Only root user can delete other users.') for user_id in user_ids: if user_id == local.model.root_user_id: abort(httplib.UNAUTHORIZED, 'Cannot delete root user.') user = local.model.get_user(user_id=user_id) if user is None: abort(httplib.NOT_FOUND, 'User %s not found' % user_id) ''' Check for owned bundles, worksheets, and groups. If any are found, then do not allow user to be deleted. ''' error_messages = [] bundles = local.model.batch_get_bundles(owner_id=user_id) if bundles is not None and len(bundles) > 0: bundle_uuids = [bundle.uuid for bundle in bundles] error_messages.append( 'User %s owns bundles, can\'t delete user. UUIDs: %s\n' % (user_id, ','.join(bundle_uuids)) ) worksheets = local.model.batch_get_worksheets(fetch_items=False, owner_id=user_id) if worksheets is not None and len(worksheets) > 0: worksheet_uuids = [worksheet.uuid for worksheet in worksheets] error_messages.append( 'User %s owns worksheets, can\'t delete. UUIDs: %s\n' % (user_id, ','.join(worksheet_uuids)) ) groups = local.model.batch_get_groups(owner_id=user_id) if groups is not None and len(groups) > 0: group_uuids = [group.uuid for group in groups] error_messages.append( 'User %s owns groups, can\'t delete. UUIDs: %s\n' % (user_id, ','.join(group_uuids)) ) if error_messages: abort(httplib.NOT_FOUND, '\n'.join(error_messages)) local.model.delete_user(user_id=user.user_id) # Return list of deleted id as meta return json_api_meta({}, {'ids': user_ids})
def fetch_users(): """ Fetch list of users, filterable by username and email. Takes the following query parameters: filter[user_name]=name1,name2,... filter[email]=email1,email2,... Query parameters: - `keywords`: Search keyword. May be provided multiple times for multiple keywords. Examples of other special keyword forms: - `name=<name> ` : More targeted search of using metadata fields. - `date_joined=.sort ` : Sort by a particular field. - `date_joined=.sort- ` : Sort by a particular field in reverse. - `.count ` : Count the number of users. - `.limit=10 ` : Limit the number of results to the top 10. """ # Combine username and email filters usernames = set(request.query.get('filter[user_name]', '').split(',')) usernames |= set(request.query.get('filter[email]', '').split(',')) usernames.discard('') # str.split(',') will return '' on empty strings keywords = query_get_list('keywords') if usernames is None and keywords is None: abort(http.client.BAD_REQUEST, "Request must include 'keywords' query parameter or usernames") if request.user.user_id != local.model.root_user_id: for key in keywords: if not all(accessed_field in key for accessed_field in USER_ACCESSIBLE_KEYWORDS): abort(http.client.FORBIDDEN, "You don't have access to search for these fields") # Handle search keywords users = local.model.get_users(keywords=(keywords or None), usernames=(usernames or None)) # Return simple dict if scalar result (e.g. .sum or .count queries) if users.get('is_aggregate'): return json_api_meta({}, {'results': users['results']}) else: users = users['results'] return allowed_user_schema()(many=True).dump(users).data
def _delete_bundles(): """ Delete the bundles specified. If |force|, allow deletion of bundles that have descendants or that appear across multiple worksheets. If |recursive|, add all bundles downstream too. If |data-only|, only remove from the bundle store, not the bundle metadata. If |dry-run|, just return list of bundles that would be deleted, but do not actually delete. """ uuids = get_resource_ids(request.json, 'bundles') force = query_get_bool('force', default=False) recursive = query_get_bool('recursive', default=False) data_only = query_get_bool('data-only', default=False) dry_run = query_get_bool('dry-run', default=False) deleted_uuids = delete_bundles(uuids, force=force, recursive=recursive, data_only=data_only, dry_run=dry_run) # Return list of deleted ids as meta return json_api_meta({}, {'ids': deleted_uuids})
def _fetch_bundles(): """ Fetch bundles in the following two ways: 1. By bundle `specs` OR search `keywords` . Behavior is undefined when both `specs` and `keywords` are provided. Query parameters: - `worksheet`: UUID of the base worksheet. Required when fetching by specs. - `specs`: Bundle spec of bundle to fetch. May be provided multiples times to fetch multiple bundle specs. A bundle spec is either: 1. a UUID (8 or 32 hex characters with a preceding '0x') 2. a bundle name referring to the last bundle with that name on the given base worksheet 3. or a reverse index of the form `^N` referring to the Nth-to-last bundle on the given base worksheet. - `keywords`: Search keyword. May be provided multiple times for multiple keywords. Bare keywords match the names and descriptions of bundles. Examples of other special keyword forms: - `name=<name> ` : More targeted search of using metadata fields. - `size=.sort ` : Sort by a particular field. - `size=.sort- ` : Sort by a particular field in reverse. - `size=.sum ` : Compute total of a particular field. - `.mine ` : Match only bundles I own. - `.floating ` : Match bundles that aren't on any worksheet. - `.count ` : Count the number of bundles. - `.limit=10 ` : Limit the number of results to the top 10. - `include_display_metadata`: `1` to include additional metadata helpful for displaying the bundle info, `0` to omit them. Default is `0`. - `include`: comma-separated list of related resources to include, such as "owner" When aggregation keywords such as `.count` are used, the resulting value is returned as: ``` { "meta": { "results": <value> } } ``` 2. By bundle `command` and/or `dependencies` (for `--memoized` option in cl [run/mimic] command). When `dependencies` is not defined, the searching result will include bundles that match with command only. Query parameters: - `command` : the command of a bundle in string - `dependencies` : the dependencies of a bundle in the format of '[{"child_path":key1, "parent_uuid":UUID1}, {"child_path":key2, "parent_uuid":UUID2}]' 1. a UUID should be in the format of 32 hex characters with a preceding '0x' (partial UUID is not allowed). 2. the key should be able to uniquely identify a (child_path, parent_uuid) pair in the list. The returning result will be aggregated in the same way as 1. """ keywords = query_get_list('keywords') specs = query_get_list('specs') worksheet_uuid = request.query.get('worksheet') descendant_depth = query_get_type(int, 'depth', None) command = query_get_type(str, 'command', '') dependencies = query_get_type(str, 'dependencies', '[]') if keywords: # Handle search keywords keywords = resolve_owner_in_keywords(keywords) search_result = local.model.search_bundles(request.user.user_id, keywords) # Return simple dict if scalar result (e.g. .sum or .count queries) if search_result['is_aggregate']: return json_api_meta({}, {'result': search_result['result']}) # If not aggregate this is a list bundle_uuids = search_result['result'] elif specs: # Resolve bundle specs bundle_uuids = canonicalize.get_bundle_uuids(local.model, request.user, worksheet_uuid, specs) elif command: bundle_uuids = local.model.get_memoized_bundles( request.user.user_id, command, dependencies) else: abort( http.client.BAD_REQUEST, "Request must include either 'keywords' " "or 'specs' query parameter", ) # Find all descendants down to the provided depth if descendant_depth is not None: bundle_uuids = local.model.get_self_and_descendants( bundle_uuids, depth=descendant_depth) return build_bundles_document(bundle_uuids)
def _fetch_bundles(): """ Fetch bundles by bundle `specs` OR search `keywords`. Behavior is undefined when both `specs` and `keywords` are provided. Query parameters: - `worksheet`: UUID of the base worksheet. Required when fetching by specs. - `specs`: Bundle spec of bundle to fetch. May be provided multiples times to fetch multiple bundle specs. A bundle spec is either: 1. a UUID (8 or 32 hex characters with a preceding '0x') 2. a bundle name referring to the last bundle with that name on the given base worksheet 3. or a reverse index of the form `^N` referring to the Nth-to-last bundle on the given base worksheet. - `keywords`: Search keyword. May be provided multiples times for multiple keywords. Bare keywords match the names and descriptions of bundles. Examples of other special keyword forms: - `name=<name> ` : More targeted search of using metadata fields. - `size=.sort ` : Sort by a particular field. - `size=.sort- ` : Sort by a particular field in reverse. - `size=.sum ` : Compute total of a particular field. - `.mine ` : Match only bundles I own. - `.floating ` : Match bundles that aren't on any worksheet. - `.count ` : Count the number of bundles. - `.limit=10 ` : Limit the number of results to the top 10. - `include_display_metadata`: `1` to include additional metadata helpful for displaying the bundle info, `0` to omit them. Default is `0`. - `include`: comma-separated list of related resources to include, such as "owner" When aggregation keywords such as `.count` are used, the resulting value is returned as: ``` { "meta": { "results": <value> } } ``` """ keywords = query_get_list('keywords') specs = query_get_list('specs') worksheet_uuid = request.query.get('worksheet') descendant_depth = query_get_type(int, 'depth', None) if keywords: # Handle search keywords keywords = resolve_owner_in_keywords(keywords) bundle_uuids = local.model.search_bundle_uuids(request.user.user_id, keywords) elif specs: # Resolve bundle specs bundle_uuids = canonicalize.get_bundle_uuids(local.model, request.user, worksheet_uuid, specs) else: abort(httplib.BAD_REQUEST, "Request must include either 'keywords' " "or 'specs' query parameter") # Find all descendants down to the provided depth if descendant_depth is not None: bundle_uuids = local.model.get_self_and_descendants(bundle_uuids, depth=descendant_depth) # Return simple dict if scalar result (e.g. .sum or .count queries) if not isinstance(bundle_uuids, list): return json_api_meta({}, {'result': bundle_uuids}) return build_bundles_document(bundle_uuids)