def _update_bundles(): """ Bulk update bundles. """ bundle_updates = BundleSchema( strict=True, many=True, dump_only=BUNDLE_UPDATE_RESTRICTED_FIELDS, ).load(request.json, partial=True).data # Check permissions bundle_uuids = [b.pop('uuid') for b in bundle_updates] check_bundles_have_all_permission(local.model, request.user, bundle_uuids) bundles = local.model.batch_get_bundles(uuid=bundle_uuids) # Update bundles for bundle, update in izip(bundles, bundle_updates): local.model.update_bundle(bundle, update) # Get updated bundles bundles_dict = get_bundle_infos(bundle_uuids) # Create list of bundles in original order updated_bundles = [bundles_dict[uuid] for uuid in bundle_uuids] return BundleSchema(many=True).dump(updated_bundles).data
def _update_bundles(): """ Bulk update bundles. """ bundle_updates = (BundleSchema( strict=True, many=True, dump_only=BUNDLE_UPDATE_RESTRICTED_FIELDS).load(request.json, partial=True).data) # Check permissions bundle_uuids = [b.pop('uuid') for b in bundle_updates] check_bundles_have_all_permission(local.model, request.user, bundle_uuids) bundles = local.model.batch_get_bundles(uuid=bundle_uuids) # Update bundles for bundle, update in zip(bundles, bundle_updates): local.model.update_bundle(bundle, update) # Get updated bundles bundles_dict = get_bundle_infos(bundle_uuids) # Create list of bundles in original order # Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted # right after being updated. updated_bundles = [ bundles_dict[uuid] for uuid in bundle_uuids if uuid in bundles_dict ] return BundleSchema(many=True).dump(updated_bundles).data
def build_bundles_document(bundle_uuids): include_set = query_get_json_api_include_set(supported={ 'owner', 'group_permissions', 'children', 'host_worksheets' }) bundles_dict = get_bundle_infos( bundle_uuids, get_children='children' in include_set, get_permissions='group_permissions' in include_set, get_host_worksheets='host_worksheets' in include_set, ignore_not_found=False, ) # Create list of bundles in original order bundles = [bundles_dict[uuid] for uuid in bundle_uuids] # Build response document document = BundleSchema(many=True).dump(bundles).data # Shim in display metadata used by the front-end application if query_get_bool('include_display_metadata', default=False): for bundle, data in zip(bundles, document['data']): bundle_class = get_bundle_subclass(bundle['bundle_type']) json_api_meta( data, { 'editable_metadata_keys': worksheet_util.get_editable_metadata_fields(bundle_class), 'metadata_type': worksheet_util.get_metadata_types(bundle_class), }, ) if 'owner' in include_set: owner_ids = set(b['owner_id'] for b in bundles if b['owner_id'] is not None) json_api_include( document, UserSchema(), local.model.get_users(user_ids=owner_ids, limit=len(owner_ids))['results'], ) if 'group_permissions' in include_set: for bundle in bundles: json_api_include(document, BundlePermissionSchema(), bundle.get('group_permissions', [])) if 'children' in include_set: for bundle in bundles: json_api_include(document, BundleSchema(), bundle.get('children', [])) if 'host_worksheets' in include_set: for bundle in bundles: json_api_include(document, WorksheetSchema(), bundle.get('host_worksheets', [])) return document
def build_bundles_document(bundle_uuids): bundles_dict = get_bundle_infos( bundle_uuids, get_children=True, get_permissions=True, get_host_worksheets=True, ) # Create list of bundles in original order try: bundles = [bundles_dict[uuid] for uuid in bundle_uuids] except KeyError as e: abort(httplib.NOT_FOUND, "Bundle %s not found" % e.args[0]) # Build response document document = BundleSchema(many=True).dump(bundles).data # Shim in editable metadata keys # Used by the front-end application for bundle, data in izip(bundles, document['data']): json_api_meta( data, { 'editable_metadata_keys': worksheet_util.get_editable_metadata_fields( get_bundle_subclass(bundle['bundle_type'])) }) # Include users owner_ids = set(b['owner_id'] for b in bundles) json_api_include(document, UserSchema(), local.model.get_users(owner_ids)) # Include permissions for bundle in bundles: json_api_include(document, BundlePermissionSchema(), bundle['group_permissions']) # Include child bundles children_uuids = set(c['uuid'] for bundle in bundles for c in bundle['children']) json_api_include(document, BundleSchema(), get_bundle_infos(children_uuids).values()) return document
def _update_bundles(): """ Bulk update bundles. """ bundle_updates = (BundleSchema( strict=True, many=True, dump_only=BUNDLE_UPDATE_RESTRICTED_FIELDS).load(request.json, partial=True).data) # Check permissions bundle_uuids = [b.pop('uuid') for b in bundle_updates] check_bundles_have_all_permission(local.model, request.user, bundle_uuids) bundles = local.model.batch_get_bundles(uuid=bundle_uuids) for bundle, update in zip(bundles, bundle_updates): if "frozen" not in update: bundle_util.check_bundle_not_frozen(bundle) else: # If we're freezing or unfreezing the bundle, check that # the bundle is in a final state. # If we're freezing, additionally check that the bundle is not already frozen. bundle_util.check_bundle_freezable(bundle) if update["frozen"]: bundle_util.check_bundle_not_frozen(bundle) # Update bundles for bundle, update in zip(bundles, bundle_updates): local.model.update_bundle(bundle, update) # Get updated bundles bundles_dict = get_bundle_infos(bundle_uuids) # Create list of bundles in original order # Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted # right after being updated. updated_bundles = [ bundles_dict[uuid] for uuid in bundle_uuids if uuid in bundles_dict ] return BundleSchema(many=True).dump(updated_bundles).data
def fetch_worksheet(uuid): worksheet = get_worksheet_info( uuid, fetch_items=True, fetch_permission=True, ) # Build response document document = WorksheetSchema().dump(worksheet).data # Include items json_api_include(document, WorksheetItemSchema(), worksheet['items']) # Include bundles bundle_uuids = { item['bundle_uuid'] for item in worksheet['items'] if item['type'] == worksheet_util.TYPE_BUNDLE and item['bundle_uuid'] is not None } bundle_infos = get_bundle_infos(bundle_uuids).values() json_api_include(document, BundleSchema(), bundle_infos) # Include users user_ids = {b['owner_id'] for b in bundle_infos} user_ids.add(worksheet['owner_id']) if user_ids: json_api_include(document, UserSchema(), local.model.get_users(user_ids)) # Include subworksheets subworksheet_uuids = { item['subworksheet_uuid'] for item in worksheet['items'] if item['type'] == worksheet_util.TYPE_WORKSHEET and item['subworksheet_uuid'] is not None } json_api_include( document, WorksheetSchema(), local.model.batch_get_worksheets(fetch_items=False, uuid=subworksheet_uuids)) # FIXME: tokenizing directive args # value_obj = formatting.string_to_tokens(value) if type == worksheet_util.TYPE_DIRECTIVE else value # Include permissions json_api_include(document, WorksheetPermissionSchema(), worksheet['group_permissions']) return document
def _create_bundles(): """ Bulk create bundles. Query parameters: - `worksheet`: UUID of the parent worksheet of the new bundle, add to this worksheet if not detached or shadowing another bundle. The new bundle also inherits permissions from this worksheet. - `shadow`: UUID of the bundle to "shadow" (the new bundle will be added as an item immediately after this bundle in its parent worksheet). - `detached`: 1 if should not add new bundle to any worksheet, or 0 otherwise. Default is 0. - `wait_for_upload`: 1 if the bundle state should be initialized to "uploading" regardless of the bundle type, or 0 otherwise. Used when copying bundles from another CodaLab instance, this prevents these new bundles from being executed by the BundleManager. Default is 0. """ worksheet_uuid = request.query.get('worksheet') shadow_parent_uuid = request.query.get('shadow') after_sort_key = request.query.get('after_sort_key') detached = query_get_bool('detached', default=False) if worksheet_uuid is None: abort( http.client.BAD_REQUEST, "Parent worksheet id must be specified as" "'worksheet' query parameter", ) # Deserialize bundle fields bundles = (BundleSchema(strict=True, many=True, dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS).load( request.json).data) # Check for all necessary permissions worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False) check_worksheet_has_all_permission(local.model, request.user, worksheet) worksheet_util.check_worksheet_not_frozen(worksheet) request.user.check_quota(need_time=True, need_disk=True) created_uuids = [] for bundle in bundles: # Prep bundle info for saving into database # Unfortunately cannot use the `construct` methods because they don't # provide a uniform interface for constructing bundles for all types # Hopefully this can all be unified after REST migration is complete bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid()) created_uuids.append(bundle_uuid) bundle_class = get_bundle_subclass(bundle['bundle_type']) bundle['owner_id'] = request.user.user_id metadata = bundle.get("metadata", {}) if metadata.get("link_url"): bundle['state'] = State.READY elif issubclass(bundle_class, UploadedBundle) or query_get_bool( 'wait_for_upload', False): bundle['state'] = State.UPLOADING else: bundle['state'] = State.CREATED bundle[ 'is_anonymous'] = worksheet.is_anonymous # inherit worksheet anonymity bundle.setdefault('metadata', {})['created'] = int(time.time()) for dep in bundle.setdefault('dependencies', []): dep['child_uuid'] = bundle_uuid # Create bundle object bundle = bundle_class(bundle, strict=False) # Save bundle into model local.model.save_bundle(bundle) # Inherit worksheet permissions group_permissions = local.model.get_group_worksheet_permissions( request.user.user_id, worksheet_uuid) set_bundle_permissions([{ 'object_uuid': bundle_uuid, 'group_uuid': p['group_uuid'], 'permission': p['permission'], } for p in group_permissions]) # Add as item to worksheet if not detached: if shadow_parent_uuid is None: local.model.add_worksheet_items( worksheet_uuid, [worksheet_util.bundle_item(bundle_uuid)], after_sort_key) else: local.model.add_shadow_worksheet_items(shadow_parent_uuid, bundle_uuid) # Get created bundles bundles_dict = get_bundle_infos(created_uuids) # Return bundles in original order # Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted # right after being created. bundles = [ bundles_dict[uuid] for uuid in created_uuids if uuid in bundles_dict ] return BundleSchema(many=True).dump(bundles).data
def fetch_worksheet(uuid): """ Fetch a single worksheet by UUID. Query parameters: - `include`: comma-separated list of related resources to include, such as "owner" """ include_set = query_get_json_api_include_set( supported={ 'owner', 'group_permissions', 'items', 'items.bundle', 'items.bundle.owner', 'items.subworksheet', }) worksheet = get_worksheet_info( uuid, fetch_items='items' in include_set, fetch_permissions='group_permissions' in include_set, ) # Build response document document = WorksheetSchema().dump(worksheet).data # Include items if 'items' in include_set: json_api_include(document, WorksheetItemSchema(), worksheet['items']) user_ids = set() # Include bundles if 'items.bundle' in include_set: bundle_uuids = { item['bundle_uuid'] for item in worksheet['items'] if item['type'] == worksheet_util.TYPE_BUNDLE and item['bundle_uuid'] is not None } bundle_infos = list(get_bundle_infos(bundle_uuids).values()) json_api_include(document, BundleSchema(), bundle_infos) if 'items.bundle.owner' in include_set: user_ids.update({b['owner_id'] for b in bundle_infos}) # Include users if 'owner' in include_set: user_ids.add(worksheet['owner_id']) if user_ids: json_api_include(document, UserSchema(), local.model.get_users(user_ids=user_ids)['results']) # Include subworksheets if 'items.subworksheets' in include_set: subworksheet_uuids = { item['subworksheet_uuid'] for item in worksheet['items'] if item['type'] == worksheet_util.TYPE_WORKSHEET and item['subworksheet_uuid'] is not None } json_api_include( document, WorksheetSchema(), local.model.batch_get_worksheets(fetch_items=False, uuid=subworksheet_uuids), ) # Include permissions if 'group_permissions' in include_set: json_api_include(document, WorksheetPermissionSchema(), worksheet['group_permissions']) return document
def _create_bundles(): """ Bulk create bundles. |worksheet_uuid| - The parent worksheet of the bundle, add to this worksheet if not detached or shadowing another bundle. Also used to inherit permissions. |shadow| - the uuid of the bundle to shadow |detached| - True ('1') if should not add new bundle to any worksheet, or False ('0') otherwise. Default is False. |wait_for_upload| - True ('1') if the bundle state should be initialized to UPLOADING regardless of the bundle type, or False ('0') otherwise. This prevents run bundles that are being copied from another instance from being run by the BundleManager. Default is False. """ worksheet_uuid = request.query.get('worksheet') shadow_parent_uuid = request.query.get('shadow') detached = query_get_bool('detached', default=False) if worksheet_uuid is None: abort( httplib.BAD_REQUEST, "Parent worksheet id must be specified as" "'worksheet' query parameter") # Deserialize bundle fields bundles = BundleSchema( strict=True, many=True, dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS, ).load(request.json).data # Check for all necessary permissions worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False) check_worksheet_has_all_permission(local.model, request.user, worksheet) worksheet_util.check_worksheet_not_frozen(worksheet) request.user.check_quota(need_time=True, need_disk=True) created_uuids = [] for bundle in bundles: # Prep bundle info for saving into database # Unfortunately cannot use the `construct` methods because they don't # provide a uniform interface for constructing bundles for all types # Hopefully this can all be unified after REST migration is complete bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid()) created_uuids.append(bundle_uuid) bundle_class = get_bundle_subclass(bundle['bundle_type']) bundle['owner_id'] = request.user.user_id bundle['state'] = ( State.UPLOADING if issubclass(bundle_class, UploadedBundle) or query_get_bool('wait_for_upload', False) else State.CREATED) bundle.setdefault('metadata', {})['created'] = int(time.time()) for dep in bundle.setdefault('dependencies', []): dep['child_uuid'] = bundle_uuid # Create bundle object bundle = bundle_class(bundle, strict=False) # Save bundle into model local.model.save_bundle(bundle) # Inherit worksheet permissions group_permissions = local.model.get_group_worksheet_permissions( request.user.user_id, worksheet_uuid) set_bundle_permissions([{ 'object_uuid': bundle_uuid, 'group_uuid': p['group_uuid'], 'permission': p['permission'], } for p in group_permissions]) # Add as item to worksheet if not detached: if shadow_parent_uuid is None: local.model.add_worksheet_item( worksheet_uuid, worksheet_util.bundle_item(bundle_uuid)) else: local.model.add_shadow_worksheet_items(shadow_parent_uuid, bundle_uuid) # Get created bundles bundles_dict = get_bundle_infos(created_uuids) # Return bundles in original order bundles = [bundles_dict[uuid] for uuid in created_uuids] return BundleSchema(many=True).dump(bundles).data