def replace_items(worksheet_uuid):
"""
Replace worksheet items with 'ids' with new 'items'.
The new items will be inserted after the after_sort_key
"""
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
worksheet_util.check_worksheet_not_frozen(worksheet)
ids = request.json.get('ids', [])
item_type = request.json.get('item_type', 'markup')
after_sort_key = request.json.get('after_sort_key')
# Default to process only markup items.
if item_type == "markup":
items = [
worksheet_util.markup_item(item)
for item in request.json.get('items', [])
]
elif item_type == "bundle":
items = [
worksheet_util.bundle_item(item)
for item in request.json.get('items', [])
]
elif item_type == "directive":
# Note: for directives, the item should not include the preceding "%" symbol
items = [
worksheet_util.directive_item(item)
for item in request.json.get('items', [])
]
local.model.add_worksheet_items(worksheet_uuid, items, after_sort_key, ids)
def interpret_search(query):
"""
Input: specification of a bundle search query.
Output: worksheet items based on the result of issuing the search query.
"""
# Perform search
keywords = rest_util.resolve_owner_in_keywords(query['keywords'])
bundle_uuids = local.model.search_bundle_uuids(request.user.user_id,
keywords)
# Single number, just print it out...
if not isinstance(bundle_uuids, list):
return interpret_items(query['schemas'],
[markup_item(str(bundle_uuids))])
# Set display
items = [directive_item(('display', ) + tuple(query['display']))]
# Show bundles
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
items.append(bundle_item(bundle_infos[bundle_uuid]))
# Finally, interpret the items
return interpret_items(query['schemas'], items)
def expand_raw_item(raw_item):
"""
Raw items that include searches must be expanded into more raw items.
Input: Raw item.
Output: Array of raw items. If raw item does not need expanding,
this returns an 1-length array that contains original raw item,
otherwise it contains the search result. You do not need to call
resolve_items_into_infos on the returned raw_items.
"""
(bundle_info, subworksheet_info, value_obj, item_type, id,
sort_key) = raw_item
is_search = item_type == TYPE_DIRECTIVE and get_command(
value_obj) == 'search'
is_wsearch = item_type == TYPE_DIRECTIVE and get_command(
value_obj) == 'wsearch'
if is_search or is_wsearch:
command = get_command(value_obj)
keywords = value_obj[1:]
raw_items = []
if is_search:
keywords = rest_util.resolve_owner_in_keywords(keywords)
search_result = local.model.search_bundles(request.user.user_id,
keywords)
if search_result['is_aggregate']:
# Add None's for the 'id' and 'sort_key' of the tuple, since these
# items are not really worksheet items.
raw_items.append(
markup_item(str(search_result['result'])) + (None, None))
else:
bundle_uuids = search_result['result']
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
# Since bundle UUID's are queried first, we can't assume a UUID exists in
# the subsequent bundle info query.
if bundle_uuid in bundle_infos:
raw_items.append(
bundle_item(bundle_infos[bundle_uuid]) +
(None, None))
elif is_wsearch:
worksheet_infos = search_worksheets(keywords)
for worksheet_info in worksheet_infos:
raw_items.append(
subworksheet_item(worksheet_info) + (None, None))
return raw_items
else:
return [raw_item]
def replace_items(worksheet_uuid):
"""
Replace worksheet items with 'ids' with new 'items'.
The new items will be inserted after the after_sort_key
"""
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
ids = request.json.get('ids', [])
after_sort_key = request.json.get('after_sort_key')
# Default to process only markup items.
items = [
worksheet_util.markup_item(item)
for item in request.json.get('items', [])
]
local.model.add_worksheet_items(worksheet_uuid, items, after_sort_key, ids)
def expand_raw_item(raw_item):
"""
Raw items that include searches must be expanded into more raw items.
Input: Raw item.
Output: Array of raw items. If raw item does not need expanding,
this returns an 1-length array that contains original raw item,
otherwise it contains the search result. You do not need to call
resolve_items_into_infos on the returned raw_items.
"""
(bundle_info, subworksheet_info, value_obj, item_type) = raw_item
is_search = item_type == TYPE_DIRECTIVE and get_command(
value_obj) == 'search'
is_wsearch = item_type == TYPE_DIRECTIVE and get_command(
value_obj) == 'wsearch'
if is_search or is_wsearch:
command = get_command(value_obj)
keywords = value_obj[1:]
raw_items = []
if is_search:
keywords = rest_util.resolve_owner_in_keywords(keywords)
search_result = local.model.search_bundle_uuids(
request.user.user_id, keywords)
if not isinstance(search_result, list):
raw_items.append(markup_item(str(search_result)))
else:
bundle_uuids = search_result
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
raw_items.append(bundle_item(bundle_infos[bundle_uuid]))
elif is_wsearch:
worksheet_infos = search_worksheets(keywords)
for worksheet_info in worksheet_infos:
raw_items.append(subworksheet_item(worksheet_info))
return raw_items
else:
return [raw_item]
def perform_search_query(value_obj):
"""
Perform a search query and return the resulting raw items.
Input: directive that is tokenized by formatting.string_to_tokens(),
such as formatting.string_to_tokens("search 0x .limit=100")
"""
command = get_command(value_obj)
is_search = command == 'search'
is_wsearch = command == 'wsearch'
if is_search or is_wsearch:
keywords = value_obj[1:]
raw_items = []
if is_search:
keywords = rest_util.resolve_owner_in_keywords(keywords)
search_result = local.model.search_bundles(request.user.user_id,
keywords)
if search_result['is_aggregate']:
# Add None's for the 'id' and 'sort_key' of the tuple, since these
# items are not really worksheet items.
raw_items.append(
markup_item(str(search_result['result'])) + (None, None))
else:
bundle_uuids = search_result['result']
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
# Since bundle UUID's are queried first, we can't assume a UUID exists in
# the subsequent bundle info query.
if bundle_uuid in bundle_infos:
raw_items.append(
bundle_item(bundle_infos[bundle_uuid]) +
(None, None))
elif is_wsearch:
worksheet_infos = search_worksheets(keywords)
for worksheet_info in worksheet_infos:
raw_items.append(
subworksheet_item(worksheet_info) + (None, None))
return raw_items
else:
# Not a search query
return []
def expand_raw_item(raw_item):
"""
Raw items that include searches must be expanded into more raw items.
Input: Raw item.
Output: Array of raw items. If raw item does not need expanding,
this returns an 1-length array that contains original raw item,
otherwise it contains the search result. You do not need to call
resolve_items_into_infos on the returned raw_items.
"""
(bundle_info, subworksheet_info, value_obj, item_type) = raw_item
is_search = item_type == TYPE_DIRECTIVE and get_command(value_obj) == 'search'
is_wsearch = item_type == TYPE_DIRECTIVE and get_command(value_obj) == 'wsearch'
if is_search or is_wsearch:
command = get_command(value_obj)
keywords = value_obj[1:]
raw_items = []
if is_search:
keywords = rest_util.resolve_owner_in_keywords(keywords)
search_result = local.model.search_bundles(request.user.user_id, keywords)
if search_result['is_aggregate']:
raw_items.append(markup_item(str(search_result['result'])))
else:
bundle_uuids = search_result['result']
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
raw_items.append(bundle_item(bundle_infos[bundle_uuid]))
elif is_wsearch:
worksheet_infos = search_worksheets(keywords)
for worksheet_info in worksheet_infos:
raw_items.append(subworksheet_item(worksheet_info))
return raw_items
else:
return [raw_item]
def mimic(self, old_inputs, old_output, new_inputs, new_output_name, worksheet_uuid, depth, shadow, dry_run):
'''
old_inputs: list of bundle uuids
old_output: bundle uuid that we produced
new_inputs: list of bundle uuids that are analogous to old_inputs
new_output_name: name of the bundle to create to be analogous to old_output (possibly None)
worksheet_uuid: add newly created bundles to this worksheet
depth: how far to do a BFS up
shadow: whether to add the new inputs right after all occurrences of the old inputs in worksheets.
'''
#print 'old_inputs: %s, new_inputs: %s, old_output: %s, new_output_name: %s' % (old_inputs, new_inputs, old_output, new_output_name)
# Return the worksheet items that occur right before the given bundle_uuid
worksheet_cache = {} # worksheet_uuid => items
def add_with_prelude_items(old_bundle_uuid, new_bundle_uuid):
'''
Add new_bundle_uuid to the current worksheet (along with the occurrences on the worksheet).
Also add the prelude (items that occur right before old_bundle_uuid on a worksheet).
Note that new_bundle_uuid might get multiple times.
'''
host_worksheet_uuids = self.model.get_host_worksheet_uuids([old_bundle_uuid])[old_bundle_uuid]
if len(host_worksheet_uuids) > 0:
# Choose a single worksheet.
if worksheet_uuid in host_worksheet_uuids:
# If current worksheet is one of them, favor that one.
host_worksheet_uuid = worksheet_uuid
else:
# Choose an arbitrary one
host_worksheet_uuid = host_worksheet_uuids[0]
if host_worksheet_uuid in worksheet_cache:
worksheet_info = worksheet_cache[host_worksheet_uuid]
else:
worksheet_info = worksheet_cache[host_worksheet_uuid] = \
self.get_worksheet_info(host_worksheet_uuid, fetch_items=True)
# Look for items that appear right before the old_bundle_uuid
collect_items = []
for item in worksheet_info['items']:
(bundle_info, subworkheet_info, value_obj, type) = item
if type == worksheet_util.TYPE_BUNDLE and bundle_info['uuid'] == old_bundle_uuid:
# Ended in the target old_bundle_uuid, flush the prelude gathered so far.
for item2 in collect_items:
self.add_worksheet_item(worksheet_uuid, worksheet_util.convert_item_to_db(item2))
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(new_bundle_uuid))
collect_items = []
elif type == worksheet_util.TYPE_MARKUP and value_obj == '':
collect_items = []
elif type == worksheet_util.TYPE_BUNDLE:
collect_items = []
else:
collect_items.append(item)
# Build the graph (get all the infos).
# If old_output is given, look at ancestors of old_output until we
# reached some depth. If it's not given, we first get all the
# descendants first, and then get their ancestors.
infos = {} # uuid -> bundle info
if old_output:
bundle_uuids = [old_output]
else:
bundle_uuids = self.model.get_self_and_descendants(old_inputs, depth=depth)
all_bundle_uuids = list(bundle_uuids) # should be infos.keys() in order
for _ in range(depth):
new_bundle_uuids = []
for bundle_uuid in bundle_uuids:
if bundle_uuid in infos: continue # Already visited
info = infos[bundle_uuid] = self.get_bundle_info(bundle_uuid)
for dep in info['dependencies']:
parent_uuid = dep['parent_uuid']
if parent_uuid not in infos:
new_bundle_uuids.append(parent_uuid)
all_bundle_uuids = new_bundle_uuids + all_bundle_uuids
bundle_uuids = new_bundle_uuids
# Make sure we have read access to all the bundles involved here.
# TODO: need to double check that this is right.
check_has_read_permission_on_bundles(self.model, self._current_user(), list(infos.keys()))
# Now go recursively create the bundles.
old_to_new = {} # old_uuid -> new_uuid
downstream = set() # old_uuid -> whether we're downstream of an input (and actually needs to be mapped onto a new uuid)
plan = [] # sequence of (old, new) bundle infos to make
for old, new in zip(old_inputs, new_inputs):
old_to_new[old] = new
downstream.add(old)
# Return corresponding new_bundle_uuid
def recurse(old_bundle_uuid):
if old_bundle_uuid in old_to_new:
#print old_bundle_uuid, 'cached'
return old_to_new[old_bundle_uuid]
# Don't have any more information (because we probably hit the maximum depth)
if old_bundle_uuid not in infos:
#print old_bundle_uuid, 'no information'
return old_bundle_uuid
# Get information about the old bundle.
info = infos[old_bundle_uuid]
new_dependencies = [{
'parent_uuid': recurse(dep['parent_uuid']),
'parent_path': dep['parent_path'],
'child_uuid': dep['child_uuid'], # This is just a placeholder to do the equality test
'child_path': dep['child_path']
} for dep in info['dependencies']]
# We're downstream, so need to make a new bundle
if any(dep['parent_uuid'] in downstream for dep in info['dependencies']):
# Now create a new bundle that mimics the old bundle.
# Only change the name if the output name is supplied.
old_bundle_name = info['metadata']['name']
new_info = copy.deepcopy(info)
new_metadata = new_info['metadata']
if new_output_name:
if old_bundle_uuid == old_output:
new_metadata['name'] = new_output_name
else:
# Just make up a name heuristically
new_metadata['name'] = new_output_name + '-' + info['metadata']['name']
# Remove all the automatically generated keys
cls = get_bundle_subclass(new_info['bundle_type'])
for spec in cls.METADATA_SPECS:
if spec.generated and spec.key in new_metadata:
new_metadata.pop(spec.key)
# Set the targets
targets = [(dep['child_path'], (dep['parent_uuid'], dep['parent_path'])) for dep in new_dependencies]
if dry_run:
new_bundle_uuid = None
else:
new_bundle_uuid = self.derive_bundle(new_info['bundle_type'], \
targets, new_info['command'], new_metadata, None)
# Add to worksheet
if shadow:
self.model.add_shadow_worksheet_items(old_bundle_uuid, new_bundle_uuid)
else:
# Copy the markup and directives right before the old
add_with_prelude_items(old_bundle_uuid, new_bundle_uuid)
new_info['uuid'] = new_bundle_uuid
plan.append((info, new_info))
downstream.add(old_bundle_uuid)
else:
new_bundle_uuid = old_bundle_uuid
old_to_new[old_bundle_uuid] = new_bundle_uuid # Cache it
return new_bundle_uuid
# Put initial newline to delimit things
if not dry_run and not shadow:
self.model.add_worksheet_item(worksheet_uuid, worksheet_util.markup_item(''))
if old_output:
recurse(old_output)
else:
# Don't have a particular output we're targetting, so just create
# new versions of all the uuids.
for uuid in all_bundle_uuids: recurse(uuid)
return plan
def newline():
self.model.add_worksheet_item(worksheet_uuid, worksheet_util.markup_item(''))
def _create_bundles():
"""
Bulk create bundles.
Query parameters:
- `worksheet`: UUID of the parent worksheet of the new bundle, add to
this worksheet if not detached or shadowing another bundle. The new
bundle also inherits permissions from this worksheet.
- `shadow`: UUID of the bundle to "shadow" (the new bundle will be added
as an item immediately after this bundle in its parent worksheet).
- `detached`: 1 if should not add new bundle to any worksheet,
so the bundle does not have a hosted worksheet.
This is set to 1, for example, if the user is uploading their avatar as a bundle.
or 0 otherwise. Default is 0.
- `wait_for_upload`: 1 if the bundle state should be initialized to
"uploading" regardless of the bundle type, or 0 otherwise. Used when
copying bundles from another CodaLab instance, this prevents these new
bundles from being executed by the BundleManager. Default is 0.
"""
worksheet_uuid = request.query.get('worksheet')
shadow_parent_uuid = request.query.get('shadow')
after_sort_key = request.query.get('after_sort_key')
detached = query_get_bool('detached', default=False)
after_image = query_get_bool('after_image', default=False)
if not detached and worksheet_uuid is None:
abort(
http.client.BAD_REQUEST,
"Parent worksheet id must be specified as" "'worksheet' query parameter",
)
# Deserialize bundle fields
bundles = (
BundleSchema(strict=True, many=True, dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS)
.load(request.json)
.data
)
# Check for all necessary permissions
if not detached:
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
worksheet_util.check_worksheet_not_frozen(worksheet)
request.user.check_quota(need_time=True, need_disk=True)
created_uuids = []
for bundle in bundles:
# Prep bundle info for saving into database
# Unfortunately cannot use the `construct` methods because they don't
# provide a uniform interface for constructing bundles for all types
# Hopefully this can all be unified after REST migration is complete
bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
created_uuids.append(bundle_uuid)
bundle_class = get_bundle_subclass(bundle['bundle_type'])
bundle['owner_id'] = request.user.user_id
metadata = bundle.get("metadata", {})
if metadata.get("link_url"):
bundle['state'] = State.READY
elif issubclass(bundle_class, UploadedBundle) or query_get_bool('wait_for_upload', False):
bundle['state'] = State.UPLOADING
else:
bundle['state'] = State.CREATED
if not detached:
bundle['is_anonymous'] = worksheet.is_anonymous # inherit worksheet anonymity
else:
bundle['is_anonymous'] = False
bundle.setdefault('metadata', {})['created'] = int(time.time())
for dep in bundle.setdefault('dependencies', []):
dep['child_uuid'] = bundle_uuid
# Create bundle object
bundle = bundle_class(bundle, strict=False)
# Save bundle into model
local.model.save_bundle(bundle)
if not detached:
# Inherit worksheet permissions; else, only the user will have all permissions on the bundle
group_permissions = local.model.get_group_worksheet_permissions(
request.user.user_id, worksheet_uuid
)
set_bundle_permissions(
[
{
'object_uuid': bundle_uuid,
'group_uuid': p['group_uuid'],
'permission': p['permission'],
}
for p in group_permissions
]
)
# Add as item to worksheet
if not detached:
if shadow_parent_uuid is None:
items = [worksheet_util.bundle_item(bundle_uuid)]
# Add a blank line after the image block in the worksheet source to ensure it is a separate block
if after_image:
items.insert(0, worksheet_util.markup_item(''))
local.model.add_worksheet_items(worksheet_uuid, items, after_sort_key)
else:
local.model.add_shadow_worksheet_items(shadow_parent_uuid, bundle_uuid)
# Get created bundles
bundles_dict = get_bundle_infos(created_uuids)
# Return bundles in original order
# Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted
# right after being created.
bundles = [bundles_dict[uuid] for uuid in created_uuids if uuid in bundles_dict]
return BundleSchema(many=True).dump(bundles).data
