def replace_items(worksheet_uuid):
"""
Replace worksheet items with 'ids' with new 'items'.
The new items will be inserted after the after_sort_key
"""
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
worksheet_util.check_worksheet_not_frozen(worksheet)
ids = request.json.get('ids', [])
item_type = request.json.get('item_type', 'markup')
after_sort_key = request.json.get('after_sort_key')
# Default to process only markup items.
if item_type == "markup":
items = [
worksheet_util.markup_item(item)
for item in request.json.get('items', [])
]
elif item_type == "bundle":
items = [
worksheet_util.bundle_item(item)
for item in request.json.get('items', [])
]
elif item_type == "directive":
# Note: for directives, the item should not include the preceding "%" symbol
items = [
worksheet_util.directive_item(item)
for item in request.json.get('items', [])
]
local.model.add_worksheet_items(worksheet_uuid, items, after_sort_key, ids)
def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks):
bundle_type = info['bundle_type']
if 'uuid' in info:
existing = True
construct_args = self.bundle_info_to_construct_args(info)
else:
existing = False
construct_args = {'metadata': info['metadata']}
metadata = construct_args['metadata']
message = 'Invalid upload bundle_type: %s' % (bundle_type,)
if not existing:
precondition(bundle_type in UPLOADED_TYPES, message)
bundle_subclass = get_bundle_subclass(bundle_type)
if not existing:
self.validate_user_metadata(bundle_subclass, metadata)
# Upload the given path and record additional metadata from the upload.
if path:
(data_hash, bundle_store_metadata) = self.bundle_store.upload(path, follow_symlinks=follow_symlinks)
metadata.update(bundle_store_metadata)
precondition(construct_args.get('data_hash', data_hash) == data_hash, \
'Provided data_hash doesn\'t match: %s versus %s' % (construct_args.get('data_hash'), data_hash))
construct_args['data_hash'] = data_hash
# Set the owner
construct_args['owner_id'] = self._current_user_id()
bundle = bundle_subclass.construct(**construct_args)
self.model.save_bundle(bundle)
if worksheet_uuid:
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle.uuid))
# TODO: don't fail if don't have permissions
return bundle.uuid
def interpret_search(query):
"""
Input: specification of a bundle search query.
Output: worksheet items based on the result of issuing the search query.
"""
# Perform search
keywords = rest_util.resolve_owner_in_keywords(query['keywords'])
bundle_uuids = local.model.search_bundle_uuids(request.user.user_id,
keywords)
# Single number, just print it out...
if not isinstance(bundle_uuids, list):
return interpret_items(query['schemas'],
[markup_item(str(bundle_uuids))])
# Set display
items = [directive_item(('display', ) + tuple(query['display']))]
# Show bundles
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
items.append(bundle_item(bundle_infos[bundle_uuid]))
# Finally, interpret the items
return interpret_items(query['schemas'], items)
def derive_bundle(self, bundle_type, targets, command, metadata, worksheet_uuid):
'''
For both make and run bundles.
Add the resulting bundle to the given worksheet_uuid.
'''
check_worksheet_has_all_permission(self.model, self._current_user(), self.model.get_worksheet(worksheet_uuid, fetch_items=False))
bundle_uuid = self._derive_bundle(bundle_type, targets, command, metadata, worksheet_uuid)
# Add to worksheet
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
return bundle_uuid
def expand_raw_item(raw_item):
"""
Raw items that include searches must be expanded into more raw items.
Input: Raw item.
Output: Array of raw items. If raw item does not need expanding,
this returns an 1-length array that contains original raw item,
otherwise it contains the search result. You do not need to call
resolve_items_into_infos on the returned raw_items.
"""
(bundle_info, subworksheet_info, value_obj, item_type, id,
sort_key) = raw_item
is_search = item_type == TYPE_DIRECTIVE and get_command(
value_obj) == 'search'
is_wsearch = item_type == TYPE_DIRECTIVE and get_command(
value_obj) == 'wsearch'
if is_search or is_wsearch:
command = get_command(value_obj)
keywords = value_obj[1:]
raw_items = []
if is_search:
keywords = rest_util.resolve_owner_in_keywords(keywords)
search_result = local.model.search_bundles(request.user.user_id,
keywords)
if search_result['is_aggregate']:
# Add None's for the 'id' and 'sort_key' of the tuple, since these
# items are not really worksheet items.
raw_items.append(
markup_item(str(search_result['result'])) + (None, None))
else:
bundle_uuids = search_result['result']
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
# Since bundle UUID's are queried first, we can't assume a UUID exists in
# the subsequent bundle info query.
if bundle_uuid in bundle_infos:
raw_items.append(
bundle_item(bundle_infos[bundle_uuid]) +
(None, None))
elif is_wsearch:
worksheet_infos = search_worksheets(keywords)
for worksheet_info in worksheet_infos:
raw_items.append(
subworksheet_item(worksheet_info) + (None, None))
return raw_items
else:
return [raw_item]
def derive_bundle(self, bundle_type, targets, command, metadata, worksheet_uuid):
'''
For both make and run bundles.
Add the resulting bundle to the given worksheet_uuid (optional).
'''
bundle_subclass = get_bundle_subclass(bundle_type)
self.validate_user_metadata(bundle_subclass, metadata)
owner_id = self._current_user_id()
bundle = bundle_subclass.construct(targets=targets, command=command, metadata=metadata, owner_id=owner_id)
self.model.save_bundle(bundle)
if worksheet_uuid:
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle.uuid))
# TODO: don't fail if don't have permissions
return bundle.uuid
def perform_search_query(value_obj):
"""
Perform a search query and return the resulting raw items.
Input: directive that is tokenized by formatting.string_to_tokens(),
such as formatting.string_to_tokens("search 0x .limit=100")
"""
command = get_command(value_obj)
is_search = command == 'search'
is_wsearch = command == 'wsearch'
if is_search or is_wsearch:
keywords = value_obj[1:]
raw_items = []
if is_search:
keywords = rest_util.resolve_owner_in_keywords(keywords)
search_result = local.model.search_bundles(request.user.user_id,
keywords)
if search_result['is_aggregate']:
# Add None's for the 'id' and 'sort_key' of the tuple, since these
# items are not really worksheet items.
raw_items.append(
markup_item(str(search_result['result'])) + (None, None))
else:
bundle_uuids = search_result['result']
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
# Since bundle UUID's are queried first, we can't assume a UUID exists in
# the subsequent bundle info query.
if bundle_uuid in bundle_infos:
raw_items.append(
bundle_item(bundle_infos[bundle_uuid]) +
(None, None))
elif is_wsearch:
worksheet_infos = search_worksheets(keywords)
for worksheet_info in worksheet_infos:
raw_items.append(
subworksheet_item(worksheet_info) + (None, None))
return raw_items
else:
# Not a search query
return []
def expand_raw_item(raw_item):
"""
Raw items that include searches must be expanded into more raw items.
Input: Raw item.
Output: Array of raw items. If raw item does not need expanding,
this returns an 1-length array that contains original raw item,
otherwise it contains the search result. You do not need to call
resolve_items_into_infos on the returned raw_items.
"""
(bundle_info, subworksheet_info, value_obj, item_type) = raw_item
is_search = item_type == TYPE_DIRECTIVE and get_command(
value_obj) == 'search'
is_wsearch = item_type == TYPE_DIRECTIVE and get_command(
value_obj) == 'wsearch'
if is_search or is_wsearch:
command = get_command(value_obj)
keywords = value_obj[1:]
raw_items = []
if is_search:
keywords = rest_util.resolve_owner_in_keywords(keywords)
search_result = local.model.search_bundle_uuids(
request.user.user_id, keywords)
if not isinstance(search_result, list):
raw_items.append(markup_item(str(search_result)))
else:
bundle_uuids = search_result
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
raw_items.append(bundle_item(bundle_infos[bundle_uuid]))
elif is_wsearch:
worksheet_infos = search_worksheets(keywords)
for worksheet_info in worksheet_infos:
raw_items.append(subworksheet_item(worksheet_info))
return raw_items
else:
return [raw_item]
def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks, exclude_patterns, add_to_worksheet):
check_worksheet_has_all_permission(self.model, self._current_user(), self.model.get_worksheet(worksheet_uuid, fetch_items=False))
bundle_type = info['bundle_type']
if 'uuid' in info:
existing = True
construct_args = self.bundle_info_to_construct_args(info)
else:
existing = False
construct_args = {'metadata': info['metadata']}
metadata = construct_args['metadata']
message = 'Invalid upload bundle_type: %s' % (bundle_type,)
if not existing:
precondition(bundle_type in UPLOADED_TYPES, message)
bundle_subclass = get_bundle_subclass(bundle_type)
if not existing:
self.validate_user_metadata(bundle_subclass, metadata)
# Upload the given path and record additional metadata from the upload.
if path:
(data_hash, bundle_store_metadata) = self.bundle_store.upload(path, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns)
metadata.update(bundle_store_metadata)
if construct_args.get('data_hash', data_hash) != data_hash:
print >>sys.stderr, 'ERROR: provided data_hash doesn\'t match: %s versus %s' % (construct_args.get('data_hash'), data_hash)
construct_args['data_hash'] = data_hash
# Set the owner
construct_args['owner_id'] = self._current_user_id()
bundle = bundle_subclass.construct(**construct_args)
self.model.save_bundle(bundle)
# Inherit properties of worksheet
self._bundle_inherit_workheet_permissions(bundle.uuid, worksheet_uuid)
# Add to worksheet
if add_to_worksheet:
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle.uuid))
return bundle.uuid
def expand_raw_item(raw_item):
"""
Raw items that include searches must be expanded into more raw items.
Input: Raw item.
Output: Array of raw items. If raw item does not need expanding,
this returns an 1-length array that contains original raw item,
otherwise it contains the search result. You do not need to call
resolve_items_into_infos on the returned raw_items.
"""
(bundle_info, subworksheet_info, value_obj, item_type) = raw_item
is_search = item_type == TYPE_DIRECTIVE and get_command(value_obj) == 'search'
is_wsearch = item_type == TYPE_DIRECTIVE and get_command(value_obj) == 'wsearch'
if is_search or is_wsearch:
command = get_command(value_obj)
keywords = value_obj[1:]
raw_items = []
if is_search:
keywords = rest_util.resolve_owner_in_keywords(keywords)
search_result = local.model.search_bundles(request.user.user_id, keywords)
if search_result['is_aggregate']:
raw_items.append(markup_item(str(search_result['result'])))
else:
bundle_uuids = search_result['result']
bundle_infos = rest_util.get_bundle_infos(bundle_uuids)
for bundle_uuid in bundle_uuids:
raw_items.append(bundle_item(bundle_infos[bundle_uuid]))
elif is_wsearch:
worksheet_infos = search_worksheets(keywords)
for worksheet_info in worksheet_infos:
raw_items.append(subworksheet_item(worksheet_info))
return raw_items
else:
return [raw_item]
def add_with_prelude_items(old_bundle_uuid, new_bundle_uuid):
'''
Add new_bundle_uuid to the current worksheet (along with the occurrences on the worksheet).
Also add the prelude (items that occur right before old_bundle_uuid on a worksheet).
Note that new_bundle_uuid might get multiple times.
'''
host_worksheet_uuids = self.model.get_host_worksheet_uuids([old_bundle_uuid])[old_bundle_uuid]
if len(host_worksheet_uuids) > 0:
# Choose a single worksheet.
if worksheet_uuid in host_worksheet_uuids:
# If current worksheet is one of them, favor that one.
host_worksheet_uuid = worksheet_uuid
else:
# Choose an arbitrary one
host_worksheet_uuid = host_worksheet_uuids[0]
if host_worksheet_uuid in worksheet_cache:
worksheet_info = worksheet_cache[host_worksheet_uuid]
else:
worksheet_info = worksheet_cache[host_worksheet_uuid] = \
self.get_worksheet_info(host_worksheet_uuid, fetch_items=True)
# Look for items that appear right before the old_bundle_uuid
collect_items = []
for item in worksheet_info['items']:
(bundle_info, subworkheet_info, value_obj, type) = item
if type == worksheet_util.TYPE_BUNDLE and bundle_info['uuid'] == old_bundle_uuid:
# Ended in the target old_bundle_uuid, flush the prelude gathered so far.
for item2 in collect_items:
self.add_worksheet_item(worksheet_uuid, worksheet_util.convert_item_to_db(item2))
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(new_bundle_uuid))
collect_items = []
elif type == worksheet_util.TYPE_MARKUP and value_obj == '':
collect_items = []
elif type == worksheet_util.TYPE_BUNDLE:
collect_items = []
else:
collect_items.append(item)
def replace_items(worksheet_uuid):
"""
Replace worksheet items with 'ids' with new 'items'.
The new items will be inserted after the after_sort_key
"""
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
ids = request.json.get('ids', [])
item_type = request.json.get('item_type', 'markup')
after_sort_key = request.json.get('after_sort_key')
# Default to process only markup items.
if item_type == "markup":
items = [
worksheet_util.markup_item(item)
for item in request.json.get('items', [])
]
elif item_type == "bundle":
items = [
worksheet_util.bundle_item(item)
for item in request.json.get('items', [])
]
local.model.add_worksheet_items(worksheet_uuid, items, after_sort_key, ids)
def _create_bundles():
"""
Bulk create bundles.
|worksheet_uuid| - The parent worksheet of the bundle, add to this worksheet
if not detached or shadowing another bundle. Also used
to inherit permissions.
|shadow| - the uuid of the bundle to shadow
|detached| - True ('1') if should not add new bundle to any worksheet,
or False ('0') otherwise. Default is False.
|wait_for_upload| - True ('1') if the bundle state should be initialized to
UPLOADING regardless of the bundle type, or False ('0')
otherwise. This prevents run bundles that are being
copied from another instance from being run by the
BundleManager. Default is False.
"""
worksheet_uuid = request.query.get('worksheet')
shadow_parent_uuid = request.query.get('shadow')
detached = query_get_bool('detached', default=False)
if worksheet_uuid is None:
abort(
httplib.BAD_REQUEST, "Parent worksheet id must be specified as"
"'worksheet' query parameter")
# Deserialize bundle fields
bundles = BundleSchema(
strict=True,
many=True,
dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS,
).load(request.json).data
# Check for all necessary permissions
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
worksheet_util.check_worksheet_not_frozen(worksheet)
request.user.check_quota(need_time=True, need_disk=True)
created_uuids = []
for bundle in bundles:
# Prep bundle info for saving into database
# Unfortunately cannot use the `construct` methods because they don't
# provide a uniform interface for constructing bundles for all types
# Hopefully this can all be unified after REST migration is complete
bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
created_uuids.append(bundle_uuid)
bundle_class = get_bundle_subclass(bundle['bundle_type'])
bundle['owner_id'] = request.user.user_id
bundle['state'] = (
State.UPLOADING if issubclass(bundle_class, UploadedBundle)
or query_get_bool('wait_for_upload', False) else State.CREATED)
bundle.setdefault('metadata', {})['created'] = int(time.time())
for dep in bundle.setdefault('dependencies', []):
dep['child_uuid'] = bundle_uuid
# Create bundle object
bundle = bundle_class(bundle, strict=False)
# Save bundle into model
local.model.save_bundle(bundle)
# Inherit worksheet permissions
group_permissions = local.model.get_group_worksheet_permissions(
request.user.user_id, worksheet_uuid)
set_bundle_permissions([{
'object_uuid': bundle_uuid,
'group_uuid': p['group_uuid'],
'permission': p['permission'],
} for p in group_permissions])
# Add as item to worksheet
if not detached:
if shadow_parent_uuid is None:
local.model.add_worksheet_item(
worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
else:
local.model.add_shadow_worksheet_items(shadow_parent_uuid,
bundle_uuid)
# Get created bundles
bundles_dict = get_bundle_infos(created_uuids)
# Return bundles in original order
bundles = [bundles_dict[uuid] for uuid in created_uuids]
return BundleSchema(many=True).dump(bundles).data
def mimic(self, old_inputs, old_output, new_inputs, new_output_name, worksheet_uuid, depth, shadow, dry_run):
'''
old_inputs: list of bundle uuids
old_output: bundle uuid that we produced
new_inputs: list of bundle uuids that are analogous to old_inputs
new_output_name: name of the bundle to create to be analogous to old_output (possibly None)
worksheet_uuid: add newly created bundles to this worksheet
depth: how far to do a BFS up from old_output.
shadow: whether to add the new inputs right after all occurrences of the old inputs in worksheets.
'''
check_worksheet_has_all_permission(self.model, self._current_user(), self.model.get_worksheet(worksheet_uuid, fetch_items=False))
#print 'old_inputs: %s, new_inputs: %s, old_output: %s, new_output_name: %s' % (old_inputs, new_inputs, old_output, new_output_name)
# Build the graph (get all the infos).
# If old_output is given, look at ancestors of old_output until we
# reached some depth. If it's not given, we first get all the
# descendants first, and then get their ancestors.
infos = {} # uuid -> bundle info
if old_output:
bundle_uuids = [old_output]
else:
bundle_uuids = self.model.get_self_and_descendants(old_inputs, depth=depth)
all_bundle_uuids = list(bundle_uuids) # should be infos.keys() in order
for _ in range(depth):
new_bundle_uuids = []
for bundle_uuid in bundle_uuids:
if bundle_uuid in infos: continue # Already visited
info = infos[bundle_uuid] = self.get_bundle_info(bundle_uuid)
for dep in info['dependencies']:
parent_uuid = dep['parent_uuid']
if parent_uuid not in infos:
new_bundle_uuids.append(parent_uuid)
all_bundle_uuids = new_bundle_uuids + all_bundle_uuids
bundle_uuids = new_bundle_uuids
# Make sure we have read access to all the bundles involved here.
check_bundles_have_read_permission(self.model, self._current_user(), list(infos.keys()))
# Now go recursively create the bundles.
old_to_new = {} # old_uuid -> new_uuid
downstream = set() # old_uuid -> whether we're downstream of an input (and actually needs to be mapped onto a new uuid)
created_uuids = set() # set of uuids which were newly created
plan = [] # sequence of (old, new) bundle infos to make
for old, new in zip(old_inputs, new_inputs):
old_to_new[old] = new
downstream.add(old)
# Return corresponding new_bundle_uuid
def recurse(old_bundle_uuid):
if old_bundle_uuid in old_to_new:
#print old_bundle_uuid, 'cached'
return old_to_new[old_bundle_uuid]
# Don't have any more information (because we probably hit the maximum depth)
if old_bundle_uuid not in infos:
#print old_bundle_uuid, 'no information'
return old_bundle_uuid
# Get information about the old bundle.
info = infos[old_bundle_uuid]
new_dependencies = [{
'parent_uuid': recurse(dep['parent_uuid']),
'parent_path': dep['parent_path'],
'child_uuid': dep['child_uuid'], # This is just a placeholder to do the equality test
'child_path': dep['child_path']
} for dep in info['dependencies']]
# If there are no inputs or if we're downstream of any inputs, we need to make a new bundle.
lone_output = (len(old_inputs) == 0 and old_bundle_uuid == old_output)
downstream_of_inputs = any(dep['parent_uuid'] in downstream for dep in info['dependencies'])
if lone_output or downstream_of_inputs:
# Now create a new bundle that mimics the old bundle.
# Only change the name if the output name is supplied.
old_bundle_name = info['metadata']['name']
new_info = copy.deepcopy(info)
new_metadata = new_info['metadata']
if new_output_name:
if old_bundle_uuid == old_output:
new_metadata['name'] = new_output_name
else:
# Just make up a name heuristically
new_metadata['name'] = new_output_name + '-' + info['metadata']['name']
# Remove all the automatically generated keys
cls = get_bundle_subclass(new_info['bundle_type'])
for spec in cls.METADATA_SPECS:
if spec.generated and spec.key in new_metadata:
new_metadata.pop(spec.key)
# Set the targets
targets = [(dep['child_path'], (dep['parent_uuid'], dep['parent_path'])) for dep in new_dependencies]
if dry_run:
new_bundle_uuid = None
else:
if new_info['bundle_type'] not in ('make', 'run'):
raise UsageError('Can\'t mimic %s since it is not make or run' % old_bundle_uuid)
new_bundle_uuid = self._derive_bundle(new_info['bundle_type'], \
targets, new_info['command'], new_metadata, worksheet_uuid)
new_info['uuid'] = new_bundle_uuid
plan.append((info, new_info))
downstream.add(old_bundle_uuid)
created_uuids.add(new_bundle_uuid)
else:
new_bundle_uuid = old_bundle_uuid
old_to_new[old_bundle_uuid] = new_bundle_uuid # Cache it
return new_bundle_uuid
if old_output:
recurse(old_output)
else:
# Don't have a particular output we're targetting, so just create
# new versions of all the uuids.
for uuid in all_bundle_uuids:
recurse(uuid)
# Add to worksheet
if not dry_run:
if shadow:
# Add each new bundle in the "shadow" of the old_bundle (right after it).
for old_bundle_uuid, new_bundle_uuid in old_to_new.items():
if new_bundle_uuid in created_uuids: # Only add novel bundles
self.model.add_shadow_worksheet_items(old_bundle_uuid, new_bundle_uuid)
else:
def newline():
self.model.add_worksheet_item(worksheet_uuid, worksheet_util.markup_item(''))
# A prelude of a bundle on a worksheet is the set of items that occur right before it (markup, directives, etc.)
# Let W be the first worksheet containing the old_inputs[0].
# Add all items on that worksheet that appear in old_to_new along with their preludes.
# For items not on this worksheet, add them at the end (instead of making them floating).
if old_output:
anchor_uuid = old_output
elif len(old_inputs) > 0:
anchor_uuid = old_inputs[0]
host_worksheet_uuids = self.model.get_host_worksheet_uuids([anchor_uuid])[anchor_uuid]
new_bundle_uuids_added = set()
skipped = True # Whether there were items that we didn't include in the prelude (in which case we want to put '')
if len(host_worksheet_uuids) > 0:
# Choose a single worksheet.
if worksheet_uuid in host_worksheet_uuids:
# If current worksheet is one of them, favor that one.
host_worksheet_uuid = worksheet_uuid
else:
# Choose an arbitrary one (in the future, have a better way of canonicalizing).
host_worksheet_uuid = host_worksheet_uuids[0]
# Fetch the worksheet
worksheet_info = self.get_worksheet_info(host_worksheet_uuid, fetch_items=True)
prelude_items = [] # The prelude that we're building up
for item in worksheet_info['items']:
(bundle_info, subworkheet_info, value_obj, item_type) = item
just_added = False
if item_type == worksheet_util.TYPE_BUNDLE:
old_bundle_uuid = bundle_info['uuid']
if old_bundle_uuid in old_to_new:
# Flush the prelude gathered so far.
new_bundle_uuid = old_to_new[old_bundle_uuid]
if new_bundle_uuid in created_uuids: # Only add novel bundles
# Stand in for things skipped (this is important so directives have proper extent).
if skipped:
newline()
# Add prelude and items
for item2 in prelude_items:
self.add_worksheet_item(worksheet_uuid, worksheet_util.convert_item_to_db(item2))
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(new_bundle_uuid))
new_bundle_uuids_added.add(new_bundle_uuid)
just_added = True
if (item_type == worksheet_util.TYPE_MARKUP and value_obj != '') or item_type == worksheet_util.TYPE_DIRECTIVE:
prelude_items.append(item) # Include in prelude
skipped = False
else:
prelude_items = [] # Reset
skipped = not just_added
# Add the bundles that haven't been added yet
for info, new_info in plan:
new_bundle_uuid = new_info['uuid']
if new_bundle_uuid not in new_bundle_uuids_added:
if skipped:
newline()
skipped = False
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(new_bundle_uuid))
return plan
def add_worksheet_item(self, worksheet_uuid, bundle_uuid):
"""
Add bundle uuid to the given worksheet.
"""
self.client.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
def add_worksheet_item(self, worksheet_uuid, bundle_uuid):
self.client.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
def add_worksheet_item(self, worksheet_uuid, bundle_uuid):
"""
Add bundle uuid to the given worksheet.
"""
self.client.add_worksheet_item(
worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
def _create_bundles():
"""
Bulk create bundles.
Query parameters:
- `worksheet`: UUID of the parent worksheet of the new bundle, add to
this worksheet if not detached or shadowing another bundle. The new
bundle also inherits permissions from this worksheet.
- `shadow`: UUID of the bundle to "shadow" (the new bundle will be added
as an item immediately after this bundle in its parent worksheet).
- `detached`: 1 if should not add new bundle to any worksheet,
or 0 otherwise. Default is 0.
- `wait_for_upload`: 1 if the bundle state should be initialized to
"uploading" regardless of the bundle type, or 0 otherwise. Used when
copying bundles from another CodaLab instance, this prevents these new
bundles from being executed by the BundleManager. Default is 0.
"""
worksheet_uuid = request.query.get('worksheet')
shadow_parent_uuid = request.query.get('shadow')
after_sort_key = request.query.get('after_sort_key')
detached = query_get_bool('detached', default=False)
if worksheet_uuid is None:
abort(
http.client.BAD_REQUEST,
"Parent worksheet id must be specified as"
"'worksheet' query parameter",
)
# Deserialize bundle fields
bundles = (BundleSchema(strict=True,
many=True,
dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS).load(
request.json).data)
# Check for all necessary permissions
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
worksheet_util.check_worksheet_not_frozen(worksheet)
request.user.check_quota(need_time=True, need_disk=True)
created_uuids = []
for bundle in bundles:
# Prep bundle info for saving into database
# Unfortunately cannot use the `construct` methods because they don't
# provide a uniform interface for constructing bundles for all types
# Hopefully this can all be unified after REST migration is complete
bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
created_uuids.append(bundle_uuid)
bundle_class = get_bundle_subclass(bundle['bundle_type'])
bundle['owner_id'] = request.user.user_id
metadata = bundle.get("metadata", {})
if metadata.get("link_url"):
bundle['state'] = State.READY
elif issubclass(bundle_class, UploadedBundle) or query_get_bool(
'wait_for_upload', False):
bundle['state'] = State.UPLOADING
else:
bundle['state'] = State.CREATED
bundle[
'is_anonymous'] = worksheet.is_anonymous # inherit worksheet anonymity
bundle.setdefault('metadata', {})['created'] = int(time.time())
for dep in bundle.setdefault('dependencies', []):
dep['child_uuid'] = bundle_uuid
# Create bundle object
bundle = bundle_class(bundle, strict=False)
# Save bundle into model
local.model.save_bundle(bundle)
# Inherit worksheet permissions
group_permissions = local.model.get_group_worksheet_permissions(
request.user.user_id, worksheet_uuid)
set_bundle_permissions([{
'object_uuid': bundle_uuid,
'group_uuid': p['group_uuid'],
'permission': p['permission'],
} for p in group_permissions])
# Add as item to worksheet
if not detached:
if shadow_parent_uuid is None:
local.model.add_worksheet_items(
worksheet_uuid, [worksheet_util.bundle_item(bundle_uuid)],
after_sort_key)
else:
local.model.add_shadow_worksheet_items(shadow_parent_uuid,
bundle_uuid)
# Get created bundles
bundles_dict = get_bundle_infos(created_uuids)
# Return bundles in original order
# Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted
# right after being created.
bundles = [
bundles_dict[uuid] for uuid in created_uuids if uuid in bundles_dict
]
return BundleSchema(many=True).dump(bundles).data
def add_worksheet_item(self, worksheet_uuid, bundle_uuid):
self.client.add_worksheet_item(
worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
