def do_upload_command(self, argv, parser):
worksheet_uuid = self.env_model.get_current_worksheet()
help_text = 'bundle_type: [%s]' % ('|'.join(sorted(UPLOADED_TYPES)))
parser.add_argument('bundle_type', help=help_text)
parser.add_argument('path', help='path of the directory to upload')
# Add metadata arguments for UploadedBundle and all of its subclasses.
metadata_keys = set()
metadata_util.add_arguments(UploadedBundle, metadata_keys, parser)
for bundle_type in UPLOADED_TYPES:
bundle_subclass = get_bundle_subclass(bundle_type)
metadata_util.add_arguments(bundle_subclass, metadata_keys, parser)
metadata_util.add_auto_argument(parser)
args = parser.parse_args(argv)
# Check that the upload path exists.
path_util.check_isvalid(path_util.normalize(args.path), 'upload')
# Pull out the upload bundle type from the arguments and validate it.
if args.bundle_type not in UPLOADED_TYPES:
raise UsageError('Invalid bundle type %s (options: [%s])' % (
args.bundle_type, '|'.join(sorted(UPLOADED_TYPES)),
))
bundle_subclass = get_bundle_subclass(args.bundle_type)
metadata = metadata_util.request_missing_data(bundle_subclass, args)
# Type-check the bundle metadata BEFORE uploading the bundle data.
# This optimization will avoid file copies on failed bundle creations.
bundle_subclass.construct(data_hash='', metadata=metadata).validate()
print self.client.upload(args.bundle_type, args.path, metadata, worksheet_uuid)
def do_upload_command(self, argv, parser):
parser.add_argument('bundle_type', help='bundle type: [program|dataset]')
parser.add_argument('path', help='path of the directory to upload')
parser.add_argument('--name', help='name: [a-zA-Z0-9_]+)')
parser.add_argument(
'--desc',
dest='description',
help='human-readable description',
metavar='DESC',
)
parser.add_argument('--tags', help='list of search tags', nargs='+')
this_machine = platform.machine()
default_architecture = [this_machine] if this_machine else []
parser.add_argument(
'--arch',
default=default_architecture,
dest='architecture',
help='viable architectures (for programs)',
metavar='ARCH',
nargs='+',
)
args = parser.parse_args(argv)
bundle_subclass = get_bundle_subclass(args.bundle_type)
metadata = {k: getattr(args, k) for k in bundle_subclass.METADATA_TYPES}
print metadata
print self.client.upload(args.bundle_type, args.path, metadata)
def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
"""
Upload |source_file| (a stream) to |worksheet_uuid|.
"""
# Construct info for creating the bundle.
bundle_subclass = get_bundle_subclass(bundle_type) # program or data
metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.filename, 'description': 'Upload ' + source_file.filename})
info = {'bundle_type': bundle_type, 'metadata': metadata}
# Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
remote_file_uuid = self.client.open_temp_file(metadata['name'])
try:
with closing(RPCFileHandle(remote_file_uuid, self.client.proxy)) as dest:
file_util.copy(source_file.file, dest, autoflush=False, print_status='Uploading %s' % metadata['name'])
pack = False # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
if not pack and zip_util.path_is_archive(metadata['name']):
metadata['name'] = zip_util.strip_archive_ext(metadata['name'])
# Then tell the client that the uploaded file handle is there.
new_bundle_uuid = self.client.finish_upload_bundle(
[remote_file_uuid],
not pack, # unpack
info,
worksheet_uuid,
True) # add_to_worksheet
except:
self.client.finalize_file(remote_file_uuid)
raise
return new_bundle_uuid
def get_bundle_info(self, uuid):
## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
# format permission data
bundle_info['permission_str'] = permission_str(bundle_info['permission'])
# format each groups as well
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def get_bundle_info(self, uuid):
## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
bundle_info = _call_with_retries(
lambda: self.client.get_bundle_info(uuid, True, True, True))
# format permission data
bundle_info['permission_str'] = permission_str(
bundle_info['permission'])
# format each groups as well
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(
group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting,
metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
"""
Upload |source_file| (a stream) to |worksheet_uuid|.
"""
# Construct info for creating the bundle.
bundle_subclass = get_bundle_subclass(
bundle_type) # program or data
metadata = metadata_util.fill_missing_metadata(
bundle_subclass, {},
initial_metadata={
'name': source_file.name,
'description': 'Upload ' + source_file.name
})
info = {'bundle_type': bundle_type, 'metadata': metadata}
# Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
remote_file_uuid = self.client.open_temp_file()
dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
file_util.copy(source_file,
dest,
autoflush=False,
print_status='Uploading %s' %
info['metadata']['name'])
dest.close()
# Then tell the client that the uploaded file handle is there.
new_bundle_uuid = self.client.upload_bundle_zip(
remote_file_uuid, info, worksheet_uuid, False, True)
return new_bundle_uuid
def get_bundle_info(self, uuid):
bundle_info = _call_with_retries(
lambda: self.client.get_bundle_info(uuid, True, True, True))
if bundle_info is None:
return None
# Set permissions
bundle_info['edit_permission'] = (
bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
bundle_info['permission_str'] = permission_str(
bundle_info['permission'])
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(
group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
for key, value in worksheet_util.get_formatted_metadata(
cls, metadata):
metadata[key] = value
bundle_info['metadata'] = metadata
bundle_info[
'editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(
cls, metadata)
return bundle_info
def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks):
bundle_type = info['bundle_type']
if 'uuid' in info:
existing = True
construct_args = self.bundle_info_to_construct_args(info)
else:
existing = False
construct_args = {'metadata': info['metadata']}
metadata = construct_args['metadata']
message = 'Invalid upload bundle_type: %s' % (bundle_type,)
if not existing:
precondition(bundle_type in UPLOADED_TYPES, message)
bundle_subclass = get_bundle_subclass(bundle_type)
if not existing:
self.validate_user_metadata(bundle_subclass, metadata)
# Upload the given path and record additional metadata from the upload.
if path:
(data_hash, bundle_store_metadata) = self.bundle_store.upload(path, follow_symlinks=follow_symlinks)
metadata.update(bundle_store_metadata)
precondition(construct_args.get('data_hash', data_hash) == data_hash, \
'Provided data_hash doesn\'t match: %s versus %s' % (construct_args.get('data_hash'), data_hash))
construct_args['data_hash'] = data_hash
# Set the owner
construct_args['owner_id'] = self._current_user_id()
bundle = bundle_subclass.construct(**construct_args)
self.model.save_bundle(bundle)
if worksheet_uuid:
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle.uuid))
# TODO: don't fail if don't have permissions
return bundle.uuid
def build_bundles_document(bundle_uuids):
include_set = query_get_json_api_include_set(supported={
'owner', 'group_permissions', 'children', 'host_worksheets'
})
bundles_dict = get_bundle_infos(
bundle_uuids,
get_children='children' in include_set,
get_permissions='group_permissions' in include_set,
get_host_worksheets='host_worksheets' in include_set,
ignore_not_found=False,
)
# Create list of bundles in original order
bundles = [bundles_dict[uuid] for uuid in bundle_uuids]
# Build response document
document = BundleSchema(many=True).dump(bundles).data
# Shim in display metadata used by the front-end application
if query_get_bool('include_display_metadata', default=False):
for bundle, data in zip(bundles, document['data']):
bundle_class = get_bundle_subclass(bundle['bundle_type'])
json_api_meta(
data,
{
'editable_metadata_keys':
worksheet_util.get_editable_metadata_fields(bundle_class),
'metadata_type':
worksheet_util.get_metadata_types(bundle_class),
},
)
if 'owner' in include_set:
owner_ids = set(b['owner_id'] for b in bundles
if b['owner_id'] is not None)
json_api_include(
document,
UserSchema(),
local.model.get_users(user_ids=owner_ids,
limit=len(owner_ids))['results'],
)
if 'group_permissions' in include_set:
for bundle in bundles:
json_api_include(document, BundlePermissionSchema(),
bundle.get('group_permissions', []))
if 'children' in include_set:
for bundle in bundles:
json_api_include(document, BundleSchema(),
bundle.get('children', []))
if 'host_worksheets' in include_set:
for bundle in bundles:
json_api_include(document, WorksheetSchema(),
bundle.get('host_worksheets', []))
return document
def upload(self, bundle_type, path, metadata):
bundle_subclass = get_bundle_subclass(bundle_type)
if not issubclass(bundle_subclass, UploadedBundle):
raise ValueError('Tried to upload %s!' % (bundle_subclass.__name__,))
data_hash = self.bundle_store.upload(path)
bundle = bundle_subclass.construct(data_hash=data_hash, metadata=metadata)
self.model.save_bundle(bundle)
return bundle.uuid
def run(self, program_target, input_target, command, metadata, worksheet_uuid=None):
program_target = self.get_bundle_target(program_target)
input_target = self.get_bundle_target(input_target)
bundle_subclass = get_bundle_subclass('run')
self.validate_user_metadata(bundle_subclass, metadata)
bundle = bundle_subclass.construct(
program_target, input_target, command, metadata)
self.model.save_bundle(bundle)
if worksheet_uuid:
self.add_worksheet_item(worksheet_uuid, bundle.uuid)
return bundle.uuid
def do_edit_command(self, argv, parser):
parser.add_argument('bundle_spec', help='identifier: [<uuid>|<name>]')
args = parser.parse_args(argv)
info = self.client.info(args.bundle_spec)
bundle_subclass = get_bundle_subclass(info['bundle_type'])
new_metadata = metadata_util.request_missing_data(
bundle_subclass,
args,
info['metadata'],
)
if new_metadata != info['metadata']:
self.client.edit(info['uuid'], new_metadata)
def make(self, targets, metadata, worksheet_uuid=None):
bundle_subclass = get_bundle_subclass('make')
self.validate_user_metadata(bundle_subclass, metadata)
targets = {
key: self.get_bundle_target(target)
for (key, target) in targets.iteritems()
}
bundle = bundle_subclass.construct(targets, metadata)
self.model.save_bundle(bundle)
if worksheet_uuid:
self.add_worksheet_item(worksheet_uuid, bundle.uuid)
return bundle.uuid
def upload(self, bundle_type, path, metadata, worksheet_uuid=None):
message = 'Invalid upload bundle_type: %s' % (bundle_type,)
precondition(bundle_type in UPLOADED_TYPES, message)
bundle_subclass = get_bundle_subclass(bundle_type)
self.validate_user_metadata(bundle_subclass, metadata)
# Upload the given path and record additional metadata from the upload.
(data_hash, bundle_store_metadata) = self.bundle_store.upload(path)
metadata.update(bundle_store_metadata)
bundle = bundle_subclass.construct(data_hash=data_hash, metadata=metadata)
self.model.save_bundle(bundle)
if worksheet_uuid:
self.add_worksheet_item(worksheet_uuid, bundle.uuid)
return bundle.uuid
def run(self, targets, command, metadata, worksheet_uuid=None):
bundle_subclass = get_bundle_subclass('run')
self.validate_user_metadata(bundle_subclass, metadata)
targets = {
key: self.get_bundle_target(target)
for (key, target) in targets.iteritems()
}
bundle = bundle_subclass.construct(targets, command, metadata)
self.model.save_bundle(bundle)
self.bundle_store.make_temp_location(bundle.uuid)
if worksheet_uuid:
self.add_worksheet_item(worksheet_uuid, bundle.uuid)
return bundle.uuid
def _derive_bundle(self, bundle_type, targets, command, metadata, worksheet_uuid):
'''
Helper function that creates the bundle but doesn't add it to the worksheet.
Returns the uuid.
'''
bundle_subclass = get_bundle_subclass(bundle_type)
self.validate_user_metadata(bundle_subclass, metadata)
owner_id = self._current_user_id()
bundle = bundle_subclass.construct(targets=targets, command=command, metadata=metadata, owner_id=owner_id)
self.model.save_bundle(bundle)
# Inherit properties of worksheet
self._bundle_inherit_workheet_permissions(bundle.uuid, worksheet_uuid)
return bundle.uuid
def derive_bundle(self, bundle_type, targets, command, metadata, worksheet_uuid):
'''
For both make and run bundles.
Add the resulting bundle to the given worksheet_uuid (optional).
'''
bundle_subclass = get_bundle_subclass(bundle_type)
self.validate_user_metadata(bundle_subclass, metadata)
owner_id = self._current_user_id()
bundle = bundle_subclass.construct(targets=targets, command=command, metadata=metadata, owner_id=owner_id)
self.model.save_bundle(bundle)
if worksheet_uuid:
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle.uuid))
# TODO: don't fail if don't have permissions
return bundle.uuid
def get_bundle(self, uuid):
'''
Retrieve a bundle from the database given its uuid.
'''
with self.engine.begin() as connection:
bundle_row = connection.execute(cl_bundle.select().where(
cl_bundle.c.uuid == uuid
)).fetchone()
if not bundle_row:
raise ValueError('Could not find bundle with uuid %s' % (uuid,))
metadata_rows = connection.execute(cl_bundle_metadata.select().where(
cl_bundle_metadata.c.bundle_uuid == uuid
)).fetchall()
bundle_value = dict(bundle_row, metadata=metadata_rows)
bundle = get_bundle_subclass(bundle_value['bundle_type'])(bundle_value)
bundle.validate()
return bundle
def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
'''
Upload |source_file| (a stream) to |worksheet_uuid|.
'''
# Construct info for creating the bundle.
bundle_subclass = get_bundle_subclass(bundle_type) # program or data
metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.name, 'description': 'Upload ' + source_file.name})
info = {'bundle_type': bundle_type, 'metadata': metadata}
# Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
remote_file_uuid = self.client.open_temp_file()
dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
file_util.copy(source_file, dest, autoflush=False, print_status='Uploading %s' % info['metadata']['name'])
dest.close()
# Then tell the client that the uploaded file handle is there.
new_bundle_uuid = self.client.upload_bundle_zip(remote_file_uuid, info, worksheet_uuid, False, True)
return new_bundle_uuid
def build_bundles_document(bundle_uuids):
bundles_dict = get_bundle_infos(
bundle_uuids,
get_children=True,
get_permissions=True,
get_host_worksheets=True,
)
# Create list of bundles in original order
try:
bundles = [bundles_dict[uuid] for uuid in bundle_uuids]
except KeyError as e:
abort(httplib.NOT_FOUND, "Bundle %s not found" % e.args[0])
# Build response document
document = BundleSchema(many=True).dump(bundles).data
# Shim in editable metadata keys
# Used by the front-end application
for bundle, data in izip(bundles, document['data']):
json_api_meta(
data, {
'editable_metadata_keys':
worksheet_util.get_editable_metadata_fields(
get_bundle_subclass(bundle['bundle_type']))
})
# Include users
owner_ids = set(b['owner_id'] for b in bundles)
json_api_include(document, UserSchema(), local.model.get_users(owner_ids))
# Include permissions
for bundle in bundles:
json_api_include(document, BundlePermissionSchema(),
bundle['group_permissions'])
# Include child bundles
children_uuids = set(c['uuid'] for bundle in bundles
for c in bundle['children'])
json_api_include(document, BundleSchema(),
get_bundle_infos(children_uuids).values())
return document
def batch_get_bundles(self, **kwargs):
'''
Return a list of bundles given a SQLAlchemy clause on the cl_bundle table.
'''
clause = self.make_kwargs_clause(cl_bundle, kwargs)
with self.engine.begin() as connection:
bundle_rows = connection.execute(
cl_bundle.select().where(clause)
).fetchall()
if not bundle_rows:
return []
uuids = set(bundle_row.uuid for bundle_row in bundle_rows)
dependency_rows = connection.execute(cl_bundle_dependency.select().where(
cl_bundle_dependency.c.child_uuid.in_(uuids)
)).fetchall()
metadata_rows = connection.execute(cl_bundle_metadata.select().where(
cl_bundle_metadata.c.bundle_uuid.in_(uuids)
)).fetchall()
# Make a dictionary for each bundle with both data and metadata.
bundle_values = {row.uuid: dict(row) for row in bundle_rows}
for bundle_value in bundle_values.itervalues():
bundle_value['dependencies'] = []
bundle_value['metadata'] = []
for dep_row in dependency_rows:
if dep_row.child_uuid not in bundle_values:
raise IntegrityError('Got dependency %s without bundle' % (dep_row,))
bundle_values[dep_row.child_uuid]['dependencies'].append(dep_row)
for metadata_row in metadata_rows:
if metadata_row.bundle_uuid not in bundle_values:
raise IntegrityError('Got metadata %s without bundle' % (metadata_row,))
bundle_values[metadata_row.bundle_uuid]['metadata'].append(metadata_row)
# Construct and validate all of the retrieved bundles.
sorted_values = sorted(bundle_values.itervalues(), key=lambda r: r['id'])
bundles = [
get_bundle_subclass(bundle_value['bundle_type'])(bundle_value)
for bundle_value in sorted_values
]
for bundle in bundles:
bundle.validate()
return bundles
def get_bundle_info(self, uuid):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
# Set permissions
bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
# Format permissions into strings
bundle_info['permission_str'] = permission_str(bundle_info['permission'])
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
for key, value in worksheet_util.get_formatted_metadata(cls, metadata):
metadata[key] = value
bundle_info['metadata'] = metadata
bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls, metadata)
return bundle_info
def get_bundle_info(self, uuid):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid))
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
"""
Upload |source_file| (a stream) to |worksheet_uuid|.
"""
# Construct info for creating the bundle.
bundle_subclass = get_bundle_subclass(bundle_type) # program or data
metadata = metadata_util.fill_missing_metadata(
bundle_subclass, {},
initial_metadata={
'name': source_file.filename,
'description': 'Upload ' + source_file.filename
})
info = {'bundle_type': bundle_type, 'metadata': metadata}
# Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
remote_file_uuid = self.client.open_temp_file(metadata['name'])
try:
with closing(RPCFileHandle(remote_file_uuid,
self.client.proxy)) as dest:
file_util.copy(source_file.file,
dest,
autoflush=False,
print_status='Uploading %s' % metadata['name'])
pack = False # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
if not pack and zip_util.path_is_archive(metadata['name']):
metadata['name'] = zip_util.strip_archive_ext(metadata['name'])
# Then tell the client that the uploaded file handle is there.
new_bundle_uuid = self.client.finish_upload_bundle(
[remote_file_uuid],
not pack, # unpack
info,
worksheet_uuid,
True) # add_to_worksheet
except:
self.client.finalize_file(remote_file_uuid)
raise
return new_bundle_uuid
def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks, exclude_patterns, add_to_worksheet):
check_worksheet_has_all_permission(self.model, self._current_user(), self.model.get_worksheet(worksheet_uuid, fetch_items=False))
bundle_type = info['bundle_type']
if 'uuid' in info:
existing = True
construct_args = self.bundle_info_to_construct_args(info)
else:
existing = False
construct_args = {'metadata': info['metadata']}
metadata = construct_args['metadata']
message = 'Invalid upload bundle_type: %s' % (bundle_type,)
if not existing:
precondition(bundle_type in UPLOADED_TYPES, message)
bundle_subclass = get_bundle_subclass(bundle_type)
if not existing:
self.validate_user_metadata(bundle_subclass, metadata)
# Upload the given path and record additional metadata from the upload.
if path:
(data_hash, bundle_store_metadata) = self.bundle_store.upload(path, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns)
metadata.update(bundle_store_metadata)
if construct_args.get('data_hash', data_hash) != data_hash:
print >>sys.stderr, 'ERROR: provided data_hash doesn\'t match: %s versus %s' % (construct_args.get('data_hash'), data_hash)
construct_args['data_hash'] = data_hash
# Set the owner
construct_args['owner_id'] = self._current_user_id()
bundle = bundle_subclass.construct(**construct_args)
self.model.save_bundle(bundle)
# Inherit properties of worksheet
self._bundle_inherit_workheet_permissions(bundle.uuid, worksheet_uuid)
# Add to worksheet
if add_to_worksheet:
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle.uuid))
return bundle.uuid
def recurse(old_bundle_uuid):
if old_bundle_uuid in old_to_new:
#print old_bundle_uuid, 'cached'
return old_to_new[old_bundle_uuid]
# Don't have any more information (because we probably hit the maximum depth)
if old_bundle_uuid not in infos:
#print old_bundle_uuid, 'no information'
return old_bundle_uuid
# Get information about the old bundle.
info = infos[old_bundle_uuid]
new_dependencies = [{
'parent_uuid': recurse(dep['parent_uuid']),
'parent_path': dep['parent_path'],
'child_uuid': dep['child_uuid'], # This is just a placeholder to do the equality test
'child_path': dep['child_path']
} for dep in info['dependencies']]
# We're downstream, so need to make a new bundle
if any(dep['parent_uuid'] in downstream for dep in info['dependencies']):
# Now create a new bundle that mimics the old bundle.
# Only change the name if the output name is supplied.
old_bundle_name = info['metadata']['name']
new_info = copy.deepcopy(info)
new_metadata = new_info['metadata']
if new_output_name:
if old_bundle_uuid == old_output:
new_metadata['name'] = new_output_name
else:
# Just make up a name heuristically
new_metadata['name'] = new_output_name + '-' + info['metadata']['name']
# Remove all the automatically generated keys
cls = get_bundle_subclass(new_info['bundle_type'])
for spec in cls.METADATA_SPECS:
if spec.generated and spec.key in new_metadata:
new_metadata.pop(spec.key)
# Set the targets
targets = [(dep['child_path'], (dep['parent_uuid'], dep['parent_path'])) for dep in new_dependencies]
if dry_run:
new_bundle_uuid = None
else:
new_bundle_uuid = self.derive_bundle(new_info['bundle_type'], \
targets, new_info['command'], new_metadata, None)
# Add to worksheet
if shadow:
self.model.add_shadow_worksheet_items(old_bundle_uuid, new_bundle_uuid)
else:
# Copy the markup and directives right before the old
add_with_prelude_items(old_bundle_uuid, new_bundle_uuid)
new_info['uuid'] = new_bundle_uuid
plan.append((info, new_info))
downstream.add(old_bundle_uuid)
else:
new_bundle_uuid = old_bundle_uuid
old_to_new[old_bundle_uuid] = new_bundle_uuid # Cache it
return new_bundle_uuid
def _create_bundles():
"""
Bulk create bundles.
|worksheet_uuid| - The parent worksheet of the bundle, add to this worksheet
if not detached or shadowing another bundle. Also used
to inherit permissions.
|shadow| - the uuid of the bundle to shadow
|detached| - True ('1') if should not add new bundle to any worksheet,
or False ('0') otherwise. Default is False.
|wait_for_upload| - True ('1') if the bundle state should be initialized to
UPLOADING regardless of the bundle type, or False ('0')
otherwise. This prevents run bundles that are being
copied from another instance from being run by the
BundleManager. Default is False.
"""
worksheet_uuid = request.query.get('worksheet')
shadow_parent_uuid = request.query.get('shadow')
detached = query_get_bool('detached', default=False)
if worksheet_uuid is None:
abort(
httplib.BAD_REQUEST, "Parent worksheet id must be specified as"
"'worksheet' query parameter")
# Deserialize bundle fields
bundles = BundleSchema(
strict=True,
many=True,
dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS,
).load(request.json).data
# Check for all necessary permissions
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
worksheet_util.check_worksheet_not_frozen(worksheet)
request.user.check_quota(need_time=True, need_disk=True)
created_uuids = []
for bundle in bundles:
# Prep bundle info for saving into database
# Unfortunately cannot use the `construct` methods because they don't
# provide a uniform interface for constructing bundles for all types
# Hopefully this can all be unified after REST migration is complete
bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
created_uuids.append(bundle_uuid)
bundle_class = get_bundle_subclass(bundle['bundle_type'])
bundle['owner_id'] = request.user.user_id
bundle['state'] = (
State.UPLOADING if issubclass(bundle_class, UploadedBundle)
or query_get_bool('wait_for_upload', False) else State.CREATED)
bundle.setdefault('metadata', {})['created'] = int(time.time())
for dep in bundle.setdefault('dependencies', []):
dep['child_uuid'] = bundle_uuid
# Create bundle object
bundle = bundle_class(bundle, strict=False)
# Save bundle into model
local.model.save_bundle(bundle)
# Inherit worksheet permissions
group_permissions = local.model.get_group_worksheet_permissions(
request.user.user_id, worksheet_uuid)
set_bundle_permissions([{
'object_uuid': bundle_uuid,
'group_uuid': p['group_uuid'],
'permission': p['permission'],
} for p in group_permissions])
# Add as item to worksheet
if not detached:
if shadow_parent_uuid is None:
local.model.add_worksheet_item(
worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
else:
local.model.add_shadow_worksheet_items(shadow_parent_uuid,
bundle_uuid)
# Get created bundles
bundles_dict = get_bundle_infos(created_uuids)
# Return bundles in original order
bundles = [bundles_dict[uuid] for uuid in created_uuids]
return BundleSchema(many=True).dump(bundles).data
def recurse(old_bundle_uuid):
if old_bundle_uuid in old_to_new:
return old_to_new[old_bundle_uuid]
# Don't have any more information (because we probably hit the maximum depth)
if old_bundle_uuid not in infos:
return old_bundle_uuid
# Get information about the old bundle.
old_info = infos[old_bundle_uuid]
new_dependencies = [
{
'parent_uuid': recurse(dep['parent_uuid']),
'parent_path': dep['parent_path'],
'child_uuid': dep[
'child_uuid'
], # This is just a placeholder to do the equality test
'child_path': dep['child_path'],
}
for dep in old_info['dependencies']
]
# If there are no inputs or if we're downstream of any inputs, we need to make a new bundle.
lone_output = len(old_inputs) == 0 and old_bundle_uuid == old_output
downstream_of_inputs = any(
dep['parent_uuid'] in downstream for dep in old_info['dependencies']
)
if lone_output or downstream_of_inputs:
# Now create a new bundle that mimics the old bundle.
new_info = copy.deepcopy(old_info)
# Make sure that new uuids are generated
new_info.pop('uuid', None)
new_info.pop('id', None)
# Only change the name if the output name is supplied.
new_metadata = new_info['metadata']
if new_output_name:
if old_bundle_uuid == old_output:
new_metadata['name'] = new_output_name
else:
# Just make up a name heuristically
new_metadata['name'] = new_output_name + '-' + old_info['metadata']['name']
# By default, the mimic bundle uses whatever image the old bundle uses
# Preferably it uses the SHA256 image digest, but it may simply copy request_docker_image
# if it is not present
if new_info['bundle_type'] == 'run' and new_metadata.get('docker_image', ''):
# Put docker_image in requested_docker_image if it is present and this is a run bundle
new_metadata['request_docker_image'] = new_metadata['docker_image']
cls = get_bundle_subclass(new_info['bundle_type'])
for spec in cls.METADATA_SPECS:
# Remove automatically generated keys
if spec.generated and spec.key in new_metadata:
del new_metadata[spec.key]
# Override original metadata keys
if spec.key in metadata_override:
new_metadata[spec.key] = metadata_override[spec.key]
# Set up info dict
new_info['metadata'] = new_metadata
new_info['dependencies'] = new_dependencies
if dry_run:
new_info['uuid'] = None
else:
if new_info['bundle_type'] not in ('make', 'run'):
raise UsageError(
'Can\'t mimic %s since it is not make or run' % old_bundle_uuid
)
# Create the new bundle, requesting to shadow the old
# bundle in its worksheet if shadow is specified, otherwise
# leave the bundle detached, to be added later below.
params = {}
params['worksheet'] = worksheet_uuid
if shadow:
params['shadow'] = old_info['uuid']
else:
params['detached'] = True
new_info = client.create('bundles', new_info, params=params)
new_bundle_uuid = new_info['uuid']
plan.append((old_info, new_info))
downstream.add(old_bundle_uuid)
created_uuids.add(new_bundle_uuid)
else:
new_bundle_uuid = old_bundle_uuid
old_to_new[old_bundle_uuid] = new_bundle_uuid # Cache it
return new_bundle_uuid
def _create_bundles():
"""
Bulk create bundles.
Query parameters:
- `worksheet`: UUID of the parent worksheet of the new bundle, add to
this worksheet if not detached or shadowing another bundle. The new
bundle also inherits permissions from this worksheet.
- `shadow`: UUID of the bundle to "shadow" (the new bundle will be added
as an item immediately after this bundle in its parent worksheet).
- `detached`: 1 if should not add new bundle to any worksheet,
or 0 otherwise. Default is 0.
- `wait_for_upload`: 1 if the bundle state should be initialized to
"uploading" regardless of the bundle type, or 0 otherwise. Used when
copying bundles from another CodaLab instance, this prevents these new
bundles from being executed by the BundleManager. Default is 0.
"""
worksheet_uuid = request.query.get('worksheet')
shadow_parent_uuid = request.query.get('shadow')
after_sort_key = request.query.get('after_sort_key')
detached = query_get_bool('detached', default=False)
if worksheet_uuid is None:
abort(
http.client.BAD_REQUEST,
"Parent worksheet id must be specified as"
"'worksheet' query parameter",
)
# Deserialize bundle fields
bundles = (BundleSchema(strict=True,
many=True,
dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS).load(
request.json).data)
# Check for all necessary permissions
worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
worksheet_util.check_worksheet_not_frozen(worksheet)
request.user.check_quota(need_time=True, need_disk=True)
created_uuids = []
for bundle in bundles:
# Prep bundle info for saving into database
# Unfortunately cannot use the `construct` methods because they don't
# provide a uniform interface for constructing bundles for all types
# Hopefully this can all be unified after REST migration is complete
bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
created_uuids.append(bundle_uuid)
bundle_class = get_bundle_subclass(bundle['bundle_type'])
bundle['owner_id'] = request.user.user_id
metadata = bundle.get("metadata", {})
if metadata.get("link_url"):
bundle['state'] = State.READY
elif issubclass(bundle_class, UploadedBundle) or query_get_bool(
'wait_for_upload', False):
bundle['state'] = State.UPLOADING
else:
bundle['state'] = State.CREATED
bundle[
'is_anonymous'] = worksheet.is_anonymous # inherit worksheet anonymity
bundle.setdefault('metadata', {})['created'] = int(time.time())
for dep in bundle.setdefault('dependencies', []):
dep['child_uuid'] = bundle_uuid
# Create bundle object
bundle = bundle_class(bundle, strict=False)
# Save bundle into model
local.model.save_bundle(bundle)
# Inherit worksheet permissions
group_permissions = local.model.get_group_worksheet_permissions(
request.user.user_id, worksheet_uuid)
set_bundle_permissions([{
'object_uuid': bundle_uuid,
'group_uuid': p['group_uuid'],
'permission': p['permission'],
} for p in group_permissions])
# Add as item to worksheet
if not detached:
if shadow_parent_uuid is None:
local.model.add_worksheet_items(
worksheet_uuid, [worksheet_util.bundle_item(bundle_uuid)],
after_sort_key)
else:
local.model.add_shadow_worksheet_items(shadow_parent_uuid,
bundle_uuid)
# Get created bundles
bundles_dict = get_bundle_infos(created_uuids)
# Return bundles in original order
# Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted
# right after being created.
bundles = [
bundles_dict[uuid] for uuid in created_uuids if uuid in bundles_dict
]
return BundleSchema(many=True).dump(bundles).data
def recurse(old_bundle_uuid):
if old_bundle_uuid in old_to_new:
return old_to_new[old_bundle_uuid]
# Don't have any more information (because we probably hit the maximum depth)
if old_bundle_uuid not in infos:
return old_bundle_uuid
# Get information about the old bundle.
old_info = infos[old_bundle_uuid]
new_dependencies = [
{
'parent_uuid': recurse(dep['parent_uuid']),
'parent_path': dep['parent_path'],
'child_uuid': dep[
'child_uuid'
], # This is just a placeholder to do the equality test
'child_path': dep['child_path'],
}
for dep in old_info['dependencies']
]
# If there are no inputs or if we're downstream of any inputs, we need to make a new bundle.
lone_output = len(old_inputs) == 0 and old_bundle_uuid == old_output
downstream_of_inputs = any(
dep['parent_uuid'] in downstream for dep in old_info['dependencies']
)
if lone_output or downstream_of_inputs:
# Now create a new bundle that mimics the old bundle.
new_info = copy.deepcopy(old_info)
# Make sure that new uuids are generated
new_info.pop('uuid', None)
new_info.pop('id', None)
# Only change the name if the output name is supplied.
new_metadata = new_info['metadata']
if new_output_name:
if old_bundle_uuid == old_output:
new_metadata['name'] = new_output_name
else:
# Just make up a name heuristically
new_metadata['name'] = new_output_name + '-' + old_info['metadata']['name']
# By default, the mimic bundle uses whatever image the old bundle uses
# Preferably it uses the SHA256 image digest, but it may simply copy request_docker_image
# if it is not present
if new_info['bundle_type'] == 'run' and new_metadata.get('docker_image', ''):
# Put docker_image in requested_docker_image if it is present and this is a run bundle
new_metadata['request_docker_image'] = new_metadata['docker_image']
cls = get_bundle_subclass(new_info['bundle_type'])
for spec in cls.METADATA_SPECS:
# Remove automatically generated keys
if spec.generated and spec.key in new_metadata:
del new_metadata[spec.key]
# Override original metadata keys
if spec.key in metadata_override:
new_metadata[spec.key] = metadata_override[spec.key]
# Set up info dict
new_info['metadata'] = new_metadata
new_info['dependencies'] = new_dependencies
if dry_run:
new_info['uuid'] = None
else:
if new_info['bundle_type'] not in ('make', 'run'):
raise UsageError(
'Can\'t mimic %s since it is not make or run' % old_bundle_uuid
)
# Create the new bundle, requesting to shadow the old
# bundle in its worksheet if shadow is specified, otherwise
# leave the bundle detached, to be added later below.
params = {}
params['worksheet'] = worksheet_uuid
if shadow:
params['shadow'] = old_info['uuid']
else:
params['detached'] = True
new_info = client.create('bundles', new_info, params=params)
new_bundle_uuid = new_info['uuid']
plan.append((old_info, new_info))
downstream.add(old_bundle_uuid)
created_uuids.add(new_bundle_uuid)
else:
new_bundle_uuid = old_bundle_uuid
old_to_new[old_bundle_uuid] = new_bundle_uuid # Cache it
return new_bundle_uuid
def run(self, program_uuid, targets, command):
bundle_subclass = get_bundle_subclass('run')
bundle = bundle_subclass.construct(program_uuid, targets, command)
self.model.save_bundle(bundle)
return bundle.uuid
def make(self, targets):
bundle_subclass = get_bundle_subclass('make')
bundle = bundle_subclass.construct(targets)
self.model.save_bundle(bundle)
return bundle.uuid
def recurse(old_bundle_uuid):
if old_bundle_uuid in old_to_new:
#print old_bundle_uuid, 'cached'
return old_to_new[old_bundle_uuid]
# Don't have any more information (because we probably hit the maximum depth)
if old_bundle_uuid not in infos:
#print old_bundle_uuid, 'no information'
return old_bundle_uuid
# Get information about the old bundle.
info = infos[old_bundle_uuid]
new_dependencies = [{
'parent_uuid': recurse(dep['parent_uuid']),
'parent_path': dep['parent_path'],
'child_uuid': dep['child_uuid'], # This is just a placeholder to do the equality test
'child_path': dep['child_path']
} for dep in info['dependencies']]
# If there are no inputs or if we're downstream of any inputs, we need to make a new bundle.
lone_output = (len(old_inputs) == 0 and old_bundle_uuid == old_output)
downstream_of_inputs = any(dep['parent_uuid'] in downstream for dep in info['dependencies'])
if lone_output or downstream_of_inputs:
# Now create a new bundle that mimics the old bundle.
# Only change the name if the output name is supplied.
old_bundle_name = info['metadata']['name']
new_info = copy.deepcopy(info)
new_metadata = new_info['metadata']
if new_output_name:
if old_bundle_uuid == old_output:
new_metadata['name'] = new_output_name
else:
# Just make up a name heuristically
new_metadata['name'] = new_output_name + '-' + info['metadata']['name']
# Remove all the automatically generated keys
cls = get_bundle_subclass(new_info['bundle_type'])
for spec in cls.METADATA_SPECS:
if spec.generated and spec.key in new_metadata:
new_metadata.pop(spec.key)
# Set the targets
targets = [(dep['child_path'], (dep['parent_uuid'], dep['parent_path'])) for dep in new_dependencies]
if dry_run:
new_bundle_uuid = None
else:
if new_info['bundle_type'] not in ('make', 'run'):
raise UsageError('Can\'t mimic %s since it is not make or run' % old_bundle_uuid)
new_bundle_uuid = self._derive_bundle(new_info['bundle_type'], \
targets, new_info['command'], new_metadata, worksheet_uuid)
new_info['uuid'] = new_bundle_uuid
plan.append((info, new_info))
downstream.add(old_bundle_uuid)
created_uuids.add(new_bundle_uuid)
else:
new_bundle_uuid = old_bundle_uuid
old_to_new[old_bundle_uuid] = new_bundle_uuid # Cache it
return new_bundle_uuid