Example #1
0
 def do_upload_command(self, argv, parser):
     worksheet_uuid = self.env_model.get_current_worksheet()
     help_text = 'bundle_type: [%s]' % ('|'.join(sorted(UPLOADED_TYPES)))
     parser.add_argument('bundle_type', help=help_text)
     parser.add_argument('path', help='path of the directory to upload')
     # Add metadata arguments for UploadedBundle and all of its subclasses.
     metadata_keys = set()
     metadata_util.add_arguments(UploadedBundle, metadata_keys, parser)
     for bundle_type in UPLOADED_TYPES:
         bundle_subclass = get_bundle_subclass(bundle_type)
         metadata_util.add_arguments(bundle_subclass, metadata_keys, parser)
     metadata_util.add_auto_argument(parser)
     args = parser.parse_args(argv)
     # Check that the upload path exists.
     path_util.check_isvalid(path_util.normalize(args.path), 'upload')
     # Pull out the upload bundle type from the arguments and validate it.
     if args.bundle_type not in UPLOADED_TYPES:
         raise UsageError('Invalid bundle type %s (options: [%s])' % (
           args.bundle_type, '|'.join(sorted(UPLOADED_TYPES)),
         ))
     bundle_subclass = get_bundle_subclass(args.bundle_type)
     metadata = metadata_util.request_missing_data(bundle_subclass, args)
     # Type-check the bundle metadata BEFORE uploading the bundle data.
     # This optimization will avoid file copies on failed bundle creations.
     bundle_subclass.construct(data_hash='', metadata=metadata).validate()
     print self.client.upload(args.bundle_type, args.path, metadata, worksheet_uuid)
Example #2
0
 def do_upload_command(self, argv, parser):
   parser.add_argument('bundle_type', help='bundle type: [program|dataset]')
   parser.add_argument('path', help='path of the directory to upload')
   parser.add_argument('--name', help='name: [a-zA-Z0-9_]+)')
   parser.add_argument(
     '--desc',
     dest='description',
     help='human-readable description',
     metavar='DESC',
   )
   parser.add_argument('--tags', help='list of search tags', nargs='+')
   this_machine = platform.machine()
   default_architecture = [this_machine] if this_machine else []
   parser.add_argument(
     '--arch',
     default=default_architecture,
     dest='architecture',
     help='viable architectures (for programs)',
     metavar='ARCH',
     nargs='+',
   )
   args = parser.parse_args(argv)
   bundle_subclass = get_bundle_subclass(args.bundle_type)
   metadata = {k: getattr(args, k) for k in bundle_subclass.METADATA_TYPES}
   print metadata
   print self.client.upload(args.bundle_type, args.path, metadata)
Example #3
0
    def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
        """
        Upload |source_file| (a stream) to |worksheet_uuid|.
        """
        # Construct info for creating the bundle.
        bundle_subclass = get_bundle_subclass(bundle_type) # program or data
        metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.filename, 'description': 'Upload ' + source_file.filename})
        info = {'bundle_type': bundle_type, 'metadata': metadata}

        # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
        remote_file_uuid = self.client.open_temp_file(metadata['name'])
        try:
            with closing(RPCFileHandle(remote_file_uuid, self.client.proxy)) as dest:
                file_util.copy(source_file.file, dest, autoflush=False, print_status='Uploading %s' % metadata['name'])
           
            pack = False  # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
            if not pack and zip_util.path_is_archive(metadata['name']):
                metadata['name'] = zip_util.strip_archive_ext(metadata['name'])
           
            # Then tell the client that the uploaded file handle is there.
            new_bundle_uuid = self.client.finish_upload_bundle(
                [remote_file_uuid],
                not pack,  # unpack
                info,
                worksheet_uuid,
                True)  # add_to_worksheet
        except:
            self.client.finalize_file(remote_file_uuid)
            raise
        return new_bundle_uuid
Example #4
0
        def get_bundle_info(self, uuid):
            ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
            # format permission data
            bundle_info['permission_str'] = permission_str(bundle_info['permission'])
            # format each groups as well
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info
Example #5
0
        def get_bundle_info(self, uuid):
            ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
            bundle_info = _call_with_retries(
                lambda: self.client.get_bundle_info(uuid, True, True, True))
            # format permission data
            bundle_info['permission_str'] = permission_str(
                bundle_info['permission'])
            # format each groups as well
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting,
                                                  metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info
Example #6
0
        def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
            """
            Upload |source_file| (a stream) to |worksheet_uuid|.
            """
            # Construct info for creating the bundle.
            bundle_subclass = get_bundle_subclass(
                bundle_type)  # program or data
            metadata = metadata_util.fill_missing_metadata(
                bundle_subclass, {},
                initial_metadata={
                    'name': source_file.name,
                    'description': 'Upload ' + source_file.name
                })
            info = {'bundle_type': bundle_type, 'metadata': metadata}

            # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
            remote_file_uuid = self.client.open_temp_file()
            dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
            file_util.copy(source_file,
                           dest,
                           autoflush=False,
                           print_status='Uploading %s' %
                           info['metadata']['name'])
            dest.close()

            # Then tell the client that the uploaded file handle is there.
            new_bundle_uuid = self.client.upload_bundle_zip(
                remote_file_uuid, info, worksheet_uuid, False, True)
            return new_bundle_uuid
Example #7
0
        def get_bundle_info(self, uuid):
            bundle_info = _call_with_retries(
                lambda: self.client.get_bundle_info(uuid, True, True, True))

            if bundle_info is None:
                return None
            # Set permissions
            bundle_info['edit_permission'] = (
                bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
            # Format permissions into strings
            bundle_info['permission_str'] = permission_str(
                bundle_info['permission'])
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(
                    group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            for key, value in worksheet_util.get_formatted_metadata(
                    cls, metadata):
                metadata[key] = value

            bundle_info['metadata'] = metadata
            bundle_info[
                'editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(
                    cls, metadata)

            return bundle_info
Example #8
0
    def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks):
        bundle_type = info['bundle_type']
        if 'uuid' in info:
            existing = True
            construct_args = self.bundle_info_to_construct_args(info)
        else:
            existing = False
            construct_args = {'metadata': info['metadata']}
        metadata = construct_args['metadata']
        message = 'Invalid upload bundle_type: %s' % (bundle_type,)
        if not existing:
            precondition(bundle_type in UPLOADED_TYPES, message)
        bundle_subclass = get_bundle_subclass(bundle_type)
        if not existing:
            self.validate_user_metadata(bundle_subclass, metadata)

        # Upload the given path and record additional metadata from the upload.
        if path:
            (data_hash, bundle_store_metadata) = self.bundle_store.upload(path, follow_symlinks=follow_symlinks)
            metadata.update(bundle_store_metadata)
            precondition(construct_args.get('data_hash', data_hash) == data_hash, \
                'Provided data_hash doesn\'t match: %s versus %s' % (construct_args.get('data_hash'), data_hash))
            construct_args['data_hash'] = data_hash
        # Set the owner
        construct_args['owner_id'] = self._current_user_id()
        bundle = bundle_subclass.construct(**construct_args)
        self.model.save_bundle(bundle)
        if worksheet_uuid:
            self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle.uuid))
            # TODO: don't fail if don't have permissions
        return bundle.uuid
Example #9
0
def build_bundles_document(bundle_uuids):
    include_set = query_get_json_api_include_set(supported={
        'owner', 'group_permissions', 'children', 'host_worksheets'
    })

    bundles_dict = get_bundle_infos(
        bundle_uuids,
        get_children='children' in include_set,
        get_permissions='group_permissions' in include_set,
        get_host_worksheets='host_worksheets' in include_set,
        ignore_not_found=False,
    )

    # Create list of bundles in original order
    bundles = [bundles_dict[uuid] for uuid in bundle_uuids]

    # Build response document
    document = BundleSchema(many=True).dump(bundles).data

    # Shim in display metadata used by the front-end application
    if query_get_bool('include_display_metadata', default=False):
        for bundle, data in zip(bundles, document['data']):
            bundle_class = get_bundle_subclass(bundle['bundle_type'])
            json_api_meta(
                data,
                {
                    'editable_metadata_keys':
                    worksheet_util.get_editable_metadata_fields(bundle_class),
                    'metadata_type':
                    worksheet_util.get_metadata_types(bundle_class),
                },
            )

    if 'owner' in include_set:
        owner_ids = set(b['owner_id'] for b in bundles
                        if b['owner_id'] is not None)
        json_api_include(
            document,
            UserSchema(),
            local.model.get_users(user_ids=owner_ids,
                                  limit=len(owner_ids))['results'],
        )

    if 'group_permissions' in include_set:
        for bundle in bundles:
            json_api_include(document, BundlePermissionSchema(),
                             bundle.get('group_permissions', []))

    if 'children' in include_set:
        for bundle in bundles:
            json_api_include(document, BundleSchema(),
                             bundle.get('children', []))

    if 'host_worksheets' in include_set:
        for bundle in bundles:
            json_api_include(document, WorksheetSchema(),
                             bundle.get('host_worksheets', []))

    return document
Example #10
0
 def upload(self, bundle_type, path, metadata):
   bundle_subclass = get_bundle_subclass(bundle_type)
   if not issubclass(bundle_subclass, UploadedBundle):
     raise ValueError('Tried to upload %s!' % (bundle_subclass.__name__,))
   data_hash = self.bundle_store.upload(path)
   bundle = bundle_subclass.construct(data_hash=data_hash, metadata=metadata)
   self.model.save_bundle(bundle)
   return bundle.uuid
Example #11
0
 def run(self, program_target, input_target, command, metadata, worksheet_uuid=None):
   program_target = self.get_bundle_target(program_target)
   input_target = self.get_bundle_target(input_target)
   bundle_subclass = get_bundle_subclass('run')
   self.validate_user_metadata(bundle_subclass, metadata)
   bundle = bundle_subclass.construct(
     program_target, input_target, command, metadata)
   self.model.save_bundle(bundle)
   if worksheet_uuid:
     self.add_worksheet_item(worksheet_uuid, bundle.uuid)
   return bundle.uuid
Example #12
0
 def do_edit_command(self, argv, parser):
     parser.add_argument('bundle_spec', help='identifier: [<uuid>|<name>]')
     args = parser.parse_args(argv)
     info = self.client.info(args.bundle_spec)
     bundle_subclass = get_bundle_subclass(info['bundle_type'])
     new_metadata = metadata_util.request_missing_data(
       bundle_subclass,
       args,
       info['metadata'],
     )
     if new_metadata != info['metadata']:
         self.client.edit(info['uuid'], new_metadata)
Example #13
0
 def make(self, targets, metadata, worksheet_uuid=None):
   bundle_subclass = get_bundle_subclass('make')
   self.validate_user_metadata(bundle_subclass, metadata)
   targets = {
     key: self.get_bundle_target(target)
     for (key, target) in targets.iteritems()
   }
   bundle = bundle_subclass.construct(targets, metadata)
   self.model.save_bundle(bundle)
   if worksheet_uuid:
     self.add_worksheet_item(worksheet_uuid, bundle.uuid)
   return bundle.uuid
Example #14
0
 def upload(self, bundle_type, path, metadata, worksheet_uuid=None):
   message = 'Invalid upload bundle_type: %s' % (bundle_type,)
   precondition(bundle_type in UPLOADED_TYPES, message)
   bundle_subclass = get_bundle_subclass(bundle_type)
   self.validate_user_metadata(bundle_subclass, metadata)
   # Upload the given path and record additional metadata from the upload.
   (data_hash, bundle_store_metadata) = self.bundle_store.upload(path)
   metadata.update(bundle_store_metadata)
   bundle = bundle_subclass.construct(data_hash=data_hash, metadata=metadata)
   self.model.save_bundle(bundle)
   if worksheet_uuid:
     self.add_worksheet_item(worksheet_uuid, bundle.uuid)
   return bundle.uuid
Example #15
0
 def run(self, targets, command, metadata, worksheet_uuid=None):
     bundle_subclass = get_bundle_subclass('run')
     self.validate_user_metadata(bundle_subclass, metadata)
     targets = {
       key: self.get_bundle_target(target)
       for (key, target) in targets.iteritems()
     }
     bundle = bundle_subclass.construct(targets, command, metadata)
     self.model.save_bundle(bundle)
     self.bundle_store.make_temp_location(bundle.uuid)
     if worksheet_uuid:
         self.add_worksheet_item(worksheet_uuid, bundle.uuid)
     return bundle.uuid
 def _derive_bundle(self, bundle_type, targets, command, metadata, worksheet_uuid):
     '''
     Helper function that creates the bundle but doesn't add it to the worksheet.
     Returns the uuid.
     '''
     bundle_subclass = get_bundle_subclass(bundle_type)
     self.validate_user_metadata(bundle_subclass, metadata)
     owner_id = self._current_user_id()
     bundle = bundle_subclass.construct(targets=targets, command=command, metadata=metadata, owner_id=owner_id)
     self.model.save_bundle(bundle)
     # Inherit properties of worksheet
     self._bundle_inherit_workheet_permissions(bundle.uuid, worksheet_uuid)
     return bundle.uuid
Example #17
0
 def derive_bundle(self, bundle_type, targets, command, metadata, worksheet_uuid):
     '''
     For both make and run bundles.
     Add the resulting bundle to the given worksheet_uuid (optional).
     '''
     bundle_subclass = get_bundle_subclass(bundle_type)
     self.validate_user_metadata(bundle_subclass, metadata)
     owner_id = self._current_user_id()
     bundle = bundle_subclass.construct(targets=targets, command=command, metadata=metadata, owner_id=owner_id)
     self.model.save_bundle(bundle)
     if worksheet_uuid:
         self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle.uuid))
         # TODO: don't fail if don't have permissions
     return bundle.uuid
Example #18
0
 def get_bundle(self, uuid):
   '''
   Retrieve a bundle from the database given its uuid.
   '''
   with self.engine.begin() as connection:
     bundle_row = connection.execute(cl_bundle.select().where(
       cl_bundle.c.uuid == uuid
     )).fetchone()
     if not bundle_row:
       raise ValueError('Could not find bundle with uuid %s' % (uuid,))
     metadata_rows = connection.execute(cl_bundle_metadata.select().where(
       cl_bundle_metadata.c.bundle_uuid == uuid
     )).fetchall()
   bundle_value = dict(bundle_row, metadata=metadata_rows)
   bundle = get_bundle_subclass(bundle_value['bundle_type'])(bundle_value)
   bundle.validate()
   return bundle
Example #19
0
        def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
            '''
            Upload |source_file| (a stream) to |worksheet_uuid|.
            '''
            # Construct info for creating the bundle.
            bundle_subclass = get_bundle_subclass(bundle_type) # program or data
            metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.name, 'description': 'Upload ' + source_file.name})
            info = {'bundle_type': bundle_type, 'metadata': metadata}

            # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
            remote_file_uuid = self.client.open_temp_file()
            dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
            file_util.copy(source_file, dest, autoflush=False, print_status='Uploading %s' % info['metadata']['name'])
            dest.close()

            # Then tell the client that the uploaded file handle is there.
            new_bundle_uuid = self.client.upload_bundle_zip(remote_file_uuid, info, worksheet_uuid, False, True)
            return new_bundle_uuid
Example #20
0
def build_bundles_document(bundle_uuids):
    bundles_dict = get_bundle_infos(
        bundle_uuids,
        get_children=True,
        get_permissions=True,
        get_host_worksheets=True,
    )

    # Create list of bundles in original order
    try:
        bundles = [bundles_dict[uuid] for uuid in bundle_uuids]
    except KeyError as e:
        abort(httplib.NOT_FOUND, "Bundle %s not found" % e.args[0])

    # Build response document
    document = BundleSchema(many=True).dump(bundles).data

    # Shim in editable metadata keys
    # Used by the front-end application
    for bundle, data in izip(bundles, document['data']):
        json_api_meta(
            data, {
                'editable_metadata_keys':
                worksheet_util.get_editable_metadata_fields(
                    get_bundle_subclass(bundle['bundle_type']))
            })

    # Include users
    owner_ids = set(b['owner_id'] for b in bundles)
    json_api_include(document, UserSchema(), local.model.get_users(owner_ids))

    # Include permissions
    for bundle in bundles:
        json_api_include(document, BundlePermissionSchema(),
                         bundle['group_permissions'])

    # Include child bundles
    children_uuids = set(c['uuid'] for bundle in bundles
                         for c in bundle['children'])
    json_api_include(document, BundleSchema(),
                     get_bundle_infos(children_uuids).values())

    return document
Example #21
0
    def batch_get_bundles(self, **kwargs):
        '''
        Return a list of bundles given a SQLAlchemy clause on the cl_bundle table.
        '''
        clause = self.make_kwargs_clause(cl_bundle, kwargs)
        with self.engine.begin() as connection:
            bundle_rows = connection.execute(
              cl_bundle.select().where(clause)
            ).fetchall()
            if not bundle_rows:
                return []
            uuids = set(bundle_row.uuid for bundle_row in bundle_rows)
            dependency_rows = connection.execute(cl_bundle_dependency.select().where(
              cl_bundle_dependency.c.child_uuid.in_(uuids)
            )).fetchall()
            metadata_rows = connection.execute(cl_bundle_metadata.select().where(
              cl_bundle_metadata.c.bundle_uuid.in_(uuids)
            )).fetchall()

        # Make a dictionary for each bundle with both data and metadata.
        bundle_values = {row.uuid: dict(row) for row in bundle_rows}
        for bundle_value in bundle_values.itervalues():
            bundle_value['dependencies'] = []
            bundle_value['metadata'] = []
        for dep_row in dependency_rows:
            if dep_row.child_uuid not in bundle_values:
                raise IntegrityError('Got dependency %s without bundle' % (dep_row,))
            bundle_values[dep_row.child_uuid]['dependencies'].append(dep_row)
        for metadata_row in metadata_rows:
            if metadata_row.bundle_uuid not in bundle_values:
                raise IntegrityError('Got metadata %s without bundle' % (metadata_row,))
            bundle_values[metadata_row.bundle_uuid]['metadata'].append(metadata_row)

        # Construct and validate all of the retrieved bundles.
        sorted_values = sorted(bundle_values.itervalues(), key=lambda r: r['id'])
        bundles = [
          get_bundle_subclass(bundle_value['bundle_type'])(bundle_value)
          for bundle_value in sorted_values
        ]
        for bundle in bundles:
            bundle.validate()
        return bundles
Example #22
0
        def get_bundle_info(self, uuid):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))

            # Set permissions
            bundle_info['edit_permission'] = (bundle_info['permission'] == GROUP_OBJECT_PERMISSION_ALL)
            # Format permissions into strings
            bundle_info['permission_str'] = permission_str(bundle_info['permission'])
            for group_permission in bundle_info['group_permissions']:
                group_permission['permission_str'] = permission_str(group_permission['permission'])

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            for key, value in worksheet_util.get_formatted_metadata(cls, metadata):
                metadata[key] = value

            bundle_info['metadata'] = metadata
            bundle_info['editable_metadata_fields'] = worksheet_util.get_editable_metadata_fields(cls, metadata)

            return bundle_info
Example #23
0
        def get_bundle_info(self, uuid):
            bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid))

            metadata = bundle_info['metadata']

            cls = get_bundle_subclass(bundle_info['bundle_type'])
            # format based on specs from the cli
            for spec in cls.METADATA_SPECS:
                key = spec.key
                if key not in metadata:
                    continue
                if metadata[key] == '' or metadata[key] == []:
                    continue
                value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
                # if isinstance(value, list):
                #     value = ', '.join(value)
                metadata[key] = value

            bundle_info['metadata'] = metadata

            return bundle_info
Example #24
0
    def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
        """
        Upload |source_file| (a stream) to |worksheet_uuid|.
        """
        # Construct info for creating the bundle.
        bundle_subclass = get_bundle_subclass(bundle_type)  # program or data
        metadata = metadata_util.fill_missing_metadata(
            bundle_subclass, {},
            initial_metadata={
                'name': source_file.filename,
                'description': 'Upload ' + source_file.filename
            })
        info = {'bundle_type': bundle_type, 'metadata': metadata}

        # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
        remote_file_uuid = self.client.open_temp_file(metadata['name'])
        try:
            with closing(RPCFileHandle(remote_file_uuid,
                                       self.client.proxy)) as dest:
                file_util.copy(source_file.file,
                               dest,
                               autoflush=False,
                               print_status='Uploading %s' % metadata['name'])

            pack = False  # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
            if not pack and zip_util.path_is_archive(metadata['name']):
                metadata['name'] = zip_util.strip_archive_ext(metadata['name'])

            # Then tell the client that the uploaded file handle is there.
            new_bundle_uuid = self.client.finish_upload_bundle(
                [remote_file_uuid],
                not pack,  # unpack
                info,
                worksheet_uuid,
                True)  # add_to_worksheet
        except:
            self.client.finalize_file(remote_file_uuid)
            raise
        return new_bundle_uuid
    def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks, exclude_patterns, add_to_worksheet):
        check_worksheet_has_all_permission(self.model, self._current_user(), self.model.get_worksheet(worksheet_uuid, fetch_items=False))

        bundle_type = info['bundle_type']
        if 'uuid' in info:
            existing = True
            construct_args = self.bundle_info_to_construct_args(info)
        else:
            existing = False
            construct_args = {'metadata': info['metadata']}
        metadata = construct_args['metadata']
        message = 'Invalid upload bundle_type: %s' % (bundle_type,)
        if not existing:
            precondition(bundle_type in UPLOADED_TYPES, message)
        bundle_subclass = get_bundle_subclass(bundle_type)
        if not existing:
            self.validate_user_metadata(bundle_subclass, metadata)

        # Upload the given path and record additional metadata from the upload.
        if path:
            (data_hash, bundle_store_metadata) = self.bundle_store.upload(path, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns)
            metadata.update(bundle_store_metadata)
            if construct_args.get('data_hash', data_hash) != data_hash:
                print >>sys.stderr, 'ERROR: provided data_hash doesn\'t match: %s versus %s' % (construct_args.get('data_hash'), data_hash)
            construct_args['data_hash'] = data_hash
        # Set the owner
        construct_args['owner_id'] = self._current_user_id()
        bundle = bundle_subclass.construct(**construct_args)
        self.model.save_bundle(bundle)

        # Inherit properties of worksheet
        self._bundle_inherit_workheet_permissions(bundle.uuid, worksheet_uuid)

        # Add to worksheet
        if add_to_worksheet:
            self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle.uuid))

        return bundle.uuid
Example #26
0
        def recurse(old_bundle_uuid):
            if old_bundle_uuid in old_to_new:
                #print old_bundle_uuid, 'cached'
                return old_to_new[old_bundle_uuid]

            # Don't have any more information (because we probably hit the maximum depth)
            if old_bundle_uuid not in infos:
                #print old_bundle_uuid, 'no information'
                return old_bundle_uuid

            # Get information about the old bundle.
            info = infos[old_bundle_uuid]
            new_dependencies = [{
                'parent_uuid': recurse(dep['parent_uuid']),
                'parent_path': dep['parent_path'],
                'child_uuid': dep['child_uuid'],  # This is just a placeholder to do the equality test
                'child_path': dep['child_path']
            } for dep in info['dependencies']]

            # We're downstream, so need to make a new bundle
            if any(dep['parent_uuid'] in downstream for dep in info['dependencies']):
                # Now create a new bundle that mimics the old bundle.
                # Only change the name if the output name is supplied.
                old_bundle_name = info['metadata']['name']
                new_info = copy.deepcopy(info)
                new_metadata = new_info['metadata']
                if new_output_name:
                    if old_bundle_uuid == old_output:
                        new_metadata['name'] = new_output_name
                    else:
                        # Just make up a name heuristically
                        new_metadata['name'] = new_output_name + '-' + info['metadata']['name']

                # Remove all the automatically generated keys
                cls = get_bundle_subclass(new_info['bundle_type'])
                for spec in cls.METADATA_SPECS:
                    if spec.generated and spec.key in new_metadata:
                        new_metadata.pop(spec.key)

                # Set the targets
                targets = [(dep['child_path'], (dep['parent_uuid'], dep['parent_path'])) for dep in new_dependencies]

                if dry_run:
                    new_bundle_uuid = None
                else:
                    new_bundle_uuid = self.derive_bundle(new_info['bundle_type'], \
                        targets, new_info['command'], new_metadata, None)
                    # Add to worksheet
                    if shadow:
                        self.model.add_shadow_worksheet_items(old_bundle_uuid, new_bundle_uuid)
                    else:
                        # Copy the markup and directives right before the old
                        add_with_prelude_items(old_bundle_uuid, new_bundle_uuid)

                new_info['uuid'] = new_bundle_uuid
                plan.append((info, new_info))
                downstream.add(old_bundle_uuid)
            else:
                new_bundle_uuid = old_bundle_uuid

            old_to_new[old_bundle_uuid] = new_bundle_uuid  # Cache it
            return new_bundle_uuid
Example #27
0
def _create_bundles():
    """
    Bulk create bundles.

    |worksheet_uuid| - The parent worksheet of the bundle, add to this worksheet
                       if not detached or shadowing another bundle. Also used
                       to inherit permissions.
    |shadow| - the uuid of the bundle to shadow
    |detached| - True ('1') if should not add new bundle to any worksheet,
                 or False ('0') otherwise. Default is False.
    |wait_for_upload| - True ('1') if the bundle state should be initialized to
                        UPLOADING regardless of the bundle type, or False ('0')
                        otherwise. This prevents run bundles that are being
                        copied from another instance from being run by the
                        BundleManager. Default is False.
    """
    worksheet_uuid = request.query.get('worksheet')
    shadow_parent_uuid = request.query.get('shadow')
    detached = query_get_bool('detached', default=False)
    if worksheet_uuid is None:
        abort(
            httplib.BAD_REQUEST, "Parent worksheet id must be specified as"
            "'worksheet' query parameter")

    # Deserialize bundle fields
    bundles = BundleSchema(
        strict=True,
        many=True,
        dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS,
    ).load(request.json).data

    # Check for all necessary permissions
    worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
    check_worksheet_has_all_permission(local.model, request.user, worksheet)
    worksheet_util.check_worksheet_not_frozen(worksheet)
    request.user.check_quota(need_time=True, need_disk=True)

    created_uuids = []
    for bundle in bundles:
        # Prep bundle info for saving into database
        # Unfortunately cannot use the `construct` methods because they don't
        # provide a uniform interface for constructing bundles for all types
        # Hopefully this can all be unified after REST migration is complete
        bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
        created_uuids.append(bundle_uuid)
        bundle_class = get_bundle_subclass(bundle['bundle_type'])
        bundle['owner_id'] = request.user.user_id
        bundle['state'] = (
            State.UPLOADING if issubclass(bundle_class, UploadedBundle)
            or query_get_bool('wait_for_upload', False) else State.CREATED)
        bundle.setdefault('metadata', {})['created'] = int(time.time())
        for dep in bundle.setdefault('dependencies', []):
            dep['child_uuid'] = bundle_uuid

        # Create bundle object
        bundle = bundle_class(bundle, strict=False)

        # Save bundle into model
        local.model.save_bundle(bundle)

        # Inherit worksheet permissions
        group_permissions = local.model.get_group_worksheet_permissions(
            request.user.user_id, worksheet_uuid)
        set_bundle_permissions([{
            'object_uuid': bundle_uuid,
            'group_uuid': p['group_uuid'],
            'permission': p['permission'],
        } for p in group_permissions])

        # Add as item to worksheet
        if not detached:
            if shadow_parent_uuid is None:
                local.model.add_worksheet_item(
                    worksheet_uuid, worksheet_util.bundle_item(bundle_uuid))
            else:
                local.model.add_shadow_worksheet_items(shadow_parent_uuid,
                                                       bundle_uuid)

    # Get created bundles
    bundles_dict = get_bundle_infos(created_uuids)

    # Return bundles in original order
    bundles = [bundles_dict[uuid] for uuid in created_uuids]
    return BundleSchema(many=True).dump(bundles).data
Example #28
0
    def recurse(old_bundle_uuid):
        if old_bundle_uuid in old_to_new:
            return old_to_new[old_bundle_uuid]

        # Don't have any more information (because we probably hit the maximum depth)
        if old_bundle_uuid not in infos:
            return old_bundle_uuid

        # Get information about the old bundle.
        old_info = infos[old_bundle_uuid]
        new_dependencies = [
            {
                'parent_uuid': recurse(dep['parent_uuid']),
                'parent_path': dep['parent_path'],
                'child_uuid': dep[
                    'child_uuid'
                ],  # This is just a placeholder to do the equality test
                'child_path': dep['child_path'],
            }
            for dep in old_info['dependencies']
        ]

        # If there are no inputs or if we're downstream of any inputs, we need to make a new bundle.
        lone_output = len(old_inputs) == 0 and old_bundle_uuid == old_output
        downstream_of_inputs = any(
            dep['parent_uuid'] in downstream for dep in old_info['dependencies']
        )
        if lone_output or downstream_of_inputs:
            # Now create a new bundle that mimics the old bundle.
            new_info = copy.deepcopy(old_info)

            # Make sure that new uuids are generated
            new_info.pop('uuid', None)
            new_info.pop('id', None)

            # Only change the name if the output name is supplied.
            new_metadata = new_info['metadata']
            if new_output_name:
                if old_bundle_uuid == old_output:
                    new_metadata['name'] = new_output_name
                else:
                    # Just make up a name heuristically
                    new_metadata['name'] = new_output_name + '-' + old_info['metadata']['name']

            # By default, the mimic bundle uses whatever image the old bundle uses
            # Preferably it uses the SHA256 image digest, but it may simply copy request_docker_image
            # if it is not present
            if new_info['bundle_type'] == 'run' and new_metadata.get('docker_image', ''):
                # Put docker_image in requested_docker_image if it is present and this is a run bundle
                new_metadata['request_docker_image'] = new_metadata['docker_image']

            cls = get_bundle_subclass(new_info['bundle_type'])
            for spec in cls.METADATA_SPECS:
                # Remove automatically generated keys
                if spec.generated and spec.key in new_metadata:
                    del new_metadata[spec.key]
                # Override original metadata keys
                if spec.key in metadata_override:
                    new_metadata[spec.key] = metadata_override[spec.key]

            # Set up info dict
            new_info['metadata'] = new_metadata
            new_info['dependencies'] = new_dependencies

            if dry_run:
                new_info['uuid'] = None
            else:
                if new_info['bundle_type'] not in ('make', 'run'):
                    raise UsageError(
                        'Can\'t mimic %s since it is not make or run' % old_bundle_uuid
                    )

                # Create the new bundle, requesting to shadow the old
                # bundle in its worksheet if shadow is specified, otherwise
                # leave the bundle detached, to be added later below.
                params = {}
                params['worksheet'] = worksheet_uuid
                if shadow:
                    params['shadow'] = old_info['uuid']
                else:
                    params['detached'] = True
                new_info = client.create('bundles', new_info, params=params)

            new_bundle_uuid = new_info['uuid']
            plan.append((old_info, new_info))
            downstream.add(old_bundle_uuid)
            created_uuids.add(new_bundle_uuid)
        else:
            new_bundle_uuid = old_bundle_uuid

        old_to_new[old_bundle_uuid] = new_bundle_uuid  # Cache it
        return new_bundle_uuid
Example #29
0
def _create_bundles():
    """
    Bulk create bundles.

    Query parameters:
    - `worksheet`: UUID of the parent worksheet of the new bundle, add to
      this worksheet if not detached or shadowing another bundle. The new
      bundle also inherits permissions from this worksheet.
    - `shadow`: UUID of the bundle to "shadow" (the new bundle will be added
      as an item immediately after this bundle in its parent worksheet).
    - `detached`: 1 if should not add new bundle to any worksheet,
      or 0 otherwise. Default is 0.
    - `wait_for_upload`: 1 if the bundle state should be initialized to
      "uploading" regardless of the bundle type, or 0 otherwise. Used when
      copying bundles from another CodaLab instance, this prevents these new
      bundles from being executed by the BundleManager. Default is 0.
    """
    worksheet_uuid = request.query.get('worksheet')
    shadow_parent_uuid = request.query.get('shadow')
    after_sort_key = request.query.get('after_sort_key')
    detached = query_get_bool('detached', default=False)
    if worksheet_uuid is None:
        abort(
            http.client.BAD_REQUEST,
            "Parent worksheet id must be specified as"
            "'worksheet' query parameter",
        )

    # Deserialize bundle fields
    bundles = (BundleSchema(strict=True,
                            many=True,
                            dump_only=BUNDLE_CREATE_RESTRICTED_FIELDS).load(
                                request.json).data)

    # Check for all necessary permissions
    worksheet = local.model.get_worksheet(worksheet_uuid, fetch_items=False)
    check_worksheet_has_all_permission(local.model, request.user, worksheet)
    worksheet_util.check_worksheet_not_frozen(worksheet)
    request.user.check_quota(need_time=True, need_disk=True)

    created_uuids = []
    for bundle in bundles:
        # Prep bundle info for saving into database
        # Unfortunately cannot use the `construct` methods because they don't
        # provide a uniform interface for constructing bundles for all types
        # Hopefully this can all be unified after REST migration is complete
        bundle_uuid = bundle.setdefault('uuid', spec_util.generate_uuid())
        created_uuids.append(bundle_uuid)
        bundle_class = get_bundle_subclass(bundle['bundle_type'])
        bundle['owner_id'] = request.user.user_id

        metadata = bundle.get("metadata", {})
        if metadata.get("link_url"):
            bundle['state'] = State.READY
        elif issubclass(bundle_class, UploadedBundle) or query_get_bool(
                'wait_for_upload', False):
            bundle['state'] = State.UPLOADING
        else:
            bundle['state'] = State.CREATED
        bundle[
            'is_anonymous'] = worksheet.is_anonymous  # inherit worksheet anonymity
        bundle.setdefault('metadata', {})['created'] = int(time.time())
        for dep in bundle.setdefault('dependencies', []):
            dep['child_uuid'] = bundle_uuid

        # Create bundle object
        bundle = bundle_class(bundle, strict=False)

        # Save bundle into model
        local.model.save_bundle(bundle)

        # Inherit worksheet permissions
        group_permissions = local.model.get_group_worksheet_permissions(
            request.user.user_id, worksheet_uuid)
        set_bundle_permissions([{
            'object_uuid': bundle_uuid,
            'group_uuid': p['group_uuid'],
            'permission': p['permission'],
        } for p in group_permissions])

        # Add as item to worksheet
        if not detached:
            if shadow_parent_uuid is None:
                local.model.add_worksheet_items(
                    worksheet_uuid, [worksheet_util.bundle_item(bundle_uuid)],
                    after_sort_key)
            else:
                local.model.add_shadow_worksheet_items(shadow_parent_uuid,
                                                       bundle_uuid)

    # Get created bundles
    bundles_dict = get_bundle_infos(created_uuids)

    # Return bundles in original order
    # Need to check if the UUID is in the dict, since there is a chance that a bundle is deleted
    # right after being created.
    bundles = [
        bundles_dict[uuid] for uuid in created_uuids if uuid in bundles_dict
    ]
    return BundleSchema(many=True).dump(bundles).data
Example #30
0
    def recurse(old_bundle_uuid):
        if old_bundle_uuid in old_to_new:
            return old_to_new[old_bundle_uuid]

        # Don't have any more information (because we probably hit the maximum depth)
        if old_bundle_uuid not in infos:
            return old_bundle_uuid

        # Get information about the old bundle.
        old_info = infos[old_bundle_uuid]
        new_dependencies = [
            {
                'parent_uuid': recurse(dep['parent_uuid']),
                'parent_path': dep['parent_path'],
                'child_uuid': dep[
                    'child_uuid'
                ],  # This is just a placeholder to do the equality test
                'child_path': dep['child_path'],
            }
            for dep in old_info['dependencies']
        ]

        # If there are no inputs or if we're downstream of any inputs, we need to make a new bundle.
        lone_output = len(old_inputs) == 0 and old_bundle_uuid == old_output
        downstream_of_inputs = any(
            dep['parent_uuid'] in downstream for dep in old_info['dependencies']
        )
        if lone_output or downstream_of_inputs:
            # Now create a new bundle that mimics the old bundle.
            new_info = copy.deepcopy(old_info)

            # Make sure that new uuids are generated
            new_info.pop('uuid', None)
            new_info.pop('id', None)

            # Only change the name if the output name is supplied.
            new_metadata = new_info['metadata']
            if new_output_name:
                if old_bundle_uuid == old_output:
                    new_metadata['name'] = new_output_name
                else:
                    # Just make up a name heuristically
                    new_metadata['name'] = new_output_name + '-' + old_info['metadata']['name']

            # By default, the mimic bundle uses whatever image the old bundle uses
            # Preferably it uses the SHA256 image digest, but it may simply copy request_docker_image
            # if it is not present
            if new_info['bundle_type'] == 'run' and new_metadata.get('docker_image', ''):
                # Put docker_image in requested_docker_image if it is present and this is a run bundle
                new_metadata['request_docker_image'] = new_metadata['docker_image']

            cls = get_bundle_subclass(new_info['bundle_type'])
            for spec in cls.METADATA_SPECS:
                # Remove automatically generated keys
                if spec.generated and spec.key in new_metadata:
                    del new_metadata[spec.key]
                # Override original metadata keys
                if spec.key in metadata_override:
                    new_metadata[spec.key] = metadata_override[spec.key]

            # Set up info dict
            new_info['metadata'] = new_metadata
            new_info['dependencies'] = new_dependencies

            if dry_run:
                new_info['uuid'] = None
            else:
                if new_info['bundle_type'] not in ('make', 'run'):
                    raise UsageError(
                        'Can\'t mimic %s since it is not make or run' % old_bundle_uuid
                    )

                # Create the new bundle, requesting to shadow the old
                # bundle in its worksheet if shadow is specified, otherwise
                # leave the bundle detached, to be added later below.
                params = {}
                params['worksheet'] = worksheet_uuid
                if shadow:
                    params['shadow'] = old_info['uuid']
                else:
                    params['detached'] = True
                new_info = client.create('bundles', new_info, params=params)

            new_bundle_uuid = new_info['uuid']
            plan.append((old_info, new_info))
            downstream.add(old_bundle_uuid)
            created_uuids.add(new_bundle_uuid)
        else:
            new_bundle_uuid = old_bundle_uuid

        old_to_new[old_bundle_uuid] = new_bundle_uuid  # Cache it
        return new_bundle_uuid
Example #31
0
 def run(self, program_uuid, targets, command):
   bundle_subclass = get_bundle_subclass('run')
   bundle = bundle_subclass.construct(program_uuid, targets, command)
   self.model.save_bundle(bundle)
   return bundle.uuid
Example #32
0
 def make(self, targets):
   bundle_subclass = get_bundle_subclass('make')
   bundle = bundle_subclass.construct(targets)
   self.model.save_bundle(bundle)
   return bundle.uuid
        def recurse(old_bundle_uuid):
            if old_bundle_uuid in old_to_new:
                #print old_bundle_uuid, 'cached'
                return old_to_new[old_bundle_uuid]

            # Don't have any more information (because we probably hit the maximum depth)
            if old_bundle_uuid not in infos:
                #print old_bundle_uuid, 'no information'
                return old_bundle_uuid

            # Get information about the old bundle.
            info = infos[old_bundle_uuid]
            new_dependencies = [{
                'parent_uuid': recurse(dep['parent_uuid']),
                'parent_path': dep['parent_path'],
                'child_uuid': dep['child_uuid'],  # This is just a placeholder to do the equality test
                'child_path': dep['child_path']
            } for dep in info['dependencies']]

            # If there are no inputs or if we're downstream of any inputs, we need to make a new bundle.
            lone_output = (len(old_inputs) == 0 and old_bundle_uuid == old_output)
            downstream_of_inputs = any(dep['parent_uuid'] in downstream for dep in info['dependencies'])
            if lone_output or downstream_of_inputs:
                # Now create a new bundle that mimics the old bundle.
                # Only change the name if the output name is supplied.
                old_bundle_name = info['metadata']['name']
                new_info = copy.deepcopy(info)
                new_metadata = new_info['metadata']
                if new_output_name:
                    if old_bundle_uuid == old_output:
                        new_metadata['name'] = new_output_name
                    else:
                        # Just make up a name heuristically
                        new_metadata['name'] = new_output_name + '-' + info['metadata']['name']

                # Remove all the automatically generated keys
                cls = get_bundle_subclass(new_info['bundle_type'])
                for spec in cls.METADATA_SPECS:
                    if spec.generated and spec.key in new_metadata:
                        new_metadata.pop(spec.key)

                # Set the targets
                targets = [(dep['child_path'], (dep['parent_uuid'], dep['parent_path'])) for dep in new_dependencies]

                if dry_run:
                    new_bundle_uuid = None
                else:
                    if new_info['bundle_type'] not in ('make', 'run'):
                        raise UsageError('Can\'t mimic %s since it is not make or run' % old_bundle_uuid)
                    new_bundle_uuid = self._derive_bundle(new_info['bundle_type'], \
                        targets, new_info['command'], new_metadata, worksheet_uuid)

                new_info['uuid'] = new_bundle_uuid
                plan.append((info, new_info))
                downstream.add(old_bundle_uuid)
                created_uuids.add(new_bundle_uuid)
            else:
                new_bundle_uuid = old_bundle_uuid

            old_to_new[old_bundle_uuid] = new_bundle_uuid  # Cache it
            return new_bundle_uuid