Example #1
0
        def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
            """
            Upload |source_file| (a stream) to |worksheet_uuid|.
            """
            # Construct info for creating the bundle.
            bundle_subclass = get_bundle_subclass(
                bundle_type)  # program or data
            metadata = metadata_util.fill_missing_metadata(
                bundle_subclass, {},
                initial_metadata={
                    'name': source_file.name,
                    'description': 'Upload ' + source_file.name
                })
            info = {'bundle_type': bundle_type, 'metadata': metadata}

            # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
            remote_file_uuid = self.client.open_temp_file()
            dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
            file_util.copy(source_file,
                           dest,
                           autoflush=False,
                           print_status='Uploading %s' %
                           info['metadata']['name'])
            dest.close()

            # Then tell the client that the uploaded file handle is there.
            new_bundle_uuid = self.client.upload_bundle_zip(
                remote_file_uuid, info, worksheet_uuid, False, True)
            return new_bundle_uuid
Example #2
0
    def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
        """
        Upload |source_file| (a stream) to |worksheet_uuid|.
        """
        # Construct info for creating the bundle.
        bundle_subclass = get_bundle_subclass(bundle_type) # program or data
        metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.filename, 'description': 'Upload ' + source_file.filename})
        info = {'bundle_type': bundle_type, 'metadata': metadata}

        # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
        remote_file_uuid = self.client.open_temp_file(metadata['name'])
        try:
            with closing(RPCFileHandle(remote_file_uuid, self.client.proxy)) as dest:
                file_util.copy(source_file.file, dest, autoflush=False, print_status='Uploading %s' % metadata['name'])
           
            pack = False  # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
            if not pack and zip_util.path_is_archive(metadata['name']):
                metadata['name'] = zip_util.strip_archive_ext(metadata['name'])
           
            # Then tell the client that the uploaded file handle is there.
            new_bundle_uuid = self.client.finish_upload_bundle(
                [remote_file_uuid],
                not pack,  # unpack
                info,
                worksheet_uuid,
                True)  # add_to_worksheet
        except:
            self.client.finalize_file(remote_file_uuid)
            raise
        return new_bundle_uuid
    def run_evaluation(self, submit_bundle, predict_bundle):
        eval_bundle_name = '{owner[user_name]}-{metadata[name]}-results'.format(
            **submit_bundle)

        # Untag any old evaluation run(s) for this submitter
        old_evaluations = self.client.fetch(
            'bundles',
            params={
                'keywords': [
                    '.mine',  # don't allow others to forge evaluations
                    'tags={evaluate[tag]}'.format(**self.config),
                    'name=' + eval_bundle_name,
                ]
            },
        )
        if old_evaluations:
            self._untag(old_evaluations, self.config['evaluate']['tag'])

        # Create evaluation runs on the predictions with leaderboard tag
        # Build up metadata
        metadata = {
            'name':
            eval_bundle_name,
            'tags': [self.config['evaluate']['tag']],
            'description':
            json.dumps({
                'submit_id': submit_bundle['id'],
                'submitter_id': submit_bundle['owner']['id'],
                'predict_id': predict_bundle['id'],
            }),
        }
        metadata.update(self.config['evaluate']['metadata'])
        metadata = fill_missing_metadata(RunBundle, argparse.Namespace(),
                                         metadata)
        # Substitute in the prediction bundle UUID where required
        dependencies = []
        for dep_spec in self.config['evaluate']['dependencies']:
            dep = dep_spec.copy()
            dep['parent_uuid'] = dep['parent_uuid'].format(
                predict=predict_bundle['uuid'])
            dependencies.append(dep)
        # Create the bundle
        eval_bundle = self.client.create(
            'bundles',
            {
                'bundle_type': 'run',
                'command': self.config['evaluate']['command'],
                'dependencies': dependencies,
                'metadata': metadata,
            },
            params={'worksheet': self.config['log_worksheet_uuid']},
        )
        self._make_public_readable(eval_bundle)
        return eval_bundle
Example #4
0
        def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
            '''
            Upload |source_file| (a stream) to |worksheet_uuid|.
            '''
            # Construct info for creating the bundle.
            bundle_subclass = get_bundle_subclass(bundle_type) # program or data
            metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.name, 'description': 'Upload ' + source_file.name})
            info = {'bundle_type': bundle_type, 'metadata': metadata}

            # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
            remote_file_uuid = self.client.open_temp_file()
            dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
            file_util.copy(source_file, dest, autoflush=False, print_status='Uploading %s' % info['metadata']['name'])
            dest.close()

            # Then tell the client that the uploaded file handle is there.
            new_bundle_uuid = self.client.upload_bundle_zip(remote_file_uuid, info, worksheet_uuid, False, True)
            return new_bundle_uuid
Example #5
0
    def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
        """
        Upload |source_file| (a stream) to |worksheet_uuid|.
        """
        # Construct info for creating the bundle.
        bundle_subclass = get_bundle_subclass(bundle_type)  # program or data
        metadata = metadata_util.fill_missing_metadata(
            bundle_subclass, {},
            initial_metadata={
                'name': source_file.filename,
                'description': 'Upload ' + source_file.filename
            })
        info = {'bundle_type': bundle_type, 'metadata': metadata}

        # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
        remote_file_uuid = self.client.open_temp_file(metadata['name'])
        try:
            with closing(RPCFileHandle(remote_file_uuid,
                                       self.client.proxy)) as dest:
                file_util.copy(source_file.file,
                               dest,
                               autoflush=False,
                               print_status='Uploading %s' % metadata['name'])

            pack = False  # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
            if not pack and zip_util.path_is_archive(metadata['name']):
                metadata['name'] = zip_util.strip_archive_ext(metadata['name'])

            # Then tell the client that the uploaded file handle is there.
            new_bundle_uuid = self.client.finish_upload_bundle(
                [remote_file_uuid],
                not pack,  # unpack
                info,
                worksheet_uuid,
                True)  # add_to_worksheet
        except:
            self.client.finalize_file(remote_file_uuid)
            raise
        return new_bundle_uuid