def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
"""
Upload |source_file| (a stream) to |worksheet_uuid|.
"""
# Construct info for creating the bundle.
bundle_subclass = get_bundle_subclass(
bundle_type) # program or data
metadata = metadata_util.fill_missing_metadata(
bundle_subclass, {},
initial_metadata={
'name': source_file.name,
'description': 'Upload ' + source_file.name
})
info = {'bundle_type': bundle_type, 'metadata': metadata}
# Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
remote_file_uuid = self.client.open_temp_file()
dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
file_util.copy(source_file,
dest,
autoflush=False,
print_status='Uploading %s' %
info['metadata']['name'])
dest.close()
# Then tell the client that the uploaded file handle is there.
new_bundle_uuid = self.client.upload_bundle_zip(
remote_file_uuid, info, worksheet_uuid, False, True)
return new_bundle_uuid
def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
"""
Upload |source_file| (a stream) to |worksheet_uuid|.
"""
# Construct info for creating the bundle.
bundle_subclass = get_bundle_subclass(bundle_type) # program or data
metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.filename, 'description': 'Upload ' + source_file.filename})
info = {'bundle_type': bundle_type, 'metadata': metadata}
# Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
remote_file_uuid = self.client.open_temp_file(metadata['name'])
try:
with closing(RPCFileHandle(remote_file_uuid, self.client.proxy)) as dest:
file_util.copy(source_file.file, dest, autoflush=False, print_status='Uploading %s' % metadata['name'])
pack = False # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
if not pack and zip_util.path_is_archive(metadata['name']):
metadata['name'] = zip_util.strip_archive_ext(metadata['name'])
# Then tell the client that the uploaded file handle is there.
new_bundle_uuid = self.client.finish_upload_bundle(
[remote_file_uuid],
not pack, # unpack
info,
worksheet_uuid,
True) # add_to_worksheet
except:
self.client.finalize_file(remote_file_uuid)
raise
return new_bundle_uuid
def run_evaluation(self, submit_bundle, predict_bundle):
eval_bundle_name = '{owner[user_name]}-{metadata[name]}-results'.format(
**submit_bundle)
# Untag any old evaluation run(s) for this submitter
old_evaluations = self.client.fetch(
'bundles',
params={
'keywords': [
'.mine', # don't allow others to forge evaluations
'tags={evaluate[tag]}'.format(**self.config),
'name=' + eval_bundle_name,
]
},
)
if old_evaluations:
self._untag(old_evaluations, self.config['evaluate']['tag'])
# Create evaluation runs on the predictions with leaderboard tag
# Build up metadata
metadata = {
'name':
eval_bundle_name,
'tags': [self.config['evaluate']['tag']],
'description':
json.dumps({
'submit_id': submit_bundle['id'],
'submitter_id': submit_bundle['owner']['id'],
'predict_id': predict_bundle['id'],
}),
}
metadata.update(self.config['evaluate']['metadata'])
metadata = fill_missing_metadata(RunBundle, argparse.Namespace(),
metadata)
# Substitute in the prediction bundle UUID where required
dependencies = []
for dep_spec in self.config['evaluate']['dependencies']:
dep = dep_spec.copy()
dep['parent_uuid'] = dep['parent_uuid'].format(
predict=predict_bundle['uuid'])
dependencies.append(dep)
# Create the bundle
eval_bundle = self.client.create(
'bundles',
{
'bundle_type': 'run',
'command': self.config['evaluate']['command'],
'dependencies': dependencies,
'metadata': metadata,
},
params={'worksheet': self.config['log_worksheet_uuid']},
)
self._make_public_readable(eval_bundle)
return eval_bundle
def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
'''
Upload |source_file| (a stream) to |worksheet_uuid|.
'''
# Construct info for creating the bundle.
bundle_subclass = get_bundle_subclass(bundle_type) # program or data
metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.name, 'description': 'Upload ' + source_file.name})
info = {'bundle_type': bundle_type, 'metadata': metadata}
# Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
remote_file_uuid = self.client.open_temp_file()
dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
file_util.copy(source_file, dest, autoflush=False, print_status='Uploading %s' % info['metadata']['name'])
dest.close()
# Then tell the client that the uploaded file handle is there.
new_bundle_uuid = self.client.upload_bundle_zip(remote_file_uuid, info, worksheet_uuid, False, True)
return new_bundle_uuid
def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
"""
Upload |source_file| (a stream) to |worksheet_uuid|.
"""
# Construct info for creating the bundle.
bundle_subclass = get_bundle_subclass(bundle_type) # program or data
metadata = metadata_util.fill_missing_metadata(
bundle_subclass, {},
initial_metadata={
'name': source_file.filename,
'description': 'Upload ' + source_file.filename
})
info = {'bundle_type': bundle_type, 'metadata': metadata}
# Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
remote_file_uuid = self.client.open_temp_file(metadata['name'])
try:
with closing(RPCFileHandle(remote_file_uuid,
self.client.proxy)) as dest:
file_util.copy(source_file.file,
dest,
autoflush=False,
print_status='Uploading %s' % metadata['name'])
pack = False # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
if not pack and zip_util.path_is_archive(metadata['name']):
metadata['name'] = zip_util.strip_archive_ext(metadata['name'])
# Then tell the client that the uploaded file handle is there.
new_bundle_uuid = self.client.finish_upload_bundle(
[remote_file_uuid],
not pack, # unpack
info,
worksheet_uuid,
True) # add_to_worksheet
except:
self.client.finalize_file(remote_file_uuid)
raise
return new_bundle_uuid
