def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
"""
Upload |source_file| (a stream) to |worksheet_uuid|.
"""
# Construct info for creating the bundle.
bundle_subclass = get_bundle_subclass(
bundle_type) # program or data
metadata = metadata_util.fill_missing_metadata(
bundle_subclass, {},
initial_metadata={
'name': source_file.name,
'description': 'Upload ' + source_file.name
})
info = {'bundle_type': bundle_type, 'metadata': metadata}
# Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
remote_file_uuid = self.client.open_temp_file()
dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
file_util.copy(source_file,
dest,
autoflush=False,
print_status='Uploading %s' %
info['metadata']['name'])
dest.close()
# Then tell the client that the uploaded file handle is there.
new_bundle_uuid = self.client.upload_bundle_zip(
remote_file_uuid, info, worksheet_uuid, False, True)
return new_bundle_uuid
def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks, exclude_patterns, add_to_worksheet):
# URLs can be directly passed to the local client.
if path and not isinstance(path, list) and path_util.path_is_url(path):
return self.upload_bundle_url(path, info, worksheet_uuid, follow_symlinks, exclude_patterns)
# First, zip path up (temporary local zip file).
if path:
name = info['metadata']['name']
zip_path = zip_util.zip(path, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns, file_name=name)
# Copy it up to the server (temporary remote zip file)
with open(zip_path, 'rb') as source:
remote_file_uuid = self.open_temp_file()
dest = RPCFileHandle(remote_file_uuid, self.proxy)
# FileServer does not expose an API for forcibly flushing writes, so
# we rely on closing the file to flush it.
file_util.copy(source, dest, autoflush=False, print_status='Uploading %s%s to %s' % (zip_path, ' ('+info['uuid']+')' if 'uuid' in info else '', self.address))
dest.close()
else:
remote_file_uuid = None
zip_path = None
# Finally, install the zip file (this will be in charge of deleting that zip file).
result = self.upload_bundle_zip(remote_file_uuid, info, worksheet_uuid, follow_symlinks, add_to_worksheet)
if zip_path:
path_util.remove(zip_path) # Remove local zip
return result
def upload_bundle(self, sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, info, worksheet_uuid, add_to_worksheet):
"""
See local_bundle_client.py for documentation on the usage.
Strategy:
1) We copy the |sources| to a temporary directory on the server
(streaming either a tar or tar.gz depending on whether compression is
needed).
2) We politely ask the server to finish_upload_bundle (performs a
LocalBundleClient.upload_bundle from the temporary directory).
"""
# URLs can be directly passed to the local client.
if all(path_util.path_is_url(source) for source in sources):
return self.upload_bundle_url(sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, info, worksheet_uuid, add_to_worksheet)
# 1) Copy sources up to the server (temporary remote zip file)
remote_file_uuids = []
for source in sources:
remote_file_uuid = self.open_temp_file(zip_util.add_packed_suffix(os.path.basename(source)))
remote_file_uuids.append(remote_file_uuid)
dest_handle = RPCFileHandle(remote_file_uuid, self.proxy)
if zip_util.path_is_archive(source):
source_handle = open(source)
else:
source_handle = zip_util.open_packed_path(source, follow_symlinks, exclude_patterns)
unpack = True # We packed it, so we have to unpack it
status = 'Uploading %s%s to %s' % (source, ' ('+info['uuid']+')' if 'uuid' in info else '', self.address)
# FileServer does not expose an API for forcibly flushing writes, so
# we rely on closing the file to flush it.
file_util.copy(source_handle, dest_handle, autoflush=False, print_status=status)
dest_handle.close()
# 2) Install upload (this call will be in charge of deleting the temporary file).
result = self.finish_upload_bundle(remote_file_uuids, unpack, info, worksheet_uuid, add_to_worksheet)
return result
def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
"""
Upload |source_file| (a stream) to |worksheet_uuid|.
"""
# Construct info for creating the bundle.
bundle_subclass = get_bundle_subclass(bundle_type) # program or data
metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.name, 'description': 'Upload ' + source_file.name})
info = {'bundle_type': bundle_type, 'metadata': metadata}
# Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
remote_file_uuid = self.client.open_temp_file(metadata['name'])
dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
file_util.copy(source_file, dest, autoflush=False, print_status='Uploading %s' % metadata['name'])
dest.close()
pack = False # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
if not pack and zip_util.path_is_archive(metadata['name']):
metadata['name'] = zip_util.strip_archive_ext(metadata['name'])
# Then tell the client that the uploaded file handle is there.
new_bundle_uuid = self.client.finish_upload_bundle(
[remote_file_uuid],
not pack, # unpack
info,
worksheet_uuid,
True) # add_to_worksheet
return new_bundle_uuid
def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks):
# URLs can be directly passed to the local client.
if path and not isinstance(path, list) and path_util.path_is_url(path):
return self.upload_bundle_url(path, info, worksheet_uuid, follow_symlinks)
# First, zip path up (temporary local zip file).
if path:
zip_path, sub_path = zip_util.zip(path, follow_symlinks=follow_symlinks)
# Copy it up to the server (temporary remote zip file)
with open(zip_path, 'rb') as source:
remote_file_uuid = self.open_temp_file()
dest = RPCFileHandle(remote_file_uuid, self.proxy)
# FileServer does not expose an API for forcibly flushing writes, so
# we rely on closing the file to flush it.
file_util.copy(source, dest, autoflush=False, print_status=True)
dest.close()
else:
remote_file_uuid = None
zip_path = None
# Finally, install the zip file (this will be in charge of deleting that zip file).
result = self.upload_bundle_zip(remote_file_uuid, info, worksheet_uuid, follow_symlinks)
if zip_path:
path_util.remove(zip_path) # Remove local zip
return result
def copy_bundle(self, source_bundle_uuid, info, dest_client,
dest_worksheet_uuid, add_to_worksheet):
'''
A streamlined combination of download_target and upload_bundle.
Copy from self to dest_client.
'''
# Open source
source_file_uuid, name = self.open_target_zip((source_bundle_uuid, ''),
False)
source = RPCFileHandle(source_file_uuid, self.proxy)
# Open target
dest_file_uuid = dest_client.open_temp_file()
dest = RPCFileHandle(dest_file_uuid, dest_client.proxy)
# Copy contents over
file_util.copy(source,
dest,
autoflush=False,
print_status='Copying %s from %s to %s' %
(source_bundle_uuid, self.address, dest_client.address))
dest.close()
# Finally, install the zip file (this will be in charge of deleting that zip file).
result = dest_client.upload_bundle_zip(dest_file_uuid, info,
dest_worksheet_uuid, False,
add_to_worksheet)
self.finalize_file(source_file_uuid, True) # Delete remote zip file
return result
def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
'''
Upload |source_file| (a stream) to |worksheet_uuid|.
'''
# Construct info for creating the bundle.
bundle_subclass = get_bundle_subclass(bundle_type) # program or data
metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.name, 'description': 'Upload ' + source_file.name})
info = {'bundle_type': bundle_type, 'metadata': metadata}
# Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
remote_file_uuid = self.client.open_temp_file()
dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
file_util.copy(source_file, dest, autoflush=False, print_status='Uploading %s' % info['metadata']['name'])
dest.close()
# Then tell the client that the uploaded file handle is there.
new_bundle_uuid = self.client.upload_bundle_zip(remote_file_uuid, info, worksheet_uuid, False, True)
return new_bundle_uuid
def copy_bundle(self, source_bundle_uuid, info, dest_client, dest_worksheet_uuid, add_to_worksheet):
'''
A streamlined combination of download_target and upload_bundle.
Copy from self to dest_client.
'''
# Open source
source_file_uuid, name = self.open_target_zip((source_bundle_uuid, ''), False)
source = RPCFileHandle(source_file_uuid, self.proxy)
# Open target
dest_file_uuid = dest_client.open_temp_file()
dest = RPCFileHandle(dest_file_uuid, dest_client.proxy)
# Copy contents over
file_util.copy(source, dest, autoflush=False, print_status='Copying %s from %s to %s' % (source_bundle_uuid, self.address, dest_client.address))
dest.close()
# Finally, install the zip file (this will be in charge of deleting that zip file).
result = dest_client.upload_bundle_zip(dest_file_uuid, info, dest_worksheet_uuid, False, add_to_worksheet)
self.finalize_file(source_file_uuid, True) # Delete remote zip file
return result
def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks,
exclude_patterns, add_to_worksheet):
# URLs can be directly passed to the local client.
if path and not isinstance(path, list) and path_util.path_is_url(path):
return self.upload_bundle_url(path, info, worksheet_uuid,
follow_symlinks, exclude_patterns)
# First, zip path up (temporary local zip file).
if path:
name = info['metadata']['name']
zip_path = zip_util.zip(path,
follow_symlinks=follow_symlinks,
exclude_patterns=exclude_patterns,
file_name=name)
# Copy it up to the server (temporary remote zip file)
with open(zip_path, 'rb') as source:
remote_file_uuid = self.open_temp_file()
dest = RPCFileHandle(remote_file_uuid, self.proxy)
# FileServer does not expose an API for forcibly flushing writes, so
# we rely on closing the file to flush it.
file_util.copy(source,
dest,
autoflush=False,
print_status='Uploading %s%s to %s' %
(zip_path, ' (' + info['uuid'] +
')' if 'uuid' in info else '', self.address))
dest.close()
else:
remote_file_uuid = None
zip_path = None
# Finally, install the zip file (this will be in charge of deleting that zip file).
result = self.upload_bundle_zip(remote_file_uuid, info, worksheet_uuid,
follow_symlinks, add_to_worksheet)
if zip_path:
path_util.remove(zip_path) # Remove local zip
return result
def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
"""
Upload |source_file| (a stream) to |worksheet_uuid|.
"""
# Construct info for creating the bundle.
bundle_subclass = get_bundle_subclass(
bundle_type) # program or data
metadata = metadata_util.fill_missing_metadata(
bundle_subclass, {},
initial_metadata={
'name': source_file.name,
'description': 'Upload ' + source_file.name
})
info = {'bundle_type': bundle_type, 'metadata': metadata}
# Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
remote_file_uuid = self.client.open_temp_file(metadata['name'])
dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
file_util.copy(source_file,
dest,
autoflush=False,
print_status='Uploading %s' % metadata['name'])
dest.close()
pack = False # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
if not pack and zip_util.path_is_archive(metadata['name']):
metadata['name'] = zip_util.strip_archive_ext(metadata['name'])
# Then tell the client that the uploaded file handle is there.
new_bundle_uuid = self.client.finish_upload_bundle(
[remote_file_uuid],
not pack, # unpack
info,
worksheet_uuid,
True) # add_to_worksheet
return new_bundle_uuid
