def test_local_file_non_archive(self):
"""This local file should be categorized as not an archive."""
path = "/tmp/file.txt"
self.assertEqual(path_is_archive(path), False)
self.assertEqual(get_archive_ext(path), "")
with self.assertRaises(UsageError):
strip_archive_ext(path)
def test_url_non_archive(self):
"""This URL should be categorized as not an archive."""
path = "https://codalab.org/file.txt"
self.assertEqual(path_is_archive(path), False)
self.assertEqual(get_archive_ext(path), "")
with self.assertRaises(UsageError):
strip_archive_ext(path)
def _unpack_file(self, source_path, dest_path, remove_source,
simplify_archive):
zip_util.unpack(zip_util.get_archive_ext(source_path), source_path,
dest_path)
if remove_source:
path_util.remove(source_path)
if simplify_archive:
self._simplify_archive(dest_path)
def get_blob(uuid, path=''):
"""
API to download the contents of a bundle or a subpath within a bundle.
For directories this method always returns a tarred and gzipped archive of
the directory.
For files, if the request has an Accept-Encoding header containing gzip,
then the returned file is gzipped.
"""
check_bundles_have_read_permission(local.model, request.user, [uuid])
bundle = local.model.get_bundle(uuid)
target_info = local.download_manager.get_target_info(uuid, path, 0)
if target_info is None:
abort(httplib.NOT_FOUND, 'Not found.')
# Figure out the file name.
if not path and bundle.metadata.name:
filename = bundle.metadata.name
else:
filename = target_info['name']
if target_info['type'] == 'directory':
# Always tar and gzip directories.
filename = filename + '.tar.gz'
fileobj = local.download_manager.stream_tarred_gzipped_directory(
uuid, path)
elif target_info['type'] == 'file':
if not zip_util.path_is_archive(
filename) and request_accepts_gzip_encoding():
# Let's gzip to save bandwidth. The browser will transparently decode
# the file.
filename = filename + '.gz'
fileobj = local.download_manager.stream_file(uuid,
path,
gzipped=True)
else:
fileobj = local.download_manager.stream_file(uuid,
path,
gzipped=False)
else:
# Symlinks.
abort(httplib.FORBIDDEN, 'Cannot download files of this type.')
# Set headers.
mimetype, _ = mimetypes.guess_type(filename, strict=False)
response.set_header('Content-Type', mimetype or 'text/plain')
if zip_util.get_archive_ext(
filename) == '.gz' and request_accepts_gzip_encoding():
filename = zip_util.strip_archive_ext(filename)
response.set_header('Content-Encoding', 'gzip')
else:
response.set_header('Content-Encoding', 'identity')
response.set_header('Content-Disposition', 'filename="%s"' % filename)
return fileobj
def get_blob(uuid, path=''):
"""
API to download the contents of a bundle or a subpath within a bundle.
For directories this method always returns a tarred and gzipped archive of
the directory.
For files, if the request has an Accept-Encoding header containing gzip,
then the returned file is gzipped.
"""
check_bundles_have_read_permission(local.model, request.user, [uuid])
bundle = local.model.get_bundle(uuid)
target_info = local.download_manager.get_target_info(uuid, path, 0)
if target_info is None:
abort(httplib.NOT_FOUND, 'Not found.')
# Figure out the file name.
if not path and bundle.metadata.name:
filename = bundle.metadata.name
else:
filename = target_info['name']
if target_info['type'] == 'directory':
# Always tar and gzip directories.
filename = filename + '.tar.gz'
fileobj = local.download_manager.stream_tarred_gzipped_directory(uuid, path)
elif target_info['type'] == 'file':
if not zip_util.path_is_archive(filename) and request_accepts_gzip_encoding():
# Let's gzip to save bandwidth. The browser will transparently decode
# the file.
filename = filename + '.gz'
fileobj = local.download_manager.stream_file(uuid, path, gzipped=True)
else:
fileobj = local.download_manager.stream_file(uuid, path, gzipped=False)
else:
# Symlinks.
abort(httplib.FORBIDDEN, 'Cannot download files of this type.')
# Set headers.
mimetype, _ = mimetypes.guess_type(filename, strict=False)
response.set_header('Content-Type', mimetype or 'text/plain')
if zip_util.get_archive_ext(filename) == '.gz' and request_accepts_gzip_encoding():
filename = zip_util.strip_archive_ext(filename)
response.set_header('Content-Encoding', 'gzip')
else:
response.set_header('Content-Encoding', 'identity')
response.set_header('Content-Disposition', 'filename="%s"' % filename)
return fileobj
def upload_to_bundle_store(self, bundle: Bundle, source: Source, git: bool,
unpack: bool):
"""Uploads the given source to the bundle store.
Given arguments are the same as UploadManager.upload_to_bundle_store().
Used when uploading from rest server."""
try:
# bundle_path = self._bundle_store.get_bundle_location(bundle.uuid)
is_url, is_fileobj, filename = self._interpret_source(source)
if is_url:
assert isinstance(source, str)
if git:
bundle_path = self._update_and_get_bundle_location(
bundle, is_directory=True)
self.write_git_repo(source, bundle_path)
else:
# If downloading from a URL, convert the source to a file object.
is_fileobj = True
source = (filename, urlopen_with_retry(source))
if is_fileobj:
source_filename, source_fileobj = cast(Tuple[str, IO[bytes]],
source)
source_ext = zip_util.get_archive_ext(source_filename)
if unpack and zip_util.path_is_archive(filename):
bundle_path = self._update_and_get_bundle_location(
bundle, is_directory=source_ext in ARCHIVE_EXTS_DIR)
self.write_fileobj(source_ext,
source_fileobj,
bundle_path,
unpack_archive=True)
else:
bundle_path = self._update_and_get_bundle_location(
bundle, is_directory=False)
self.write_fileobj(source_ext,
source_fileobj,
bundle_path,
unpack_archive=False)
except UsageError:
if FileSystems.exists(bundle_path):
path_util.remove(bundle_path)
raise
def _unpack_fileobj(self, source_filename, source_fileobj, dest_path, simplify_archive):
zip_util.unpack(zip_util.get_archive_ext(source_filename), source_fileobj, dest_path)
if simplify_archive:
self._simplify_archive(dest_path)
def _unpack_file(self, source_path, dest_path, remove_source, simplify_archive):
zip_util.unpack(zip_util.get_archive_ext(source_path), source_path, dest_path)
if remove_source:
path_util.remove(source_path)
if simplify_archive:
self._simplify_archive(dest_path)
def _unpack_fileobj(self, source_filename, source_fileobj, dest_path,
simplify_archive):
zip_util.unpack(zip_util.get_archive_ext(source_filename),
source_fileobj, dest_path)
if simplify_archive:
self._simplify_archive(dest_path)
def upload_to_bundle_store(
self,
bundle: Dict,
packed_source: Dict,
use_azure_blob_beta: bool,
destination_bundle_store=None,
):
"""
Bypass server upload. Upload from client directly to different blob storage (Azure, GCS, Disk storage).
Bypass server uploading is used in following situations:
# 1. The server set CODALAB_DEFAULT_BUNDLE_STORE_NAME
# 2. If the user specify `--store` and blob storage is on Azure
"""
need_bypass = True
bundle_store_uuid = None
# 1) Read destination store from --store if user has specified it
if destination_bundle_store is not None and destination_bundle_store != '':
storage_info = self._client.fetch_one(
'bundle_stores',
params={
'name': destination_bundle_store,
'include': ['uuid', 'storage_type', 'url'],
},
)
bundle_store_uuid = storage_info['uuid']
if storage_info['storage_type'] in (
StorageType.DISK_STORAGE.value, ):
need_bypass = False # The user specify --store to upload to disk storage
# 2) Pack the files to be uploaded
source_ext = zip_util.get_archive_ext(packed_source['filename'])
if packed_source['should_unpack'] and zip_util.path_is_archive(
packed_source['filename']):
unpack_before_upload = True
is_dir = source_ext in zip_util.ARCHIVE_EXTS_DIR
else:
unpack_before_upload = False
is_dir = False
# 3) Create a bundle location for the bundle
params = {'need_bypass': need_bypass, 'is_dir': is_dir}
data = self._client.add_bundle_location(bundle['id'],
bundle_store_uuid,
params)[0].get('attributes')
# 4) If bundle location has bundle_conn_str, we should bypass the server when uploading.
if data.get('bundle_conn_str', None) is not None:
# Mimic the rest server behavior
# decided the bundle type (file/directory) and decide whether need to unpack
bundle_conn_str = data.get('bundle_conn_str')
index_conn_str = data.get('index_conn_str')
bundle_url = data.get('bundle_url')
bundle_read_str = data.get('bundle_read_url', bundle_url)
try:
progress = FileTransferProgress('Sent ', f=self.stderr)
with closing(packed_source['fileobj']), progress:
self.upload_Azure_blob_storage(
fileobj=packed_source['fileobj'],
bundle_url=bundle_url,
bundle_conn_str=bundle_conn_str,
bundle_read_str=bundle_read_str,
index_conn_str=index_conn_str,
source_ext=source_ext,
should_unpack=unpack_before_upload,
progress_callback=progress.update,
)
except Exception as err:
self._client.update_bundle_state(
bundle['id'],
params={
'success': False,
'error_msg': f'Bypass server upload error. {err}',
},
)
raise err
else:
self._client.update_bundle_state(bundle['id'],
params={'success': True})
else:
# 5) Otherwise, upload the bundle directly through the server.
progress = FileTransferProgress('Sent ',
packed_source['filesize'],
f=self.stderr)
with closing(packed_source['fileobj']), progress:
self._client.upload_contents_blob(
bundle['id'],
fileobj=packed_source['fileobj'],
params={
'filename': packed_source['filename'],
'unpack': packed_source['should_unpack'],
'state_on_success': State.READY,
'finalize_on_success': True,
'use_azure_blob_beta': use_azure_blob_beta,
'store': destination_bundle_store or '',
},
progress_callback=progress.update,
)
def _fetch_bundle_contents_blob(uuid, path=''):
"""
API to download the contents of a bundle or a subpath within a bundle.
For directories this method always returns a tarred and gzipped archive of
the directory.
For files, if the request has an Accept-Encoding header containing gzip,
then the returned file is gzipped.
"""
byte_range = get_request_range()
head_lines = query_get_type(int, 'head', default=0)
tail_lines = query_get_type(int, 'tail', default=0)
max_line_length = query_get_type(int, 'max_line_length', default=128)
check_bundles_have_read_permission(local.model, request.user, [uuid])
bundle = local.model.get_bundle(uuid)
target_info = local.download_manager.get_target_info(uuid, path, 0)
if target_info is None:
abort(httplib.NOT_FOUND, 'Not found.')
# Figure out the file name.
if not path and bundle.metadata.name:
filename = bundle.metadata.name
else:
filename = target_info['name']
if target_info['type'] == 'directory':
if byte_range:
abort(httplib.BAD_REQUEST,
'Range not supported for directory blobs.')
if head_lines:
abort(httplib.BAD_REQUEST,
'Head not supported for directory blobs.')
# Always tar and gzip directories.
filename = filename + '.tar.gz'
fileobj = local.download_manager.stream_tarred_gzipped_directory(
uuid, path)
elif target_info['type'] == 'file':
gzipped = False
if not zip_util.path_is_archive(
filename) and request_accepts_gzip_encoding():
# Let's gzip to save bandwidth. The browser will transparently decode
# the file.
filename = filename + '.gz'
gzipped = True
if byte_range and (head_lines or tail_lines):
abort(httplib.BAD_REQUEST,
'Head and range not supported on the same request.')
elif byte_range:
start, end = byte_range
fileobj = local.download_manager.read_file_section(
uuid, path, start, end - start + 1, gzipped)
elif head_lines or tail_lines:
fileobj = local.download_manager.summarize_file(
uuid, path, head_lines, tail_lines, max_line_length, None,
gzipped)
else:
fileobj = local.download_manager.stream_file(uuid, path, gzipped)
else:
# Symlinks.
abort(httplib.FORBIDDEN, 'Cannot download files of this type.')
# Set headers.
mimetype, _ = mimetypes.guess_type(filename, strict=False)
response.set_header('Content-Type', mimetype or 'text/plain')
if zip_util.get_archive_ext(
filename) == '.gz' and request_accepts_gzip_encoding():
filename = zip_util.strip_archive_ext(filename)
response.set_header('Content-Encoding', 'gzip')
else:
response.set_header('Content-Encoding', 'identity')
response.set_header('Content-Disposition', 'filename="%s"' % filename)
return fileobj
def test_url_archive(self):
"""This URL should be categorized as an archive."""
path = "https://codalab.org/file.tar.gz"
self.assertEqual(path_is_archive(path), True)
self.assertEqual(get_archive_ext(path), ".tar.gz")
self.assertEqual(strip_archive_ext(path), "https://codalab.org/file")
def test_local_file_archive(self):
"""This local should be categorized as an archive."""
path = "/tmp/file.tar.gz"
self.assertEqual(path_is_archive(path), True)
self.assertEqual(get_archive_ext(path), ".tar.gz")
self.assertEqual(strip_archive_ext(path), "/tmp/file")
def upload(self, sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, uuid):
"""
|sources|: specifies the locations of the contents to upload. Each element is either a URL or a local path.
|follow_symlinks|: for local path(s), whether to follow (resolve) symlinks
|exclude_patterns|: for local path(s), don't upload these patterns (e.g., *.o)
|git|: for URL, whether |source| is a git repo to clone.
|unpack|: for each source in |sources|, whether to unpack it if it's an archive.
|remove_sources|: remove |sources|.
If |sources| contains one source, then the bundle contents will be that source.
Otherwise, the bundle contents will be a directory with each of the sources.
Exceptions:
- If |git|, then each source is replaced with the result of running 'git clone |source|'
- If |unpack| is True or a source is an archive (zip, tar.gz, etc.), then unpack the source.
Install the contents of the directory at |source| into
DATA_SUBDIRECTORY in a subdirectory named by a hash of the contents.
Return a (data_hash, metadata) pair, where the metadata is a dict mapping
keys to precomputed statistics about the new data directory.
"""
to_delete = []
# If just a single file, set the final path to be equal to that file
single_path = len(sources) == 1
# Determine which disk this will go on
disk_choice = self.ring.get_node(uuid)
final_path = os.path.join(self.partitions, disk_choice, self.DATA_SUBDIRECTORY, uuid)
if os.path.exists(final_path):
raise UsageError('Path %s already present in bundle store' % final_path)
# Only make if not there
elif not single_path:
path_util.make_directory(final_path)
# Paths to resources
subpaths = []
for source in sources:
# Where to save |source| to (might change this value if we unpack).
if not single_path:
subpath = os.path.join(final_path, os.path.basename(source))
else:
subpath = final_path
if remove_sources:
to_delete.append(source)
source_unpack = unpack and zip_util.path_is_archive(source)
if source_unpack and single_path:
# Load the file into the bundle store under the given path
subpath += zip_util.get_archive_ext(source)
if path_util.path_is_url(source):
# Download the URL.
print_util.open_line('BundleStore.upload: downloading %s to %s' % (source, subpath))
if git:
file_util.git_clone(source, subpath)
else:
file_util.download_url(source, subpath, print_status=True)
if source_unpack:
zip_util.unpack(subpath, zip_util.strip_archive_ext(subpath))
path_util.remove(subpath)
subpath = zip_util.strip_archive_ext(subpath)
print_util.clear_line()
else:
# Copy the local path.
source_path = path_util.normalize(source)
path_util.check_isvalid(source_path, 'upload')
# Recursively copy the directory into the BundleStore
print_util.open_line('BundleStore.upload: %s => %s' % (source_path, subpath))
if source_unpack:
zip_util.unpack(source_path, zip_util.strip_archive_ext(subpath))
subpath = zip_util.strip_archive_ext(subpath)
else:
if remove_sources:
path_util.rename(source_path, subpath)
else:
path_util.copy(source_path, subpath, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns)
print_util.clear_line()
subpaths.append(subpath)
dirs_and_files = None
if os.path.isdir(final_path):
dirs_and_files = path_util.recursive_ls(final_path)
else:
dirs_and_files = [], [final_path]
# Hash the contents of the bundle directory. Update the data_hash attribute
# for the bundle
print_util.open_line('BundleStore.upload: hashing %s' % final_path)
data_hash = '0x%s' % (path_util.hash_directory(final_path, dirs_and_files))
print_util.clear_line()
print_util.open_line('BundleStore.upload: computing size of %s' % final_path)
data_size = path_util.get_size(final_path, dirs_and_files)
print_util.clear_line()
# Delete paths.
for path in to_delete:
if os.path.exists(path):
path_util.remove(path)
# After this operation there should always be a directory at the final path.
assert (os.path.lexists(final_path)), 'Uploaded to %s failed!' % (final_path,)
return (data_hash, {'data_size': data_size})