def test_local_file_non_archive(self): """This local file should be categorized as not an archive.""" path = "/tmp/file.txt" self.assertEqual(path_is_archive(path), False) self.assertEqual(get_archive_ext(path), "") with self.assertRaises(UsageError): strip_archive_ext(path)
def test_url_non_archive(self): """This URL should be categorized as not an archive.""" path = "https://codalab.org/file.txt" self.assertEqual(path_is_archive(path), False) self.assertEqual(get_archive_ext(path), "") with self.assertRaises(UsageError): strip_archive_ext(path)
def _unpack_file(self, source_path, dest_path, remove_source, simplify_archive): zip_util.unpack(zip_util.get_archive_ext(source_path), source_path, dest_path) if remove_source: path_util.remove(source_path) if simplify_archive: self._simplify_archive(dest_path)
def get_blob(uuid, path=''): """ API to download the contents of a bundle or a subpath within a bundle. For directories this method always returns a tarred and gzipped archive of the directory. For files, if the request has an Accept-Encoding header containing gzip, then the returned file is gzipped. """ check_bundles_have_read_permission(local.model, request.user, [uuid]) bundle = local.model.get_bundle(uuid) target_info = local.download_manager.get_target_info(uuid, path, 0) if target_info is None: abort(httplib.NOT_FOUND, 'Not found.') # Figure out the file name. if not path and bundle.metadata.name: filename = bundle.metadata.name else: filename = target_info['name'] if target_info['type'] == 'directory': # Always tar and gzip directories. filename = filename + '.tar.gz' fileobj = local.download_manager.stream_tarred_gzipped_directory( uuid, path) elif target_info['type'] == 'file': if not zip_util.path_is_archive( filename) and request_accepts_gzip_encoding(): # Let's gzip to save bandwidth. The browser will transparently decode # the file. filename = filename + '.gz' fileobj = local.download_manager.stream_file(uuid, path, gzipped=True) else: fileobj = local.download_manager.stream_file(uuid, path, gzipped=False) else: # Symlinks. abort(httplib.FORBIDDEN, 'Cannot download files of this type.') # Set headers. mimetype, _ = mimetypes.guess_type(filename, strict=False) response.set_header('Content-Type', mimetype or 'text/plain') if zip_util.get_archive_ext( filename) == '.gz' and request_accepts_gzip_encoding(): filename = zip_util.strip_archive_ext(filename) response.set_header('Content-Encoding', 'gzip') else: response.set_header('Content-Encoding', 'identity') response.set_header('Content-Disposition', 'filename="%s"' % filename) return fileobj
def get_blob(uuid, path=''): """ API to download the contents of a bundle or a subpath within a bundle. For directories this method always returns a tarred and gzipped archive of the directory. For files, if the request has an Accept-Encoding header containing gzip, then the returned file is gzipped. """ check_bundles_have_read_permission(local.model, request.user, [uuid]) bundle = local.model.get_bundle(uuid) target_info = local.download_manager.get_target_info(uuid, path, 0) if target_info is None: abort(httplib.NOT_FOUND, 'Not found.') # Figure out the file name. if not path and bundle.metadata.name: filename = bundle.metadata.name else: filename = target_info['name'] if target_info['type'] == 'directory': # Always tar and gzip directories. filename = filename + '.tar.gz' fileobj = local.download_manager.stream_tarred_gzipped_directory(uuid, path) elif target_info['type'] == 'file': if not zip_util.path_is_archive(filename) and request_accepts_gzip_encoding(): # Let's gzip to save bandwidth. The browser will transparently decode # the file. filename = filename + '.gz' fileobj = local.download_manager.stream_file(uuid, path, gzipped=True) else: fileobj = local.download_manager.stream_file(uuid, path, gzipped=False) else: # Symlinks. abort(httplib.FORBIDDEN, 'Cannot download files of this type.') # Set headers. mimetype, _ = mimetypes.guess_type(filename, strict=False) response.set_header('Content-Type', mimetype or 'text/plain') if zip_util.get_archive_ext(filename) == '.gz' and request_accepts_gzip_encoding(): filename = zip_util.strip_archive_ext(filename) response.set_header('Content-Encoding', 'gzip') else: response.set_header('Content-Encoding', 'identity') response.set_header('Content-Disposition', 'filename="%s"' % filename) return fileobj
def upload_to_bundle_store(self, bundle: Bundle, source: Source, git: bool, unpack: bool): """Uploads the given source to the bundle store. Given arguments are the same as UploadManager.upload_to_bundle_store(). Used when uploading from rest server.""" try: # bundle_path = self._bundle_store.get_bundle_location(bundle.uuid) is_url, is_fileobj, filename = self._interpret_source(source) if is_url: assert isinstance(source, str) if git: bundle_path = self._update_and_get_bundle_location( bundle, is_directory=True) self.write_git_repo(source, bundle_path) else: # If downloading from a URL, convert the source to a file object. is_fileobj = True source = (filename, urlopen_with_retry(source)) if is_fileobj: source_filename, source_fileobj = cast(Tuple[str, IO[bytes]], source) source_ext = zip_util.get_archive_ext(source_filename) if unpack and zip_util.path_is_archive(filename): bundle_path = self._update_and_get_bundle_location( bundle, is_directory=source_ext in ARCHIVE_EXTS_DIR) self.write_fileobj(source_ext, source_fileobj, bundle_path, unpack_archive=True) else: bundle_path = self._update_and_get_bundle_location( bundle, is_directory=False) self.write_fileobj(source_ext, source_fileobj, bundle_path, unpack_archive=False) except UsageError: if FileSystems.exists(bundle_path): path_util.remove(bundle_path) raise
def _unpack_fileobj(self, source_filename, source_fileobj, dest_path, simplify_archive): zip_util.unpack(zip_util.get_archive_ext(source_filename), source_fileobj, dest_path) if simplify_archive: self._simplify_archive(dest_path)
def upload_to_bundle_store( self, bundle: Dict, packed_source: Dict, use_azure_blob_beta: bool, destination_bundle_store=None, ): """ Bypass server upload. Upload from client directly to different blob storage (Azure, GCS, Disk storage). Bypass server uploading is used in following situations: # 1. The server set CODALAB_DEFAULT_BUNDLE_STORE_NAME # 2. If the user specify `--store` and blob storage is on Azure """ need_bypass = True bundle_store_uuid = None # 1) Read destination store from --store if user has specified it if destination_bundle_store is not None and destination_bundle_store != '': storage_info = self._client.fetch_one( 'bundle_stores', params={ 'name': destination_bundle_store, 'include': ['uuid', 'storage_type', 'url'], }, ) bundle_store_uuid = storage_info['uuid'] if storage_info['storage_type'] in ( StorageType.DISK_STORAGE.value, ): need_bypass = False # The user specify --store to upload to disk storage # 2) Pack the files to be uploaded source_ext = zip_util.get_archive_ext(packed_source['filename']) if packed_source['should_unpack'] and zip_util.path_is_archive( packed_source['filename']): unpack_before_upload = True is_dir = source_ext in zip_util.ARCHIVE_EXTS_DIR else: unpack_before_upload = False is_dir = False # 3) Create a bundle location for the bundle params = {'need_bypass': need_bypass, 'is_dir': is_dir} data = self._client.add_bundle_location(bundle['id'], bundle_store_uuid, params)[0].get('attributes') # 4) If bundle location has bundle_conn_str, we should bypass the server when uploading. if data.get('bundle_conn_str', None) is not None: # Mimic the rest server behavior # decided the bundle type (file/directory) and decide whether need to unpack bundle_conn_str = data.get('bundle_conn_str') index_conn_str = data.get('index_conn_str') bundle_url = data.get('bundle_url') bundle_read_str = data.get('bundle_read_url', bundle_url) try: progress = FileTransferProgress('Sent ', f=self.stderr) with closing(packed_source['fileobj']), progress: self.upload_Azure_blob_storage( fileobj=packed_source['fileobj'], bundle_url=bundle_url, bundle_conn_str=bundle_conn_str, bundle_read_str=bundle_read_str, index_conn_str=index_conn_str, source_ext=source_ext, should_unpack=unpack_before_upload, progress_callback=progress.update, ) except Exception as err: self._client.update_bundle_state( bundle['id'], params={ 'success': False, 'error_msg': f'Bypass server upload error. {err}', }, ) raise err else: self._client.update_bundle_state(bundle['id'], params={'success': True}) else: # 5) Otherwise, upload the bundle directly through the server. progress = FileTransferProgress('Sent ', packed_source['filesize'], f=self.stderr) with closing(packed_source['fileobj']), progress: self._client.upload_contents_blob( bundle['id'], fileobj=packed_source['fileobj'], params={ 'filename': packed_source['filename'], 'unpack': packed_source['should_unpack'], 'state_on_success': State.READY, 'finalize_on_success': True, 'use_azure_blob_beta': use_azure_blob_beta, 'store': destination_bundle_store or '', }, progress_callback=progress.update, )
def _fetch_bundle_contents_blob(uuid, path=''): """ API to download the contents of a bundle or a subpath within a bundle. For directories this method always returns a tarred and gzipped archive of the directory. For files, if the request has an Accept-Encoding header containing gzip, then the returned file is gzipped. """ byte_range = get_request_range() head_lines = query_get_type(int, 'head', default=0) tail_lines = query_get_type(int, 'tail', default=0) max_line_length = query_get_type(int, 'max_line_length', default=128) check_bundles_have_read_permission(local.model, request.user, [uuid]) bundle = local.model.get_bundle(uuid) target_info = local.download_manager.get_target_info(uuid, path, 0) if target_info is None: abort(httplib.NOT_FOUND, 'Not found.') # Figure out the file name. if not path and bundle.metadata.name: filename = bundle.metadata.name else: filename = target_info['name'] if target_info['type'] == 'directory': if byte_range: abort(httplib.BAD_REQUEST, 'Range not supported for directory blobs.') if head_lines: abort(httplib.BAD_REQUEST, 'Head not supported for directory blobs.') # Always tar and gzip directories. filename = filename + '.tar.gz' fileobj = local.download_manager.stream_tarred_gzipped_directory( uuid, path) elif target_info['type'] == 'file': gzipped = False if not zip_util.path_is_archive( filename) and request_accepts_gzip_encoding(): # Let's gzip to save bandwidth. The browser will transparently decode # the file. filename = filename + '.gz' gzipped = True if byte_range and (head_lines or tail_lines): abort(httplib.BAD_REQUEST, 'Head and range not supported on the same request.') elif byte_range: start, end = byte_range fileobj = local.download_manager.read_file_section( uuid, path, start, end - start + 1, gzipped) elif head_lines or tail_lines: fileobj = local.download_manager.summarize_file( uuid, path, head_lines, tail_lines, max_line_length, None, gzipped) else: fileobj = local.download_manager.stream_file(uuid, path, gzipped) else: # Symlinks. abort(httplib.FORBIDDEN, 'Cannot download files of this type.') # Set headers. mimetype, _ = mimetypes.guess_type(filename, strict=False) response.set_header('Content-Type', mimetype or 'text/plain') if zip_util.get_archive_ext( filename) == '.gz' and request_accepts_gzip_encoding(): filename = zip_util.strip_archive_ext(filename) response.set_header('Content-Encoding', 'gzip') else: response.set_header('Content-Encoding', 'identity') response.set_header('Content-Disposition', 'filename="%s"' % filename) return fileobj
def test_url_archive(self): """This URL should be categorized as an archive.""" path = "https://codalab.org/file.tar.gz" self.assertEqual(path_is_archive(path), True) self.assertEqual(get_archive_ext(path), ".tar.gz") self.assertEqual(strip_archive_ext(path), "https://codalab.org/file")
def test_local_file_archive(self): """This local should be categorized as an archive.""" path = "/tmp/file.tar.gz" self.assertEqual(path_is_archive(path), True) self.assertEqual(get_archive_ext(path), ".tar.gz") self.assertEqual(strip_archive_ext(path), "/tmp/file")
def upload(self, sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, uuid): """ |sources|: specifies the locations of the contents to upload. Each element is either a URL or a local path. |follow_symlinks|: for local path(s), whether to follow (resolve) symlinks |exclude_patterns|: for local path(s), don't upload these patterns (e.g., *.o) |git|: for URL, whether |source| is a git repo to clone. |unpack|: for each source in |sources|, whether to unpack it if it's an archive. |remove_sources|: remove |sources|. If |sources| contains one source, then the bundle contents will be that source. Otherwise, the bundle contents will be a directory with each of the sources. Exceptions: - If |git|, then each source is replaced with the result of running 'git clone |source|' - If |unpack| is True or a source is an archive (zip, tar.gz, etc.), then unpack the source. Install the contents of the directory at |source| into DATA_SUBDIRECTORY in a subdirectory named by a hash of the contents. Return a (data_hash, metadata) pair, where the metadata is a dict mapping keys to precomputed statistics about the new data directory. """ to_delete = [] # If just a single file, set the final path to be equal to that file single_path = len(sources) == 1 # Determine which disk this will go on disk_choice = self.ring.get_node(uuid) final_path = os.path.join(self.partitions, disk_choice, self.DATA_SUBDIRECTORY, uuid) if os.path.exists(final_path): raise UsageError('Path %s already present in bundle store' % final_path) # Only make if not there elif not single_path: path_util.make_directory(final_path) # Paths to resources subpaths = [] for source in sources: # Where to save |source| to (might change this value if we unpack). if not single_path: subpath = os.path.join(final_path, os.path.basename(source)) else: subpath = final_path if remove_sources: to_delete.append(source) source_unpack = unpack and zip_util.path_is_archive(source) if source_unpack and single_path: # Load the file into the bundle store under the given path subpath += zip_util.get_archive_ext(source) if path_util.path_is_url(source): # Download the URL. print_util.open_line('BundleStore.upload: downloading %s to %s' % (source, subpath)) if git: file_util.git_clone(source, subpath) else: file_util.download_url(source, subpath, print_status=True) if source_unpack: zip_util.unpack(subpath, zip_util.strip_archive_ext(subpath)) path_util.remove(subpath) subpath = zip_util.strip_archive_ext(subpath) print_util.clear_line() else: # Copy the local path. source_path = path_util.normalize(source) path_util.check_isvalid(source_path, 'upload') # Recursively copy the directory into the BundleStore print_util.open_line('BundleStore.upload: %s => %s' % (source_path, subpath)) if source_unpack: zip_util.unpack(source_path, zip_util.strip_archive_ext(subpath)) subpath = zip_util.strip_archive_ext(subpath) else: if remove_sources: path_util.rename(source_path, subpath) else: path_util.copy(source_path, subpath, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns) print_util.clear_line() subpaths.append(subpath) dirs_and_files = None if os.path.isdir(final_path): dirs_and_files = path_util.recursive_ls(final_path) else: dirs_and_files = [], [final_path] # Hash the contents of the bundle directory. Update the data_hash attribute # for the bundle print_util.open_line('BundleStore.upload: hashing %s' % final_path) data_hash = '0x%s' % (path_util.hash_directory(final_path, dirs_and_files)) print_util.clear_line() print_util.open_line('BundleStore.upload: computing size of %s' % final_path) data_size = path_util.get_size(final_path, dirs_and_files) print_util.clear_line() # Delete paths. for path in to_delete: if os.path.exists(path): path_util.remove(path) # After this operation there should always be a directory at the final path. assert (os.path.lexists(final_path)), 'Uploaded to %s failed!' % (final_path,) return (data_hash, {'data_size': data_size})