Exemple #1
0
 def test_local_file_non_archive(self):
     """This local file should be categorized as not an archive."""
     path = "/tmp/file.txt"
     self.assertEqual(path_is_archive(path), False)
     self.assertEqual(get_archive_ext(path), "")
     with self.assertRaises(UsageError):
         strip_archive_ext(path)
Exemple #2
0
 def test_url_non_archive(self):
     """This URL should be categorized as not an archive."""
     path = "https://codalab.org/file.txt"
     self.assertEqual(path_is_archive(path), False)
     self.assertEqual(get_archive_ext(path), "")
     with self.assertRaises(UsageError):
         strip_archive_ext(path)
Exemple #3
0
 def _unpack_file(self, source_path, dest_path, remove_source,
                  simplify_archive):
     zip_util.unpack(zip_util.get_archive_ext(source_path), source_path,
                     dest_path)
     if remove_source:
         path_util.remove(source_path)
     if simplify_archive:
         self._simplify_archive(dest_path)
Exemple #4
0
def get_blob(uuid, path=''):
    """
    API to download the contents of a bundle or a subpath within a bundle.

    For directories this method always returns a tarred and gzipped archive of
    the directory.

    For files, if the request has an Accept-Encoding header containing gzip,
    then the returned file is gzipped.
    """
    check_bundles_have_read_permission(local.model, request.user, [uuid])
    bundle = local.model.get_bundle(uuid)

    target_info = local.download_manager.get_target_info(uuid, path, 0)
    if target_info is None:
        abort(httplib.NOT_FOUND, 'Not found.')

    # Figure out the file name.
    if not path and bundle.metadata.name:
        filename = bundle.metadata.name
    else:
        filename = target_info['name']

    if target_info['type'] == 'directory':
        # Always tar and gzip directories.
        filename = filename + '.tar.gz'
        fileobj = local.download_manager.stream_tarred_gzipped_directory(
            uuid, path)
    elif target_info['type'] == 'file':
        if not zip_util.path_is_archive(
                filename) and request_accepts_gzip_encoding():
            # Let's gzip to save bandwidth. The browser will transparently decode
            # the file.
            filename = filename + '.gz'
            fileobj = local.download_manager.stream_file(uuid,
                                                         path,
                                                         gzipped=True)
        else:
            fileobj = local.download_manager.stream_file(uuid,
                                                         path,
                                                         gzipped=False)
    else:
        # Symlinks.
        abort(httplib.FORBIDDEN, 'Cannot download files of this type.')

    # Set headers.
    mimetype, _ = mimetypes.guess_type(filename, strict=False)
    response.set_header('Content-Type', mimetype or 'text/plain')
    if zip_util.get_archive_ext(
            filename) == '.gz' and request_accepts_gzip_encoding():
        filename = zip_util.strip_archive_ext(filename)
        response.set_header('Content-Encoding', 'gzip')
    else:
        response.set_header('Content-Encoding', 'identity')
    response.set_header('Content-Disposition', 'filename="%s"' % filename)

    return fileobj
Exemple #5
0
def get_blob(uuid, path=''):
    """
    API to download the contents of a bundle or a subpath within a bundle.

    For directories this method always returns a tarred and gzipped archive of
    the directory.

    For files, if the request has an Accept-Encoding header containing gzip,
    then the returned file is gzipped.
    """
    check_bundles_have_read_permission(local.model, request.user, [uuid])
    bundle = local.model.get_bundle(uuid)

    target_info = local.download_manager.get_target_info(uuid, path, 0)
    if target_info is None:
        abort(httplib.NOT_FOUND, 'Not found.')

    # Figure out the file name.
    if not path and bundle.metadata.name:
        filename = bundle.metadata.name
    else:
        filename = target_info['name']

    if target_info['type'] == 'directory':
        # Always tar and gzip directories.
        filename = filename + '.tar.gz'
        fileobj = local.download_manager.stream_tarred_gzipped_directory(uuid, path)
    elif target_info['type'] == 'file':
        if not zip_util.path_is_archive(filename) and request_accepts_gzip_encoding():
            # Let's gzip to save bandwidth. The browser will transparently decode
            # the file.
            filename = filename + '.gz'
            fileobj = local.download_manager.stream_file(uuid, path, gzipped=True)
        else:
            fileobj = local.download_manager.stream_file(uuid, path, gzipped=False)
    else:
        # Symlinks.
        abort(httplib.FORBIDDEN, 'Cannot download files of this type.')
    
    # Set headers.
    mimetype, _ = mimetypes.guess_type(filename, strict=False)
    response.set_header('Content-Type', mimetype or 'text/plain')
    if zip_util.get_archive_ext(filename) == '.gz' and request_accepts_gzip_encoding():
        filename = zip_util.strip_archive_ext(filename)
        response.set_header('Content-Encoding', 'gzip')
    else:
        response.set_header('Content-Encoding', 'identity')
    response.set_header('Content-Disposition', 'filename="%s"' % filename)

    return fileobj
    def upload_to_bundle_store(self, bundle: Bundle, source: Source, git: bool,
                               unpack: bool):
        """Uploads the given source to the bundle store.
        Given arguments are the same as UploadManager.upload_to_bundle_store().
        Used when uploading from rest server."""
        try:
            # bundle_path = self._bundle_store.get_bundle_location(bundle.uuid)
            is_url, is_fileobj, filename = self._interpret_source(source)
            if is_url:
                assert isinstance(source, str)
                if git:
                    bundle_path = self._update_and_get_bundle_location(
                        bundle, is_directory=True)
                    self.write_git_repo(source, bundle_path)
                else:
                    # If downloading from a URL, convert the source to a file object.
                    is_fileobj = True
                    source = (filename, urlopen_with_retry(source))
            if is_fileobj:
                source_filename, source_fileobj = cast(Tuple[str, IO[bytes]],
                                                       source)
                source_ext = zip_util.get_archive_ext(source_filename)
                if unpack and zip_util.path_is_archive(filename):
                    bundle_path = self._update_and_get_bundle_location(
                        bundle, is_directory=source_ext in ARCHIVE_EXTS_DIR)
                    self.write_fileobj(source_ext,
                                       source_fileobj,
                                       bundle_path,
                                       unpack_archive=True)
                else:
                    bundle_path = self._update_and_get_bundle_location(
                        bundle, is_directory=False)
                    self.write_fileobj(source_ext,
                                       source_fileobj,
                                       bundle_path,
                                       unpack_archive=False)

        except UsageError:
            if FileSystems.exists(bundle_path):
                path_util.remove(bundle_path)
            raise
 def _unpack_fileobj(self, source_filename, source_fileobj, dest_path, simplify_archive):
     zip_util.unpack(zip_util.get_archive_ext(source_filename), source_fileobj, dest_path)
     if simplify_archive:
         self._simplify_archive(dest_path)
 def _unpack_file(self, source_path, dest_path, remove_source, simplify_archive):
     zip_util.unpack(zip_util.get_archive_ext(source_path), source_path, dest_path)
     if remove_source:
         path_util.remove(source_path)
     if simplify_archive:
         self._simplify_archive(dest_path)
Exemple #9
0
 def _unpack_fileobj(self, source_filename, source_fileobj, dest_path,
                     simplify_archive):
     zip_util.unpack(zip_util.get_archive_ext(source_filename),
                     source_fileobj, dest_path)
     if simplify_archive:
         self._simplify_archive(dest_path)
    def upload_to_bundle_store(
        self,
        bundle: Dict,
        packed_source: Dict,
        use_azure_blob_beta: bool,
        destination_bundle_store=None,
    ):
        """
        Bypass server upload. Upload from client directly to different blob storage (Azure, GCS, Disk storage).
        Bypass server uploading is used in following situations:
        # 1. The server set CODALAB_DEFAULT_BUNDLE_STORE_NAME
        # 2. If the user specify `--store` and blob storage is on Azure
        """

        need_bypass = True
        bundle_store_uuid = None
        # 1) Read destination store from --store if user has specified it
        if destination_bundle_store is not None and destination_bundle_store != '':
            storage_info = self._client.fetch_one(
                'bundle_stores',
                params={
                    'name': destination_bundle_store,
                    'include': ['uuid', 'storage_type', 'url'],
                },
            )
            bundle_store_uuid = storage_info['uuid']
            if storage_info['storage_type'] in (
                    StorageType.DISK_STORAGE.value, ):
                need_bypass = False  # The user specify --store to upload to disk storage

        # 2) Pack the files to be uploaded
        source_ext = zip_util.get_archive_ext(packed_source['filename'])
        if packed_source['should_unpack'] and zip_util.path_is_archive(
                packed_source['filename']):
            unpack_before_upload = True
            is_dir = source_ext in zip_util.ARCHIVE_EXTS_DIR
        else:
            unpack_before_upload = False
            is_dir = False

        # 3) Create a bundle location for the bundle
        params = {'need_bypass': need_bypass, 'is_dir': is_dir}
        data = self._client.add_bundle_location(bundle['id'],
                                                bundle_store_uuid,
                                                params)[0].get('attributes')

        # 4) If bundle location has bundle_conn_str, we should bypass the server when uploading.
        if data.get('bundle_conn_str', None) is not None:
            # Mimic the rest server behavior
            # decided the bundle type (file/directory) and decide whether need to unpack
            bundle_conn_str = data.get('bundle_conn_str')
            index_conn_str = data.get('index_conn_str')
            bundle_url = data.get('bundle_url')
            bundle_read_str = data.get('bundle_read_url', bundle_url)
            try:
                progress = FileTransferProgress('Sent ', f=self.stderr)
                with closing(packed_source['fileobj']), progress:
                    self.upload_Azure_blob_storage(
                        fileobj=packed_source['fileobj'],
                        bundle_url=bundle_url,
                        bundle_conn_str=bundle_conn_str,
                        bundle_read_str=bundle_read_str,
                        index_conn_str=index_conn_str,
                        source_ext=source_ext,
                        should_unpack=unpack_before_upload,
                        progress_callback=progress.update,
                    )
            except Exception as err:
                self._client.update_bundle_state(
                    bundle['id'],
                    params={
                        'success': False,
                        'error_msg': f'Bypass server upload error. {err}',
                    },
                )
                raise err
            else:
                self._client.update_bundle_state(bundle['id'],
                                                 params={'success': True})
        else:
            # 5) Otherwise, upload the bundle directly through the server.
            progress = FileTransferProgress('Sent ',
                                            packed_source['filesize'],
                                            f=self.stderr)
            with closing(packed_source['fileobj']), progress:
                self._client.upload_contents_blob(
                    bundle['id'],
                    fileobj=packed_source['fileobj'],
                    params={
                        'filename': packed_source['filename'],
                        'unpack': packed_source['should_unpack'],
                        'state_on_success': State.READY,
                        'finalize_on_success': True,
                        'use_azure_blob_beta': use_azure_blob_beta,
                        'store': destination_bundle_store or '',
                    },
                    progress_callback=progress.update,
                )
Exemple #11
0
def _fetch_bundle_contents_blob(uuid, path=''):
    """
    API to download the contents of a bundle or a subpath within a bundle.

    For directories this method always returns a tarred and gzipped archive of
    the directory.

    For files, if the request has an Accept-Encoding header containing gzip,
    then the returned file is gzipped.
    """
    byte_range = get_request_range()
    head_lines = query_get_type(int, 'head', default=0)
    tail_lines = query_get_type(int, 'tail', default=0)
    max_line_length = query_get_type(int, 'max_line_length', default=128)
    check_bundles_have_read_permission(local.model, request.user, [uuid])
    bundle = local.model.get_bundle(uuid)

    target_info = local.download_manager.get_target_info(uuid, path, 0)
    if target_info is None:
        abort(httplib.NOT_FOUND, 'Not found.')

    # Figure out the file name.
    if not path and bundle.metadata.name:
        filename = bundle.metadata.name
    else:
        filename = target_info['name']

    if target_info['type'] == 'directory':
        if byte_range:
            abort(httplib.BAD_REQUEST,
                  'Range not supported for directory blobs.')
        if head_lines:
            abort(httplib.BAD_REQUEST,
                  'Head not supported for directory blobs.')
        # Always tar and gzip directories.
        filename = filename + '.tar.gz'
        fileobj = local.download_manager.stream_tarred_gzipped_directory(
            uuid, path)
    elif target_info['type'] == 'file':
        gzipped = False
        if not zip_util.path_is_archive(
                filename) and request_accepts_gzip_encoding():
            # Let's gzip to save bandwidth. The browser will transparently decode
            # the file.
            filename = filename + '.gz'
            gzipped = True

        if byte_range and (head_lines or tail_lines):
            abort(httplib.BAD_REQUEST,
                  'Head and range not supported on the same request.')
        elif byte_range:
            start, end = byte_range
            fileobj = local.download_manager.read_file_section(
                uuid, path, start, end - start + 1, gzipped)
        elif head_lines or tail_lines:
            fileobj = local.download_manager.summarize_file(
                uuid, path, head_lines, tail_lines, max_line_length, None,
                gzipped)
        else:
            fileobj = local.download_manager.stream_file(uuid, path, gzipped)
    else:
        # Symlinks.
        abort(httplib.FORBIDDEN, 'Cannot download files of this type.')

    # Set headers.
    mimetype, _ = mimetypes.guess_type(filename, strict=False)
    response.set_header('Content-Type', mimetype or 'text/plain')
    if zip_util.get_archive_ext(
            filename) == '.gz' and request_accepts_gzip_encoding():
        filename = zip_util.strip_archive_ext(filename)
        response.set_header('Content-Encoding', 'gzip')
    else:
        response.set_header('Content-Encoding', 'identity')
    response.set_header('Content-Disposition', 'filename="%s"' % filename)

    return fileobj
Exemple #12
0
 def test_url_archive(self):
     """This URL should be categorized as an archive."""
     path = "https://codalab.org/file.tar.gz"
     self.assertEqual(path_is_archive(path), True)
     self.assertEqual(get_archive_ext(path), ".tar.gz")
     self.assertEqual(strip_archive_ext(path), "https://codalab.org/file")
Exemple #13
0
 def test_local_file_archive(self):
     """This local should be categorized as an archive."""
     path = "/tmp/file.tar.gz"
     self.assertEqual(path_is_archive(path), True)
     self.assertEqual(get_archive_ext(path), ".tar.gz")
     self.assertEqual(strip_archive_ext(path), "/tmp/file")
    def upload(self, sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, uuid):
        """
        |sources|: specifies the locations of the contents to upload.  Each element is either a URL or a local path.
        |follow_symlinks|: for local path(s), whether to follow (resolve) symlinks
        |exclude_patterns|: for local path(s), don't upload these patterns (e.g., *.o)
        |git|: for URL, whether |source| is a git repo to clone.
        |unpack|: for each source in |sources|, whether to unpack it if it's an archive.
        |remove_sources|: remove |sources|.

        If |sources| contains one source, then the bundle contents will be that source.
        Otherwise, the bundle contents will be a directory with each of the sources.
        Exceptions:
        - If |git|, then each source is replaced with the result of running 'git clone |source|'
        - If |unpack| is True or a source is an archive (zip, tar.gz, etc.), then unpack the source.

        Install the contents of the directory at |source| into
        DATA_SUBDIRECTORY in a subdirectory named by a hash of the contents.

        Return a (data_hash, metadata) pair, where the metadata is a dict mapping
        keys to precomputed statistics about the new data directory.
        """
        to_delete = []

        # If just a single file, set the final path to be equal to that file
        single_path = len(sources) == 1

        # Determine which disk this will go on
        disk_choice = self.ring.get_node(uuid)

        final_path = os.path.join(self.partitions, disk_choice, self.DATA_SUBDIRECTORY, uuid)
        if os.path.exists(final_path):
            raise UsageError('Path %s already present in bundle store' % final_path)
        # Only make if not there
        elif not single_path:
            path_util.make_directory(final_path)

        # Paths to resources
        subpaths = []

        for source in sources:
            # Where to save |source| to (might change this value if we unpack).
            if not single_path:
                subpath = os.path.join(final_path, os.path.basename(source))
            else:
                subpath = final_path

            if remove_sources:
                to_delete.append(source)
            source_unpack = unpack and zip_util.path_is_archive(source)

            if source_unpack and single_path:
                # Load the file into the bundle store under the given path
                subpath += zip_util.get_archive_ext(source)

            if path_util.path_is_url(source):
                # Download the URL.
                print_util.open_line('BundleStore.upload: downloading %s to %s' % (source, subpath))
                if git:
                    file_util.git_clone(source, subpath)
                else:
                    file_util.download_url(source, subpath, print_status=True)
                    if source_unpack:
                        zip_util.unpack(subpath, zip_util.strip_archive_ext(subpath))
                        path_util.remove(subpath)
                        subpath = zip_util.strip_archive_ext(subpath)
                print_util.clear_line()
            else:
                # Copy the local path.
                source_path = path_util.normalize(source)
                path_util.check_isvalid(source_path, 'upload')

                # Recursively copy the directory into the BundleStore
                print_util.open_line('BundleStore.upload: %s => %s' % (source_path, subpath))
                if source_unpack:
                    zip_util.unpack(source_path, zip_util.strip_archive_ext(subpath))
                    subpath = zip_util.strip_archive_ext(subpath)
                else:
                    if remove_sources:
                        path_util.rename(source_path, subpath)
                    else:
                        path_util.copy(source_path, subpath, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns)
                print_util.clear_line()

            subpaths.append(subpath)

        dirs_and_files = None
        if os.path.isdir(final_path):
            dirs_and_files = path_util.recursive_ls(final_path)
        else:
            dirs_and_files = [], [final_path]

        # Hash the contents of the bundle directory. Update the data_hash attribute
        # for the bundle
        print_util.open_line('BundleStore.upload: hashing %s' % final_path)
        data_hash = '0x%s' % (path_util.hash_directory(final_path, dirs_and_files))
        print_util.clear_line()
        print_util.open_line('BundleStore.upload: computing size of %s' % final_path)
        data_size = path_util.get_size(final_path, dirs_and_files)
        print_util.clear_line()

        # Delete paths.
        for path in to_delete:
            if os.path.exists(path):
                path_util.remove(path)

        # After this operation there should always be a directory at the final path.
        assert (os.path.lexists(final_path)), 'Uploaded to %s failed!' % (final_path,)
        return (data_hash, {'data_size': data_size})