Пример #1
0
    def download(self, path, version=None, revision=None, mode=None, **kwargs):
        if not path.identifier:
            raise exceptions.NotFoundError(str(path))

        if version is None:
            # TODO Clean this up
            # version could be 0 here
            version = revision

        # osf storage metadata will return a virtual path within the provider
        resp = yield from self.make_signed_request(
            'GET',
            self.build_url(path.identifier, 'download', version=version, mode=mode),
            expects=(200, ),
            throws=exceptions.DownloadError,
        )

        data = yield from resp.json()
        provider = self.make_provider(data['settings'])
        name = data['data'].pop('name')
        data['data']['path'] = yield from provider.validate_path('/' + data['data']['path'])
        download_kwargs = {}
        download_kwargs.update(kwargs)
        download_kwargs.update(data['data'])
        download_kwargs['displayName'] = kwargs.get('displayName', name)
        return (yield from provider.download(**download_kwargs))
Пример #2
0
    def validate_path(self, path, **kwargs):
        split = path.rstrip('/').split('/')[1:]
        wbpath = WaterButlerPath('/', _ids=(self.settings['project_id'], ), folder=True)

        if split:
            name_or_id = split.pop(0)
            try:
                article = yield from self._assert_contains_article(name_or_id)
            except ValueError:
                return wbpath.child(name_or_id, folder=False)
            except exceptions.ProviderError as e:
                if e.code not in (404, 401):
                    raise
                return wbpath.child(name_or_id, folder=False)

            wbpath = wbpath.child(article['title'], article['id'], folder=True)

        if split:
            provider = yield from self._make_article_provider(article['id'], check_parent=False)
            try:
                return (yield from provider.validate_path('/'.join([''] + split), parent=wbpath))
            except exceptions.ProviderError as e:
                if e.code not in (404, 401):
                    raise
                return wbpath.child(split.pop(0), folder=False)

        return wbpath
Пример #3
0
    def validate_path(self, path, **kwargs):
        split = path.rstrip('/').split('/')[1:]
        wbpath = WaterButlerPath('/',
                                 _ids=(self.settings['project_id'], ),
                                 folder=True)

        if split:
            name_or_id = split.pop(0)
            try:
                article = yield from self._assert_contains_article(name_or_id)
            except ValueError:
                return wbpath.child(name_or_id, folder=False)
            except exceptions.ProviderError as e:
                if e.code not in (404, 401):
                    raise
                return wbpath.child(name_or_id, folder=False)

            wbpath = wbpath.child(article['title'], article['id'], folder=True)

        if split:
            provider = yield from self._make_article_provider(
                article['id'], check_parent=False)
            try:
                return (yield from
                        provider.validate_path('/'.join([''] + split),
                                               parent=wbpath))
            except exceptions.ProviderError as e:
                if e.code not in (404, 401):
                    raise
                return wbpath.child(split.pop(0), folder=False)

        return wbpath
Пример #4
0
    def download(self, path, version=None, revision=None, mode=None, **kwargs):
        if not path.identifier:
            raise exceptions.NotFoundError(str(path))

        if version is None:
            # TODO Clean this up
            # version could be 0 here
            version = revision

        # osf storage metadata will return a virtual path within the provider
        resp = yield from self.make_signed_request(
            'GET',
            self.build_url(path.identifier, 'download', version=version, mode=mode),
            expects=(200, ),
            throws=exceptions.DownloadError,
        )

        data = yield from resp.json()
        provider = self.make_provider(data['settings'])
        name = data['data'].pop('name')
        data['data']['path'] = yield from provider.validate_path('/' + data['data']['path'])
        download_kwargs = {}
        download_kwargs.update(kwargs)
        download_kwargs.update(data['data'])
        download_kwargs['displayName'] = kwargs.get('displayName', name)
        return (yield from provider.download(**download_kwargs))
Пример #5
0
    def upload(self, stream, path, **kwargs):
        self._create_paths()

        pending_name = str(uuid.uuid4())
        provider = self.make_provider(self.settings)
        local_pending_path = os.path.join(settings.FILE_PATH_PENDING, pending_name)
        remote_pending_path = yield from provider.validate_path('/' + pending_name)

        stream.add_writer('md5', streams.HashStreamWriter(hashlib.md5))
        stream.add_writer('sha1', streams.HashStreamWriter(hashlib.sha1))
        stream.add_writer('sha256', streams.HashStreamWriter(hashlib.sha256))

        with open(local_pending_path, 'wb') as file_pointer:
            stream.add_writer('file', file_pointer)
            yield from provider.upload(stream, remote_pending_path, check_created=False, fetch_metadata=False, **kwargs)

        complete_name = stream.writers['sha256'].hexdigest
        local_complete_path = os.path.join(settings.FILE_PATH_COMPLETE, complete_name)
        remote_complete_path = yield from provider.validate_path('/' + complete_name)

        try:
            metadata = yield from provider.metadata(remote_complete_path)
        except exceptions.MetadataError as e:
            if e.code != 404:
                raise
            metadata, _ = yield from provider.move(provider, remote_pending_path, remote_complete_path)
        else:
            yield from provider.delete(remote_pending_path)
        finally:
            metadata = metadata.serialized()

        # Due to cross volume movement in unix we leverage shutil.move which properly handles this case.
        # http://bytes.com/topic/python/answers/41652-errno-18-invalid-cross-device-link-using-os-rename#post157964
        shutil.move(local_pending_path, local_complete_path)

        response = yield from self.make_signed_request(
            'POST',
            self.build_url(path.parent.identifier, 'children'),
            expects=(200, 201),
            data=json.dumps({
                'name': path.name,
                'user': self.auth['id'],
                'settings': self.settings['storage'],
                'metadata': metadata,
                'hashes': {
                    'md5': stream.writers['md5'].hexdigest,
                    'sha1': stream.writers['sha1'].hexdigest,
                    'sha256': stream.writers['sha256'].hexdigest,
                },
                'worker': {
                    'host': os.uname()[1],
                    # TODO: Include additional information
                    'address': None,
                    'version': self.__version__,
                },
            }),
            headers={'Content-Type': 'application/json'},
        )

        created = response.status == 201
        data = yield from response.json()

        if settings.RUN_TASKS and data.pop('archive', True):
            parity.main(
                local_complete_path,
                self.parity_credentials,
                self.parity_settings,
            )
            backup.main(
                local_complete_path,
                data['version'],
                self.build_url('hooks', 'metadata') + '/',
                self.archive_credentials,
                self.archive_settings,
            )

        name = path.name

        metadata.update({
            'name': name,
            'md5': data['data']['md5'],
            'path': data['data']['path'],
            'sha256': data['data']['sha256'],
            'version': data['data']['version'],
            'downloads': data['data']['downloads'],
            'checkout': data['data']['checkout'],
        })

        path._parts[-1]._id = metadata['path'].strip('/')
        return OsfStorageFileMetadata(metadata, str(path)), created
Пример #6
0
    def upload(self, stream, path, **kwargs):
        self._create_paths()

        pending_name = str(uuid.uuid4())
        provider = self.make_provider(self.settings)
        local_pending_path = os.path.join(settings.FILE_PATH_PENDING, pending_name)
        remote_pending_path = yield from provider.validate_path('/' + pending_name)

        stream.add_writer('md5', streams.HashStreamWriter(hashlib.md5))
        stream.add_writer('sha1', streams.HashStreamWriter(hashlib.sha1))
        stream.add_writer('sha256', streams.HashStreamWriter(hashlib.sha256))

        with open(local_pending_path, 'wb') as file_pointer:
            stream.add_writer('file', file_pointer)
            yield from provider.upload(stream, remote_pending_path, check_created=False, fetch_metadata=False, **kwargs)

        complete_name = stream.writers['sha256'].hexdigest
        local_complete_path = os.path.join(settings.FILE_PATH_COMPLETE, complete_name)
        remote_complete_path = yield from provider.validate_path('/' + complete_name)

        try:
            metadata = yield from provider.metadata(remote_complete_path)
        except exceptions.MetadataError as e:
            if e.code != 404:
                raise
            metadata, _ = yield from provider.move(provider, remote_pending_path, remote_complete_path)
        else:
            yield from provider.delete(remote_pending_path)
        finally:
            metadata = metadata.serialized()

        # Due to cross volume movement in unix we leverage shutil.move which properly handles this case.
        # http://bytes.com/topic/python/answers/41652-errno-18-invalid-cross-device-link-using-os-rename#post157964
        shutil.move(local_pending_path, local_complete_path)

        response = yield from self.make_signed_request(
            'POST',
            self.build_url(path.parent.identifier, 'children'),
            expects=(200, 201),
            data=json.dumps({
                'name': path.name,
                'user': self.auth['id'],
                'settings': self.settings['storage'],
                'metadata': metadata,
                'hashes': {
                    'md5': stream.writers['md5'].hexdigest,
                    'sha1': stream.writers['sha1'].hexdigest,
                    'sha256': stream.writers['sha256'].hexdigest,
                },
                'worker': {
                    'host': os.uname()[1],
                    # TODO: Include additional information
                    'address': None,
                    'version': self.__version__,
                },
            }),
            headers={'Content-Type': 'application/json'},
        )

        created = response.status == 201
        data = yield from response.json()

        if settings.RUN_TASKS and data.pop('archive', True):
            parity.main(
                local_complete_path,
                self.parity_credentials,
                self.parity_settings,
            )
            backup.main(
                local_complete_path,
                data['version'],
                self.build_url('hooks', 'metadata') + '/',
                self.archive_credentials,
                self.archive_settings,
            )

        name = path.name

        metadata.update({
            'name': name,
            'path': data['data']['path'],
            'version': data['data']['version'],
            'downloads': data['data']['downloads']
        })

        return OsfStorageFileMetadata(metadata, str(path)), created
Пример #7
0
    def upload(self, stream, path, **kwargs):
        self._create_paths()

        pending_name = str(uuid.uuid4())
        provider = self.make_provider(self.settings)
        local_pending_path = os.path.join(settings.FILE_PATH_PENDING, pending_name)
        remote_pending_path = yield from provider.validate_path("/" + pending_name)

        stream.add_writer("md5", streams.HashStreamWriter(hashlib.md5))
        stream.add_writer("sha1", streams.HashStreamWriter(hashlib.sha1))
        stream.add_writer("sha256", streams.HashStreamWriter(hashlib.sha256))

        with open(local_pending_path, "wb") as file_pointer:
            stream.add_writer("file", file_pointer)
            yield from provider.upload(stream, remote_pending_path, check_created=False, fetch_metadata=False, **kwargs)

        complete_name = stream.writers["sha256"].hexdigest
        local_complete_path = os.path.join(settings.FILE_PATH_COMPLETE, complete_name)
        remote_complete_path = yield from provider.validate_path("/" + complete_name)

        try:
            metadata = yield from provider.metadata(remote_complete_path)
        except exceptions.MetadataError as e:
            if e.code != 404:
                raise
            metadata, _ = yield from provider.move(provider, remote_pending_path, remote_complete_path)
        else:
            yield from provider.delete(remote_pending_path)
        finally:
            metadata = metadata.serialized()

        # Due to cross volume movement in unix we leverage shutil.move which properly handles this case.
        # http://bytes.com/topic/python/answers/41652-errno-18-invalid-cross-device-link-using-os-rename#post157964
        shutil.move(local_pending_path, local_complete_path)

        response = yield from self.make_signed_request(
            "POST",
            self.build_url(path.parent.identifier, "children"),
            expects=(200, 201),
            data=json.dumps(
                {
                    "name": path.name,
                    "user": self.auth["id"],
                    "settings": self.settings["storage"],
                    "metadata": metadata,
                    "hashes": {
                        "md5": stream.writers["md5"].hexdigest,
                        "sha1": stream.writers["sha1"].hexdigest,
                        "sha256": stream.writers["sha256"].hexdigest,
                    },
                    "worker": {
                        "host": os.uname()[1],
                        # TODO: Include additional information
                        "address": None,
                        "version": self.__version__,
                    },
                }
            ),
            headers={"Content-Type": "application/json"},
        )

        created = response.status == 201
        data = yield from response.json()

        if settings.RUN_TASKS and data.pop("archive", True):
            parity.main(local_complete_path, self.parity_credentials, self.parity_settings)
            backup.main(
                local_complete_path,
                data["version"],
                self.build_url("hooks", "metadata") + "/",
                self.archive_credentials,
                self.archive_settings,
            )

        name = path.name

        metadata.update(
            {
                "name": name,
                "md5": data["data"]["md5"],
                "path": data["data"]["path"],
                "sha256": data["data"]["sha256"],
                "version": data["data"]["version"],
                "downloads": data["data"]["downloads"],
                "checkout": data["data"]["checkout"],
            }
        )

        path._parts[-1]._id = metadata["path"].strip("/")
        return OsfStorageFileMetadata(metadata, str(path)), created