def download(self, path, version=None, revision=None, mode=None, **kwargs):
if not path.identifier:
raise exceptions.NotFoundError(str(path))
if version is None:
# TODO Clean this up
# version could be 0 here
version = revision
# osf storage metadata will return a virtual path within the provider
resp = yield from self.make_signed_request(
'GET',
self.build_url(path.identifier, 'download', version=version, mode=mode),
expects=(200, ),
throws=exceptions.DownloadError,
)
data = yield from resp.json()
provider = self.make_provider(data['settings'])
name = data['data'].pop('name')
data['data']['path'] = yield from provider.validate_path('/' + data['data']['path'])
download_kwargs = {}
download_kwargs.update(kwargs)
download_kwargs.update(data['data'])
download_kwargs['displayName'] = kwargs.get('displayName', name)
return (yield from provider.download(**download_kwargs))
def validate_path(self, path, **kwargs):
split = path.rstrip('/').split('/')[1:]
wbpath = WaterButlerPath('/', _ids=(self.settings['project_id'], ), folder=True)
if split:
name_or_id = split.pop(0)
try:
article = yield from self._assert_contains_article(name_or_id)
except ValueError:
return wbpath.child(name_or_id, folder=False)
except exceptions.ProviderError as e:
if e.code not in (404, 401):
raise
return wbpath.child(name_or_id, folder=False)
wbpath = wbpath.child(article['title'], article['id'], folder=True)
if split:
provider = yield from self._make_article_provider(article['id'], check_parent=False)
try:
return (yield from provider.validate_path('/'.join([''] + split), parent=wbpath))
except exceptions.ProviderError as e:
if e.code not in (404, 401):
raise
return wbpath.child(split.pop(0), folder=False)
return wbpath
def validate_path(self, path, **kwargs):
split = path.rstrip('/').split('/')[1:]
wbpath = WaterButlerPath('/',
_ids=(self.settings['project_id'], ),
folder=True)
if split:
name_or_id = split.pop(0)
try:
article = yield from self._assert_contains_article(name_or_id)
except ValueError:
return wbpath.child(name_or_id, folder=False)
except exceptions.ProviderError as e:
if e.code not in (404, 401):
raise
return wbpath.child(name_or_id, folder=False)
wbpath = wbpath.child(article['title'], article['id'], folder=True)
if split:
provider = yield from self._make_article_provider(
article['id'], check_parent=False)
try:
return (yield from
provider.validate_path('/'.join([''] + split),
parent=wbpath))
except exceptions.ProviderError as e:
if e.code not in (404, 401):
raise
return wbpath.child(split.pop(0), folder=False)
return wbpath
def download(self, path, version=None, revision=None, mode=None, **kwargs):
if not path.identifier:
raise exceptions.NotFoundError(str(path))
if version is None:
# TODO Clean this up
# version could be 0 here
version = revision
# osf storage metadata will return a virtual path within the provider
resp = yield from self.make_signed_request(
'GET',
self.build_url(path.identifier, 'download', version=version, mode=mode),
expects=(200, ),
throws=exceptions.DownloadError,
)
data = yield from resp.json()
provider = self.make_provider(data['settings'])
name = data['data'].pop('name')
data['data']['path'] = yield from provider.validate_path('/' + data['data']['path'])
download_kwargs = {}
download_kwargs.update(kwargs)
download_kwargs.update(data['data'])
download_kwargs['displayName'] = kwargs.get('displayName', name)
return (yield from provider.download(**download_kwargs))
def upload(self, stream, path, **kwargs):
self._create_paths()
pending_name = str(uuid.uuid4())
provider = self.make_provider(self.settings)
local_pending_path = os.path.join(settings.FILE_PATH_PENDING, pending_name)
remote_pending_path = yield from provider.validate_path('/' + pending_name)
stream.add_writer('md5', streams.HashStreamWriter(hashlib.md5))
stream.add_writer('sha1', streams.HashStreamWriter(hashlib.sha1))
stream.add_writer('sha256', streams.HashStreamWriter(hashlib.sha256))
with open(local_pending_path, 'wb') as file_pointer:
stream.add_writer('file', file_pointer)
yield from provider.upload(stream, remote_pending_path, check_created=False, fetch_metadata=False, **kwargs)
complete_name = stream.writers['sha256'].hexdigest
local_complete_path = os.path.join(settings.FILE_PATH_COMPLETE, complete_name)
remote_complete_path = yield from provider.validate_path('/' + complete_name)
try:
metadata = yield from provider.metadata(remote_complete_path)
except exceptions.MetadataError as e:
if e.code != 404:
raise
metadata, _ = yield from provider.move(provider, remote_pending_path, remote_complete_path)
else:
yield from provider.delete(remote_pending_path)
finally:
metadata = metadata.serialized()
# Due to cross volume movement in unix we leverage shutil.move which properly handles this case.
# http://bytes.com/topic/python/answers/41652-errno-18-invalid-cross-device-link-using-os-rename#post157964
shutil.move(local_pending_path, local_complete_path)
response = yield from self.make_signed_request(
'POST',
self.build_url(path.parent.identifier, 'children'),
expects=(200, 201),
data=json.dumps({
'name': path.name,
'user': self.auth['id'],
'settings': self.settings['storage'],
'metadata': metadata,
'hashes': {
'md5': stream.writers['md5'].hexdigest,
'sha1': stream.writers['sha1'].hexdigest,
'sha256': stream.writers['sha256'].hexdigest,
},
'worker': {
'host': os.uname()[1],
# TODO: Include additional information
'address': None,
'version': self.__version__,
},
}),
headers={'Content-Type': 'application/json'},
)
created = response.status == 201
data = yield from response.json()
if settings.RUN_TASKS and data.pop('archive', True):
parity.main(
local_complete_path,
self.parity_credentials,
self.parity_settings,
)
backup.main(
local_complete_path,
data['version'],
self.build_url('hooks', 'metadata') + '/',
self.archive_credentials,
self.archive_settings,
)
name = path.name
metadata.update({
'name': name,
'md5': data['data']['md5'],
'path': data['data']['path'],
'sha256': data['data']['sha256'],
'version': data['data']['version'],
'downloads': data['data']['downloads'],
'checkout': data['data']['checkout'],
})
path._parts[-1]._id = metadata['path'].strip('/')
return OsfStorageFileMetadata(metadata, str(path)), created
def upload(self, stream, path, **kwargs):
self._create_paths()
pending_name = str(uuid.uuid4())
provider = self.make_provider(self.settings)
local_pending_path = os.path.join(settings.FILE_PATH_PENDING, pending_name)
remote_pending_path = yield from provider.validate_path('/' + pending_name)
stream.add_writer('md5', streams.HashStreamWriter(hashlib.md5))
stream.add_writer('sha1', streams.HashStreamWriter(hashlib.sha1))
stream.add_writer('sha256', streams.HashStreamWriter(hashlib.sha256))
with open(local_pending_path, 'wb') as file_pointer:
stream.add_writer('file', file_pointer)
yield from provider.upload(stream, remote_pending_path, check_created=False, fetch_metadata=False, **kwargs)
complete_name = stream.writers['sha256'].hexdigest
local_complete_path = os.path.join(settings.FILE_PATH_COMPLETE, complete_name)
remote_complete_path = yield from provider.validate_path('/' + complete_name)
try:
metadata = yield from provider.metadata(remote_complete_path)
except exceptions.MetadataError as e:
if e.code != 404:
raise
metadata, _ = yield from provider.move(provider, remote_pending_path, remote_complete_path)
else:
yield from provider.delete(remote_pending_path)
finally:
metadata = metadata.serialized()
# Due to cross volume movement in unix we leverage shutil.move which properly handles this case.
# http://bytes.com/topic/python/answers/41652-errno-18-invalid-cross-device-link-using-os-rename#post157964
shutil.move(local_pending_path, local_complete_path)
response = yield from self.make_signed_request(
'POST',
self.build_url(path.parent.identifier, 'children'),
expects=(200, 201),
data=json.dumps({
'name': path.name,
'user': self.auth['id'],
'settings': self.settings['storage'],
'metadata': metadata,
'hashes': {
'md5': stream.writers['md5'].hexdigest,
'sha1': stream.writers['sha1'].hexdigest,
'sha256': stream.writers['sha256'].hexdigest,
},
'worker': {
'host': os.uname()[1],
# TODO: Include additional information
'address': None,
'version': self.__version__,
},
}),
headers={'Content-Type': 'application/json'},
)
created = response.status == 201
data = yield from response.json()
if settings.RUN_TASKS and data.pop('archive', True):
parity.main(
local_complete_path,
self.parity_credentials,
self.parity_settings,
)
backup.main(
local_complete_path,
data['version'],
self.build_url('hooks', 'metadata') + '/',
self.archive_credentials,
self.archive_settings,
)
name = path.name
metadata.update({
'name': name,
'path': data['data']['path'],
'version': data['data']['version'],
'downloads': data['data']['downloads']
})
return OsfStorageFileMetadata(metadata, str(path)), created
def upload(self, stream, path, **kwargs):
self._create_paths()
pending_name = str(uuid.uuid4())
provider = self.make_provider(self.settings)
local_pending_path = os.path.join(settings.FILE_PATH_PENDING, pending_name)
remote_pending_path = yield from provider.validate_path("/" + pending_name)
stream.add_writer("md5", streams.HashStreamWriter(hashlib.md5))
stream.add_writer("sha1", streams.HashStreamWriter(hashlib.sha1))
stream.add_writer("sha256", streams.HashStreamWriter(hashlib.sha256))
with open(local_pending_path, "wb") as file_pointer:
stream.add_writer("file", file_pointer)
yield from provider.upload(stream, remote_pending_path, check_created=False, fetch_metadata=False, **kwargs)
complete_name = stream.writers["sha256"].hexdigest
local_complete_path = os.path.join(settings.FILE_PATH_COMPLETE, complete_name)
remote_complete_path = yield from provider.validate_path("/" + complete_name)
try:
metadata = yield from provider.metadata(remote_complete_path)
except exceptions.MetadataError as e:
if e.code != 404:
raise
metadata, _ = yield from provider.move(provider, remote_pending_path, remote_complete_path)
else:
yield from provider.delete(remote_pending_path)
finally:
metadata = metadata.serialized()
# Due to cross volume movement in unix we leverage shutil.move which properly handles this case.
# http://bytes.com/topic/python/answers/41652-errno-18-invalid-cross-device-link-using-os-rename#post157964
shutil.move(local_pending_path, local_complete_path)
response = yield from self.make_signed_request(
"POST",
self.build_url(path.parent.identifier, "children"),
expects=(200, 201),
data=json.dumps(
{
"name": path.name,
"user": self.auth["id"],
"settings": self.settings["storage"],
"metadata": metadata,
"hashes": {
"md5": stream.writers["md5"].hexdigest,
"sha1": stream.writers["sha1"].hexdigest,
"sha256": stream.writers["sha256"].hexdigest,
},
"worker": {
"host": os.uname()[1],
# TODO: Include additional information
"address": None,
"version": self.__version__,
},
}
),
headers={"Content-Type": "application/json"},
)
created = response.status == 201
data = yield from response.json()
if settings.RUN_TASKS and data.pop("archive", True):
parity.main(local_complete_path, self.parity_credentials, self.parity_settings)
backup.main(
local_complete_path,
data["version"],
self.build_url("hooks", "metadata") + "/",
self.archive_credentials,
self.archive_settings,
)
name = path.name
metadata.update(
{
"name": name,
"md5": data["data"]["md5"],
"path": data["data"]["path"],
"sha256": data["data"]["sha256"],
"version": data["data"]["version"],
"downloads": data["data"]["downloads"],
"checkout": data["data"]["checkout"],
}
)
path._parts[-1]._id = metadata["path"].strip("/")
return OsfStorageFileMetadata(metadata, str(path)), created