def test_package_objects(client, client2, dataset):
"""
Only super-users are allowed to create/modify package sources/files.
"""
pkg = DataPackage('Some Video', package_type='Video')
assert not pkg.exists
# some files (local for now)
source = File(name='My Source File',
s3_key='s3/source',
s3_bucket='my-bucket',
file_type="JSON",
size=1000)
file = File(name='My File',
s3_key='s3/file',
s3_bucket='my-bucket',
file_type="CSV",
size=1000)
view = File(name='My View File',
s3_key='s3/view',
s3_bucket='my-bucket',
file_type="NIFTI",
size=1000)
# get dataset (but as different client)
dataset2 = client2._api.datasets.get(dataset.id)
assert dataset2.id == dataset.id
assert dataset2.exists
# create package (super-admin user session)
dataset2.add(pkg)
assert pkg.exists
# create package (normal user owns)
pkg = DataPackage('Some Video', package_type='Video')
assert not pkg.exists
dataset.add(pkg)
assert pkg.exists
# try doing as normal user - should error
with pytest.raises(UnauthorizedException):
pkg.set_sources(source)
with pytest.raises(UnauthorizedException):
pkg.set_files(file)
with pytest.raises(UnauthorizedException):
pkg.set_view(view)
def get_sources(self, pkg):
"""
Returns the sources of a DataPackage. Sources are the raw, unmodified
files (if they exist) that contains the package's data.
"""
pkg_id = self._get_id(pkg)
resp = self._get(self._uri('/{id}/sources', id=pkg_id))
for r in resp:
r['content'].update(dict(pkg_id=pkg_id))
return [File.from_dict(r, api=self.session) for r in resp]
def create(self, file, destination=None):
"""
Creates a file under the given destination or its current parent
"""
container = file.parent if destination is None else destination
body = file.as_dict()
body["container"] = container
response = self._post('', json=body)
return File.from_dict(response, api=self.session)
def get_view(self, pkg):
"""
Returns the object(s) used to view the package. This is typically a set of
file objects, that may be the DataPackage's sources or files, but could also be
a unique object specific for the viewer.
"""
pkg_id = self._get_id(pkg)
resp = self._get(self._uri('/{id}/view', id=pkg_id))
for r in resp:
r['content'].update(dict(pkg_id=pkg_id))
return [File.from_dict(r, api=self.session) for r in resp]
def get_files(self, pkg):
"""
Returns the files of a DataPackage. Files are the possibly modified
source files (e.g. converted to a different format), but they could also
be the source files themselves.
"""
pkg_id = self._get_id(pkg)
resp = self._get(self._uri('/{id}/files', id=pkg_id))
for r in resp:
r['content'].update(dict(pkg_id=pkg_id))
return [File.from_dict(r, api=self.session) for r in resp]
def set_view(self, pkg, *files, **kwargs):
"""
Set the object(s) used to view the package, if not the file(s) or source(s).
"""
pkg_id = self._get_id(pkg)
data = [x.as_dict() for x in files]
path = self._uri('/{id}/view', id=pkg_id)
resp = self._put(path, json=data, params=kwargs)
return [
File.from_dict(r, api=self.session) for r in resp
if not isinstance(r, basestring)
]
def set_files(self, pkg, *files, **kwargs):
"""
Sets the files of a DataPackage. Files are typically modified
source files (e.g. converted to a different format).
"""
pkg_id = self._get_id(pkg)
data = [x.as_dict() for x in files]
path = self._uri('/{id}/files', id=pkg_id)
resp = self._put(path, json=data, params=kwargs)
return [
File.from_dict(r, api=self.session) for r in resp
if not isinstance(r, basestring)
]
def set_sources(self, pkg, *files, **kwargs):
"""
Sets the sources of a DataPackage. Sources are the raw, unmodified
files (if they exist) that contains the package's data.
"""
pkg_id = self._get_id(pkg)
data = [x.as_dict() for x in files]
path = self._uri('/{id}/sources', id=pkg_id)
# Note: PUT returns list of IDs
resp = self._put(path, json=data, params=kwargs)
return [
File.from_dict(r, api=self.session) for r in resp
if not isinstance(r, basestring)
]
def test_package_objects(client, superuser_client, dataset):
"""
Only super-users are allowed to create/modify package sources/files.
"""
pkg = DataPackage('Some Video', package_type='Video')
assert not pkg.exists
# some files (local for now)
source = File(name='My Source File',
s3_key='s3/source',
s3_bucket='my-bucket',
file_type="JSON")
file = File(name='My File',
s3_key='s3/file',
s3_bucket='my-bucket',
file_type="CSV")
view = File(name='My View File',
s3_key='s3/view',
s3_bucket='my-bucket',
file_type="NIFTI")
# get dataset (but as super-user)
superuser_dataset = superuser_client.get(dataset.id)
assert superuser_dataset.id == dataset.id
assert superuser_dataset.exists
assert superuser_dataset.type == dataset.type
# create package (super-admin user session)
superuser_dataset.add(pkg)
assert pkg.exists
# add source (super-admin)
pkg.set_sources(source)
# get as normal user
pkg2 = client.get(pkg)
print "sources =", pkg2.sources
assert len(pkg2.sources) > 0
assert pkg2.sources[0].name == 'My Source File'
del pkg2
# add files (super-admin)
pkg.set_files(file)
# get as normal user
pkg2 = client.get(pkg)
print "files =", pkg2.files
assert len(pkg2.files) > 0
del pkg2
# add views (super-admin)
pkg.set_view(view)
# get as normal user
pkg2 = client.get(pkg)
print "view =", pkg2.view
assert len(pkg2.view) > 0
del pkg2
del pkg
# create package (normal user owns)
pkg = DataPackage('Some Video', package_type='Video', parent=dataset)
assert not pkg.exists
dataset.add(pkg)
assert pkg.exists
# try doing as normal user - should error
with pytest.raises(UnauthorizedException):
pkg.set_sources(source)
with pytest.raises(UnauthorizedException):
pkg.set_files(file)
with pytest.raises(UnauthorizedException):
pkg.set_view(view)