def test_package_proxy_import(bf, dataset, organization, assert_in_neo4j):
"""
Needs to be run in non-prod, on an organization that has not been fully migrated.
Test that multiple proxy relationships can be imported for the same proxy
concept, and that we can import edges both too and from proxy concepts.
"""
person = dataset.create_model(
"Person",
schema=[
ModelProperty("name", data_type=str, title=True, required=True)
],
)
alice = person.create_record({"name": "Alice"})
bob = person.create_record({"name": "Bob"})
pkg = DataPackage("Some MRI", package_type="MRI")
dataset.add(pkg)
pkg.relate_to(alice, bob)
alice_files = alice.get_files()
bob_files = bob.get_files()
migrate_dataset(organization_id=organization.int_id,
dataset_ids=[dataset.int_id])
assert_in_neo4j()
assert alice.get_files() == alice_files
assert bob.get_files() == bob_files
def test_package_objects(client, client2, dataset):
"""
Only super-users are allowed to create/modify package sources/files.
"""
pkg = DataPackage('Some Video', package_type='Video')
assert not pkg.exists
# some files (local for now)
source = File(name='My Source File',
s3_key='s3/source',
s3_bucket='my-bucket',
file_type="JSON",
size=1000)
file = File(name='My File',
s3_key='s3/file',
s3_bucket='my-bucket',
file_type="CSV",
size=1000)
view = File(name='My View File',
s3_key='s3/view',
s3_bucket='my-bucket',
file_type="NIFTI",
size=1000)
# get dataset (but as different client)
dataset2 = client2._api.datasets.get(dataset.id)
assert dataset2.id == dataset.id
assert dataset2.exists
# create package (super-admin user session)
dataset2.add(pkg)
assert pkg.exists
# create package (normal user owns)
pkg = DataPackage('Some Video', package_type='Video')
assert not pkg.exists
dataset.add(pkg)
assert pkg.exists
# try doing as normal user - should error
with pytest.raises(UnauthorizedException):
pkg.set_sources(source)
with pytest.raises(UnauthorizedException):
pkg.set_files(file)
with pytest.raises(UnauthorizedException):
pkg.set_view(view)
def test_datasets(client, dataset):
ds_items = len(dataset)
# create package locally
pkg = DataPackage("Child of Dataset", package_type="Text")
assert not pkg.exists
assert pkg not in dataset
# add package to dataset
dataset.add(pkg)
assert pkg.exists
assert pkg in dataset
assert pkg.id in dataset
assert len(dataset) == ds_items + 1
# remove from dataset
dataset.remove(pkg)
assert not pkg.exists
assert pkg not in dataset
assert len(dataset) == ds_items
# can't create dataset with same name
with pytest.raises(Exception):
client.create_dataset(dataset.name)
def test_upload(client, dataset):
"""
Note: ETL will fail since destination will likely be removed
before being processed.
"""
srcdir = os.path.dirname(__file__)
file1 = os.path.join(srcdir, 'test-upload.txt')
files = [file1]
# upload a file into dataset
r = client.upload(dataset, *files)
assert len(r['files']) == len(files)
assert r['destination'] == dataset.id
assert r['appendToPackage'] == False
# let's do it using model method
r = dataset.upload_files(*files)
assert len(r['files']) == len(files)
assert r['destination'] == dataset.id
assert r['appendToPackage'] == False
# try uploading into a DataPackage
pkg = DataPackage('Rando Thing', package_type='MRI')
dataset.add(pkg)
assert pkg.exists
# should definitely raise an error
with pytest.raises(Exception):
client.upload(pkg, *files)
def test_package_type_count(client, dataset):
n = dataset.package_count()
pkg = DataPackage("Some MRI", package_type="MRI")
assert not pkg.exists
# create
dataset.add(pkg)
assert pkg.exists
client.update(pkg)
pkg = DataPackage("Something else", package_type="TimeSeries")
assert not pkg.exists
dataset.add(pkg)
assert pkg.exists
client.update(pkg)
m = dataset.package_count()
assert m == n + 2
def get_packages_by_filename(self, ds, filename):
id = self._get_id(ds)
resp = self._get(
self._uri("/{id}/packages?filename={filename}",
id=id,
filename=filename))
return [
DataPackage.from_dict(p, api=self.session)
for p in resp.get("packages")
]
def test_can_remove_multiple_items(dataset):
pkg1 = DataPackage("Some MRI", package_type="MRI")
dataset.add(pkg1)
pkg1.update()
pkg2 = DataPackage("Some Video", package_type="Video")
dataset.add(pkg2)
pkg2.update()
assert pkg1 in dataset.items
assert pkg2 in dataset.items
dataset.remove(pkg1)
dataset.remove(pkg2)
assert pkg1 not in dataset.items
assert pkg2 not in dataset.items
def files(self, dataset, concept, instance):
"""
Return list of files (i.e. packages) related to record.
"""
dataset_id = self._get_id(dataset)
concept_id = self._get_id(concept)
instance_id = self._get_id(instance)
resp = self._get(
self._uri('/{dataset_id}/concepts/{concept_id}/instances/{instance_id}/files',
dataset_id=dataset_id,
concept_id=concept_id,
instance_id=instance_id
))
return [DataPackage.from_dict(pkg, api=self.session) for r,pkg in resp]
def test_upload(client, dataset):
"""
Note: ETL will fail since destination will likely be removed
before being processed.
"""
srcdir = os.path.dirname(__file__)
file1 = os.path.join(srcdir, 'test-upload.txt')
files = [file1]
# upload a file into dataset
r = dataset.upload(*files)
assert len(r) == len(files)
manifest = r[0][0]['manifest']
assert manifest['content'] is not None
assert manifest['type'] == 'upload'
# try uploading into a DataPackage
pkg = DataPackage('Rando Thing', package_type='MRI')
dataset.add(pkg)
assert pkg.exists
# should definitely raise an error
with pytest.raises(Exception):
pkg.upload(*files)
def test_package_states(client, dataset):
pkg = DataPackage('My Stateful Package', package_type='Slide')
assert not pkg.exists
dataset.add(pkg)
assert pkg.exists
assert pkg.state == "UNAVAILABLE"
pkg.set_ready()
pkg2 = client.get(pkg.id)
assert pkg2.id == pkg.id
assert pkg2.state == pkg.state
assert pkg2.state == "READY"
pkg.set_error()
del pkg2
pkg2 = client.get(pkg.id)
assert pkg2.id == pkg.id
assert pkg2.state == pkg.state
assert pkg2.state == "ERROR"
del pkg2
pkg.delete()
def test_packages_create_delete(client, dataset):
# init
pkg = DataPackage("Some MRI", package_type="MRI")
assert not pkg.exists
# create
dataset.add(pkg)
assert pkg.exists
assert pkg.id is not None
assert pkg.name == "Some MRI"
assert pkg.owner_id == client.profile.int_id
# TODO: (once we auto-include in parent)
assert pkg in dataset
# update package name
pkg.name = "Some Other MRI"
pkg = client.update(pkg)
pkg2 = client.get(pkg.id)
assert pkg2.name == "Some Other MRI"
assert pkg2.id == pkg.id
assert pkg2.owner_id == client.profile.int_id
# delete all packages
client.delete(pkg)
assert not pkg.exists
pkg = DataPackage("Something else", package_type="TimeSeries")
assert not pkg.exists
dataset.add(pkg)
assert pkg.exists
pid = pkg.id
pkg.delete()
assert not pkg.exists
pkg2 = client.get(pid)
assert pkg2 is None
def test_datasets(dataset):
ds_items = len(dataset)
# create package locally
pkg = DataPackage('Child of Dataset', package_type='Text')
assert not pkg.exists
assert pkg not in dataset
# add package to dataset
dataset.add(pkg)
assert pkg.exists
assert pkg in dataset
assert pkg.id in dataset
assert len(dataset) == ds_items + 1
# remove from dataset
dataset.remove(pkg)
assert not pkg.exists
assert pkg not in dataset
assert len(dataset) == ds_items
def test_packages_create_delete(client, dataset):
# init
pkg = DataPackage('Some MRI', package_type='MRI')
assert not pkg.exists
# create
dataset.add(pkg)
assert pkg.exists
assert pkg.id is not None
assert pkg.name == 'Some MRI'
# TODO: (once we auto-include in parent)
assert pkg in dataset
# update package name
pkg.name = 'Some Other MRI'
pkg = client.update(pkg)
pkg2 = client.get(pkg.id)
assert pkg2.name == 'Some Other MRI'
assert pkg2.id == pkg.id
# delete all packages
client.delete(pkg)
assert not pkg.exists
pkg = DataPackage('Something else', package_type='TimeSeries')
assert not pkg.exists
dataset.add(pkg)
assert pkg.exists
pid = pkg.id
pkg.delete()
assert not pkg.exists
pkg2 = client.get(pid)
assert pkg2 is None
def test_properties(client, dataset):
pkg = DataPackage("Some Video", package_type="Video")
assert not pkg.exists
dataset.add(pkg)
assert pkg.exists
pkg.insert_property("my-key", "my-value")
pkg2 = client.get(pkg)
print("properties =", pkg2.properties)
assert pkg2.id == pkg.id
assert pkg2.get_property("my-key").data_type == "string"
assert pkg2.get_property("my-key").value == "my-value"
explicit_ptypes = {
"my-int1": ("integer", 123123),
"my-int2": ("integer", "123123"),
"my-float": ("double", 123.123),
"my-float2": ("double", "123.123"),
"my-float3": ("double", "123123"),
"my-date": ("date", 1488847449697),
"my-date2": ("date", 1488847449697.123),
"my-date3": ("date", datetime.datetime.now()),
"my-string": ("string", "my-123123"),
"my-string2": ("string", "123123"),
"my-string3": ("string", "123123.123"),
"my-string4": ("string", "According to plants, humans are blurry."),
}
for key, (ptype, val) in explicit_ptypes.items():
pkg.insert_property(key, val, data_type=ptype)
assert pkg.get_property(key).data_type == ptype
inferred_ptypes = {
"my-int1": ("integer", 123123),
"my-int2": ("integer", "123123"),
"my-float1": ("double", 123.123),
"my-float2": ("double", "123.123"),
"my-date": ("date", datetime.datetime.now()),
"my-string": ("string", "i123123"),
"my-string2": ("string", "#1231"),
}
for key, (ptype, val) in inferred_ptypes.items():
pkg.insert_property(key, val)
prop = pkg.get_property(key)
assert prop.data_type == ptype
# remove property
pkg.remove_property("my-key")
assert pkg.get_property("my-key") is None
pkg2 = client.get(pkg.id)
assert pkg2.get_property("my-key") is None
def test_models(dataset):
schema = [('an_integer', int, 'An Integer', True), ('a_bool', bool), ('a_string', str), ('a_datetime', datetime.datetime)]
display_name = 'A New Property'
description = 'a new description'
values = {'an_integer': 100, 'a_bool': True, 'a_string': 'fnsdlkn#$#42nlfds$3nlds$#@$23fdsnfkls', 'a_datetime': datetime.datetime.now()}
#################################
## Models
################################
models = dataset.models()
new_model = dataset.create_model('New_Model_{}'.format(current_ts()), 'A New Model', 'a new model', schema)
assert len(dataset.models()) == len(models) + 1
assert dataset.get_model(new_model.id) == new_model
assert dataset.get_model(new_model.type) == new_model
# Check that local changes get propagated
new_model.add_property('a_new_property', str, display_name)
new_model.description = description
new_model.update()
new_model = dataset.get_model(new_model.id)
assert new_model.description == description
assert new_model.get_property('a_new_property').display_name == display_name
new_model.add_properties([('a_new_float', float), {'name': 'a_new_int', 'data_type': int}, 'a_new_string'])
assert new_model.get_property('a_new_float').type == float
assert new_model.get_property('a_new_int').type == int
assert new_model.get_property('a_new_string').type == unicode
nc_one = new_model.create_record(values)
nc_two = new_model.create_record({'an_integer': 1, 'a_bool': False, 'a_string': '', 'a_datetime': datetime.datetime.now()})
nc_three = new_model.create_record({'an_integer': 10000, 'a_bool': False, 'a_string': '43132312', 'a_datetime': datetime.datetime.now()})
nc_four = new_model.create_record({'a_datetime': datetime.datetime.now()})
nc_delete_one = new_model.create_record({'a_datetime': datetime.datetime.now()})
nc_delete_two = new_model.create_record({'a_datetime': datetime.datetime.now()})
with pytest.raises(Exception):
new_model.create_record()
new_models_old = new_model.get_all()
assert new_model.get_all(limit=1) == new_models_old[:1]
assert new_model.get_all(limit=2, offset=2) == new_models_old[2:4]
new_model.delete_records(nc_delete_one, nc_delete_two.id)
new_models = new_model.get_all()
assert len(new_models) == (len(new_models_old) - 2)
assert nc_two.model == new_model
assert nc_two.get('a_string') == ''
nc_four.set('a_string', 'hello')
assert nc_four.get('a_string') == new_model.get(nc_four).get('a_string')
with pytest.raises(Exception):
nc_four.set('an_integer', datetime.datetime.now())
assert nc_four.get('an_integer') == None
nc_four.set('an_integer', 10)
assert nc_four.get('an_integer') == 10
nc_delete_three = new_model.create_record({'a_string': 'delete me'})
assert len(new_model.get_all()) == len(new_models) + 1
nc_delete_three.delete()
assert len(new_model.get_all()) == len(new_models)
# cannot add a record id column using an existing name
with pytest.raises(ValueError):
new_model.get_all().as_dataframe(
record_id_column_name=list(new_model.get_all().type.schema.keys())[0]
)
# assert no extra columns are added by default
df_cs_no_rec = new_model.get_all().as_dataframe()
assert len(df_cs_no_rec.columns) == len(new_model.get_all().type.schema.keys())
# assert record id column is added when arg is present and valid
df_cs = new_model.get_all().as_dataframe(
record_id_column_name='record_id'
)
# confirm that all record ids are present in this dataframe
assert 'record_id' in df_cs.columns
for record in new_model.get_all():
assert not df_cs.query('record_id == @record.id').empty
#################################
## Relationships
################################
relationships = dataset.relationships()
new_relationship = dataset.create_relationship_type('New_Relationship_{}'.format(
current_ts()), 'a new relationship')
assert len(dataset.relationships()) == len(relationships) + 1
assert dataset.get_relationship(new_relationship.id) == new_relationship
assert dataset.get_relationship(new_relationship.type) == new_relationship
nr_one = new_relationship.relate(nc_one, nc_two)
nr_four = new_relationship.relate(nc_four, nc_one)
nr_five = new_relationship.relate(nc_four, nc_two)
nr_two = nc_two.relate_to(nc_three, new_relationship)
nr_three = nc_three.relate_to(nc_four, new_relationship)
nr_six = nc_four.relate_to(nc_three, new_relationship)
nr_seven = nc_four.relate_to(nc_one, relationship_type="goes_to")
assert nr_seven[0].destination == nc_one.id
assert nr_seven[0].source == nc_four.id
assert nr_seven[0].type == "goes_to"
assert len(nc_four.get_related(new_model.type)) == 5
new_relationships = new_relationship.get_all()
nr_delete_three = new_relationship.relate(nc_one, nc_two)
assert len(new_relationship.get_all()) == len(new_relationships) + 1
nr_delete_three.delete()
assert len(new_relationship.get_all()) == len(new_relationships)
df_rs = new_relationship.get_all().as_dataframe()
p = DataPackage('test-csv', package_type='Tabular')
dataset.add(p)
dataset.update()
assert p.exists
p.relate_to(nc_one)
p.relate_to(nc_two)
nc_three.relate_to(p, new_relationship)
new_relationship.relate(nc_four, p)
assert len(nc_four.get_related(new_model.type)) == 5
def test_concepts(dataset):
current_ts = lambda: int(round(time.time() * 1000))
schema = [('an_integer', int, 'An Integer', True), ('a_long', int), ('a_bool', bool), ('a_string', str), ('a_datetime', datetime.datetime)]
display_name = 'A New Property'
description = 'a new description'
values = {'an_integer': 100, 'a_long': 100000, 'a_bool': True, 'a_string': 'fnsdlkn#$#42nlfds$3nlds$#@$23fdsnfkls', 'a_datetime': datetime.datetime.now()}
#################################
## Models
################################
concepts = dataset.concepts()
new_concept = dataset.create_concept('New_Model_{}'.format(current_ts()), 'A New Model', 'a new concept', schema)
assert len(dataset.concepts()) == len(concepts) + 1
assert dataset.get_concept(new_concept.id) == new_concept
assert dataset.get_concept(new_concept.type) == new_concept
new_concept.add_property('a_new_property', str, display_name)
new_concept.description = description
new_concept.update()
new_concept = dataset.get_concept(new_concept.id)
assert new_concept.description == description
assert new_concept.get_property('a_new_property').display_name == display_name
new_concept.add_properties([('a_new_float', float), {'name': 'a_new_int', 'data_type': int}, 'a_new_string'])
assert new_concept.get_property('a_new_float').type == float
assert new_concept.get_property('a_new_int').type == int
assert new_concept.get_property('a_new_string').type == str
nc_one = new_concept.create(values)
nc_two = new_concept.create({'an_integer': 1, 'a_long': 0, 'a_bool': False, 'a_string': '', 'a_datetime': datetime.datetime.now()})
nc_three = new_concept.create({'an_integer': 10000, 'a_long': 9349234, 'a_bool': False, 'a_string': '43132312', 'a_datetime': datetime.datetime.now()})
nc_four = new_concept.create({'a_datetime': datetime.datetime.now()})
nc_delete_one = new_concept.create({'a_datetime': datetime.datetime.now()})
nc_delete_two = new_concept.create({'a_datetime': datetime.datetime.now()})
try:
new_concept.create()
assert False
except:
assert True
new_concepts_old = new_concept.get_all()
new_concept.delete_items(nc_delete_one, nc_delete_two.id)
new_concepts = new_concept.get_all()
assert len(new_concepts) == (len(new_concepts_old) - 2)
assert nc_two.concept() == new_concept
assert nc_two.get('a_string') == ''
nc_four.set('a_string', 'hello')
assert nc_four.get('a_string') == new_concept.get(nc_four).get('a_string')
try:
nc_four.set('an_integer', datetime.datetime.now())
assert False
except:
assert True
assert nc_four.get('an_integer') == None
nc_four.set('an_integer', 10)
assert nc_four.get('an_integer') == 10
nc_delete_three = new_concept.create({'a_string': 'delete me'})
assert len(new_concept.get_all()) == len(new_concepts) + 1
nc_delete_three.delete()
assert len(new_concept.get_all()) == len(new_concepts)
df_cs = new_concept.get_all().as_dataframe()
#################################
## Relationships
################################
relationships = dataset.relationships()
new_relationship = dataset.create_relationship('New_Relationship_{}'.format(current_ts()), 'a new relationship')
assert len(dataset.relationships()) == len(relationships) + 1
assert dataset.get_relationship(new_relationship.id) == new_relationship
assert dataset.get_relationship(new_relationship.type) == new_relationship
nr_one = new_relationship.link(nc_one, nc_two)
nr_two = nc_two.link(new_relationship, nc_three)
nr_three = nc_three.link(new_relationship.type, nc_four)
nr_four = new_relationship.link(nc_four, nc_one)
nr_five = new_relationship.link(nc_four, nc_two)
nr_six = nc_four.link(new_relationship, nc_three)
nc_four.update()
assert len(nc_four.relationships(new_concept)) == 4
assert len(nc_four.neighbors(new_concept)) == 4
assert len(nc_four.links(new_concept)) == 4
assert len(nc_four.relationships(new_concept.type, new_relationship.type)) == 4
assert len(nc_four.neighbors(new_concept.type, new_relationship.type)) == 4
assert len(nc_four.links(new_concept.type, new_relationship.type)) == 4
new_relationships = new_relationship.get_all()
assert nr_two.relationship() == new_relationship
try:
nr_four.set('an_integer', datetime.datetime.now())
assert False
except:
assert True
assert nr_four.get('an_integer') == None
nr_delete_three = new_relationship.link(nc_one, nc_two)
assert len(new_relationship.get_all()) == len(new_relationships) + 1
nr_delete_three.delete()
assert len(new_relationship.get_all()) == len(new_relationships)
try:
new_relationship.link(nc_one, nc_one)
assert False
except:
assert True
df_rs = new_relationship.get_all().as_dataframe()
p = DataPackage('test-csv', package_type='Tabular')
dataset.add(p)
dataset.update()
assert p.exists
p.link(new_relationship, nc_one)
p.link(new_relationship.type, nc_two)
nc_three.link(new_relationship, p)
new_relationship.link(nc_four, p)
nc_four.update()
assert len(nc_four.relationships(new_concept)) == 4
assert len(nc_four.neighbors(new_concept)) == 4
assert len(nc_four.links(new_concept)) == 4
assert len(nc_four.relationships(new_concept.type, new_relationship.type)) == 4
assert len(nc_four.neighbors(new_concept.type, new_relationship.type)) == 4
assert len(nc_four.links(new_concept.type, new_relationship.type)) == 4
def test_properties(client, dataset):
pkg = DataPackage('Some Video', package_type='Video')
assert not pkg.exists
dataset.add(pkg)
assert pkg.exists
pkg.insert_property('my-key', 'my-value')
pkg2 = client.get(pkg)
print 'properties =', pkg2.properties
assert pkg2.id == pkg.id
assert pkg2.get_property('my-key').data_type == 'string'
assert pkg2.get_property('my-key').value == 'my-value'
explicit_ptypes = {
'my-int1': ('integer', 123123),
'my-int2': ('integer', '123123'),
'my-float': ('double', 123.123),
'my-float2': ('double', '123.123'),
'my-float3': ('double', '123123'),
'my-date': ('date', 1488847449697),
'my-date2': ('date', 1488847449697.123),
'my-date3': ('date', datetime.datetime.now()),
'my-string': ('string', 'my-123123'),
'my-string2': ('string', '123123'),
'my-string3': ('string', '123123.123'),
'my-string4': ('string', 'According to plants, humans are blurry.'),
}
for key, (ptype, val) in explicit_ptypes.items():
pkg.insert_property(key, val, data_type=ptype)
assert pkg.get_property(key).data_type == ptype
inferred_ptypes = {
'my-int1': ('integer', 123123),
'my-int2': ('integer', '123123'),
'my-float1': ('double', 123.123),
'my-float2': ('double', '123.123'),
'my-date': ('date', datetime.datetime.now()),
'my-string': ('string', 'i123123'),
'my-string2': ('string', '#1231'),
}
for key, (ptype, val) in inferred_ptypes.items():
pkg.insert_property(key, val)
prop = pkg.get_property(key)
assert prop.data_type == ptype
# remove property
pkg.remove_property('my-key')
with pytest.raises(Exception):
assert pkg.get_property('my-key')
pkg2 = client.get(pkg.id)
with pytest.raises(Exception):
assert pkg2.get_property('my-key')
def test_models(dataset):
schema = [
("an_integer", int, "An Integer", True),
("a_bool", bool),
("a_string", str),
("a_datetime", datetime.datetime),
]
display_name = "A New Property"
description = "a new description"
values = {
"an_integer": 100,
"a_bool": True,
"a_string": "fnsdlkn#$#42nlfds$3nlds$#@$23fdsnfkls",
"a_datetime": datetime.datetime.now(),
}
#################################
## Models
################################
models = dataset.models()
new_model = dataset.create_model("New_Model_{}".format(current_ts()),
"A New Model", "a new model", schema)
assert len(dataset.models()) == len(models) + 1
assert dataset.get_model(new_model.id) == new_model
assert dataset.get_model(new_model.type) == new_model
# Check that local changes get propagated
new_model.add_property("a_new_property", str, display_name)
new_model.description = description
new_model.update()
new_model = dataset.get_model(new_model.id)
assert new_model.description == description
assert new_model.get_property(
"a_new_property").display_name == display_name
new_model.add_properties([
("a_new_float", float),
{
"name": "a_new_int",
"data_type": int
},
"a_new_string",
])
assert new_model.get_property("a_new_float").type == float
assert new_model.get_property("a_new_int").type == int
assert new_model.get_property("a_new_string").type == unicode
nc_one = new_model.create_record(values)
nc_two = new_model.create_record({
"an_integer": 1,
"a_bool": False,
"a_string": "",
"a_datetime": datetime.datetime.now(),
})
nc_three = new_model.create_record({
"an_integer": 10000,
"a_bool": False,
"a_string": "43132312",
"a_datetime": datetime.datetime.now(),
})
nc_four = new_model.create_record({
"an_integer": 9292,
"a_datetime": datetime.datetime.now()
})
nc_delete_one = new_model.create_record({
"an_integer":
28,
"a_datetime":
datetime.datetime.now()
})
nc_delete_two = new_model.create_record({
"an_integer":
300,
"a_datetime":
datetime.datetime.now()
})
with pytest.raises(Exception):
new_model.create_record()
new_models_old = new_model.get_all()
assert new_model.get_all(limit=1) == new_models_old[:1]
assert new_model.get_all(limit=2, offset=2) == new_models_old[2:4]
new_model.delete_records(nc_delete_one, nc_delete_two.id)
new_models = new_model.get_all()
assert len(new_models) == (len(new_models_old) - 2)
assert nc_two.model == new_model
assert nc_two.get("a_string") == ""
nc_four.set("a_string", "hello")
assert nc_four.get("a_string") == new_model.get(nc_four).get("a_string")
with pytest.raises(Exception):
nc_four.set("an_integer", datetime.datetime.now())
assert nc_four.get("an_integer") == 9292
nc_four.set("an_integer", 10)
assert nc_four.get("an_integer") == 10
nc_delete_three = new_model.create_record({
"an_integer": 684,
"a_string": "delete me"
})
assert len(new_model.get_all()) == len(new_models) + 1
nc_delete_three.delete()
assert len(new_model.get_all()) == len(new_models)
# cannot add a record id column using an existing name
with pytest.raises(ValueError):
new_model.get_all().as_dataframe(record_id_column_name=list(
new_model.get_all().type.schema.keys())[0])
# assert no extra columns are added by default
df_cs_no_rec = new_model.get_all().as_dataframe()
assert len(df_cs_no_rec.columns) == len(
new_model.get_all().type.schema.keys())
# assert record id column is added when arg is present and valid
df_cs = new_model.get_all().as_dataframe(record_id_column_name="record_id")
# confirm that all record ids are present in this dataframe
assert "record_id" in df_cs.columns
for record in new_model.get_all():
assert not df_cs.query("record_id == @record.id").empty
#################################
## Relationships
################################
relationships = dataset.relationships()
new_relationship = dataset.create_relationship_type(
"New_Relationship_{}".format(current_ts()), "a new relationship")
assert len(dataset.relationships()) == len(relationships) + 1
assert dataset.get_relationship(new_relationship.id) == new_relationship
assert dataset.get_relationship(new_relationship.type) == new_relationship
nr_one = new_relationship.relate(nc_one, nc_two)
nr_four = new_relationship.relate(nc_four, nc_one)
nr_five = new_relationship.relate(nc_four, nc_two)
nr_two = nc_two.relate_to(nc_three, new_relationship)
nr_three = nc_three.relate_to(nc_four, new_relationship)
nr_six = nc_four.relate_to(nc_three, new_relationship)
nr_seven = nc_four.relate_to(nc_one, relationship_type="goes_to")
assert nr_seven[0].destination == nc_one.id
assert nr_seven[0].source == nc_four.id
assert nr_seven[0].type == "goes_to"
assert len(nc_four.get_related(new_model.type)) == 5
new_relationships = new_relationship.get_all()
nr_delete_three = new_relationship.relate(nc_one, nc_three)
assert len(new_relationship.get_all()) == len(new_relationships) + 1
nr_delete_three.delete()
assert len(new_relationship.get_all()) == len(new_relationships)
df_rs = new_relationship.get_all().as_dataframe()
p = DataPackage("test-csv", package_type="CSV")
dataset.add(p)
dataset.update()
assert p.exists
p.relate_to(nc_one)
p.relate_to(nc_two)
nc_three.relate_to(p, new_relationship)
new_relationship.relate(nc_four, p)
assert len(nc_four.get_related(new_model.type)) == 5
def test_package_objects(client, superuser_client, dataset):
"""
Only super-users are allowed to create/modify package sources/files.
"""
pkg = DataPackage('Some Video', package_type='Video')
assert not pkg.exists
# some files (local for now)
source = File(name='My Source File',
s3_key='s3/source',
s3_bucket='my-bucket',
file_type="JSON")
file = File(name='My File',
s3_key='s3/file',
s3_bucket='my-bucket',
file_type="CSV")
view = File(name='My View File',
s3_key='s3/view',
s3_bucket='my-bucket',
file_type="NIFTI")
# get dataset (but as super-user)
superuser_dataset = superuser_client.get(dataset.id)
assert superuser_dataset.id == dataset.id
assert superuser_dataset.exists
assert superuser_dataset.type == dataset.type
# create package (super-admin user session)
superuser_dataset.add(pkg)
assert pkg.exists
# add source (super-admin)
pkg.set_sources(source)
# get as normal user
pkg2 = client.get(pkg)
print "sources =", pkg2.sources
assert len(pkg2.sources) > 0
assert pkg2.sources[0].name == 'My Source File'
del pkg2
# add files (super-admin)
pkg.set_files(file)
# get as normal user
pkg2 = client.get(pkg)
print "files =", pkg2.files
assert len(pkg2.files) > 0
del pkg2
# add views (super-admin)
pkg.set_view(view)
# get as normal user
pkg2 = client.get(pkg)
print "view =", pkg2.view
assert len(pkg2.view) > 0
del pkg2
del pkg
# create package (normal user owns)
pkg = DataPackage('Some Video', package_type='Video', parent=dataset)
assert not pkg.exists
dataset.add(pkg)
assert pkg.exists
# try doing as normal user - should error
with pytest.raises(UnauthorizedException):
pkg.set_sources(source)
with pytest.raises(UnauthorizedException):
pkg.set_files(file)
with pytest.raises(UnauthorizedException):
pkg.set_view(view)
nr_five = new_relationship.relate(nc_four, nc_two)
nr_two = nc_two.relate_to(nc_three, new_relationship)
nr_three = nc_three.relate_to(nc_four, new_relationship)
nr_six = nc_four.relate_to(nc_three, new_relationship)
nc_four.update()
assert len(nc_four.get_related(new_model.type)) == 4
new_relationships = new_relationship.get_all()
nr_delete_three = new_relationship.relate(nc_one, nc_two)
assert len(new_relationship.get_all()) == len(new_relationships) + 1
nr_delete_three.delete()
assert len(new_relationship.get_all()) == len(new_relationships)
df_rs = new_relationship.get_all().as_dataframe()
p = DataPackage('test-csv', package_type='Tabular')
dataset.add(p)
dataset.update()
assert p.exists
p.relate_to(nc_one)
p.relate_to(nc_two)
nc_three.relate_to(p, new_relationship)
new_relationship.relate(nc_four, p)
nc_four.update()
assert len(nc_four.get_related(new_model.type)) == 4