def test_complex_model_properties(dataset):
model_with_complex_props = dataset.create_model('Complex_Props', description='a new description', schema=[
ModelProperty('name', data_type=ModelPropertyType(data_type=str), title=True),
ModelProperty('age', data_type=ModelPropertyType(data_type=int)),
ModelProperty('email', data_type=ModelPropertyType(data_type=str, format='email'))
])
assert dataset.get_model(model_with_complex_props.id) == model_with_complex_props
# Add a property
model_with_complex_props.add_property('weight', ModelPropertyType(data_type=float, unit='kg'), display_name='Weight')
updated_model = dataset.get_model(model_with_complex_props.id)
weight_property = updated_model.get_property('weight')
assert (weight_property.display_name == 'Weight')
assert (weight_property.type == float)
assert (weight_property._type.data_type == float)
assert (weight_property._type.format == None)
assert (weight_property.unit == 'kg')
email_property = model_with_complex_props.get_property('email')
assert (email_property.type == unicode)
assert (email_property._type.data_type == unicode)
assert (email_property._type.format.lower() == 'email')
assert (email_property.unit == None)
good_values = {'name': 'Bob', 'age': 1, 'email': '*****@*****.**', 'weight': 10}
good_record = updated_model.create_record(good_values)
bad_values = {'name': 'Bob', 'age': 1, 'email': '123455', 'weight': 10}
with pytest.raises(Exception):
bad_record = updated_model.create_record(bad_values)
def test_get_connected(dataset):
model_1 = dataset.create_model('Model_A', description="model a", schema=[
ModelProperty("prop1", data_type=ModelPropertyType(data_type=str),
title=True)])
related_models = model_1.get_connected()
# For a single, unconnected model, it should return nothing
assert len(related_models) == 0
model_2 = dataset.create_model('Model_B', description="model b", schema=[
ModelProperty("prop1", data_type=ModelPropertyType(data_type=str),
title=True)])
relationship = dataset.create_relationship_type(
'New_Relationship_{}'.format(current_ts()), 'a new relationship')
model_instance_1 = model_1.create_record({"prop1": "val1"})
model_instance_2 = model_2.create_record({"prop1": "val1"})
model_instance_1.relate_to(model_instance_2, relationship)
related_models = model_1.get_connected()
assert len(related_models) == 2
# For a connected model, return all connections + the model itself
related_models = model_2.get_connected()
assert len(related_models) == 2
# Check that get_connected_models from the dataset object also works
related_models = dataset.get_connected_models(model_1.id)
assert len(related_models) == 2
related_models = dataset.get_connected_models(model_2.id)
assert len(related_models) == 2
def test_related_records_pagination(dataset):
patient = dataset.create_model(
"patient",
description="patient",
schema=[
ModelProperty("name",
data_type=ModelPropertyType(data_type=str),
title=True)
],
)
visit = dataset.create_model(
"visit",
description="visit",
schema=[
ModelProperty("field",
data_type=ModelPropertyType(data_type=str),
title=True)
],
)
attends = dataset.create_relationship_type("attends", "an attendance")
fred = patient.create_record({"name": "Fred"})
visits = visit.create_records([{"field": str(i)} for i in range(200)])
fred.relate_to(visits, attends)
# Get all records
gotten = fred.get_related()
assert len(gotten) == 200
assert [int(r.get("field")) for r in gotten] == list(range(200))
def test_date_formatting():
d1 = datetime.datetime(2018, 8, 24, 15, 11, 25)
assert (ModelPropertyType(datetime.datetime)._encode_value(d1) ==
"2018-08-24T15:11:25.000000+00:00")
d2 = datetime.datetime(2018, 8, 24, 15, 11, 25, 1)
assert (ModelPropertyType(datetime.datetime)._encode_value(d2) ==
"2018-08-24T15:11:25.000001+00:00")
def test_complex_model_properties(dataset):
model_with_complex_props = dataset.create_model(
"Complex_Props",
description="a new description",
schema=[
ModelProperty("name",
data_type=ModelPropertyType(data_type=str),
title=True),
ModelProperty("age", data_type=ModelPropertyType(data_type=int)),
ModelProperty("email",
data_type=ModelPropertyType(data_type=str,
format="email")),
],
)
assert dataset.get_model(
model_with_complex_props.id) == model_with_complex_props
# Add a property
model_with_complex_props.add_property("weight",
ModelPropertyType(data_type=float,
unit="kg"),
display_name="Weight")
updated_model = dataset.get_model(model_with_complex_props.id)
weight_property = updated_model.get_property("weight")
assert weight_property.display_name == "Weight"
assert weight_property.type == float
assert weight_property._type.data_type == float
assert weight_property._type.format == None
assert weight_property.unit == "kg"
email_property = model_with_complex_props.get_property("email")
assert email_property.type == unicode
assert email_property._type.data_type == unicode
assert email_property._type.format.lower() == "email"
assert email_property.unit == None
good_values = {
"name": "Bob",
"age": 1,
"email": "*****@*****.**",
"weight": 10
}
good_record = updated_model.create_record(good_values)
bad_values = {"name": "Bob", "age": 1, "email": "123455", "weight": 10}
with pytest.raises(Exception):
bad_record = updated_model.create_record(bad_values)
def simple_graph(client):
"""
Creates a small test graph in an independent dataset to de-couple
from other tests
"""
test_dataset = create_test_dataset(client)
model_1 = test_dataset.create_model(
"Model_A",
description="model a",
schema=[
ModelProperty("prop1",
data_type=ModelPropertyType(data_type=str),
title=True)
],
)
model_2 = test_dataset.create_model(
"Model_B",
description="model b",
schema=[
ModelProperty("prop1",
data_type=ModelPropertyType(data_type=str),
title=True)
],
)
relationship = test_dataset.create_relationship_type(
"New_Relationship_{}".format(current_ts()), "a new relationship")
model_instance_1 = model_1.create_record({"prop1": "val1"})
model_instance_2 = model_2.create_record({"prop1": "val1"})
model_instance_1.relate_to(model_instance_2, relationship)
graph = Graph(
test_dataset,
models=[model_1, model_2],
model_records=[model_instance_1, model_instance_2],
relationships=[relationship],
relationship_records=None,
)
yield graph
ds_id = test_dataset.id
client._api.datasets.delete(test_dataset)
all_dataset_ids = [x.id for x in client.datasets()]
assert ds_id not in all_dataset_ids
assert not test_dataset.exists
def test_related_records_pagination(dataset):
patient = dataset.create_model(
'patient', description="patient", schema=[
ModelProperty("name", data_type=ModelPropertyType(data_type=str),
title=True)])
visit = dataset.create_model(
'visit', description="visit", schema=[
ModelProperty("field", data_type=ModelPropertyType(data_type=str),
title=True)])
attends = dataset.create_relationship_type('attends', 'an attendance')
patient1 = patient.create_record({"name": "Fred"})
visits = visit.create_records([{"field": str(i)} for i in range(200)])
patient1.relate_to(visits, attends)
# Get all records
gotten = patient1.get_related()
assert len(gotten) == 200
assert [r.get("field") for r in gotten] == list(map(str, range(200)))
def test_stringified_boolean_values(dataset):
ppatient = dataset.create_model(
"potential_patient",
description="potential patient",
schema=[
ModelProperty("name",
data_type=ModelPropertyType(data_type=str),
title=True),
ModelProperty("sick", data_type=ModelPropertyType(data_type=bool)),
],
)
ppatient.create_record({"name": "Fred", "sick": "false"})
ppatient.create_record({"name": "Joe", "sick": "true"})
ppatient.create_record({"name": "Adele", "sick": "plop"})
# Get all records
gotten = ppatient.get_all()
assert len(gotten) == 3
assert gotten[0]._values["sick"].value == False
assert gotten[1]._values["sick"].value == True
assert gotten[2]._values["sick"].value == True
def create_models(ds):
from blackfynn import ModelProperty
try:
ds.get_model('SPARC_Dataset')
except:
dataset_schema = [
ModelProperty('name', title=True, display_name='Name'),
ModelProperty('owner', data_type=str, display_name='Owner'),
ModelProperty('owner_email',
display_name='Owner email',
data_type=ModelPropertyType(data_type=str,
format='email')),
ModelProperty('sparc_award',
data_type=str,
display_name='SPARC Award'),
ModelProperty('award_valid',
data_type=bool,
display_name='Award Valid'),
ModelProperty('milestone_complete',
data_type='date',
display_name='Milestone Complete Date'),
ModelProperty('date_created',
data_type='date',
display_name='Submission Date'),
ModelProperty('first_published',
data_type='date',
display_name='First published'),
ModelProperty('status', data_type=str, display_name='Status'),
ModelProperty('blackfynn_url',
display_name='Blackfynn URL',
data_type=ModelPropertyType(data_type=str,
format='url')),
ModelProperty('discover_url',
display_name='Discover URL',
data_type=ModelPropertyType(data_type=str,
format='url')),
ModelProperty('error_index',
data_type=int,
display_name='Error Index'),
ModelProperty('number_of_files',
data_type=int,
display_name='Number of files'),
ModelProperty('number_of_folders',
data_type=int,
display_name='Number of folders'),
ModelProperty('total_size',
display_name='Total size',
data_type=ModelPropertyType(data_type=float,
unit='GB')),
ModelProperty('doi',
display_name='DOI',
data_type=ModelPropertyType(data_type=str,
format='url')),
ModelProperty('last_updated',
data_type='date',
display_name='Last Updated'),
ModelProperty('last_published',
data_type='date',
display_name='Last published'),
ModelProperty('status_log',
display_name='Status Log',
data_type=ModelPropertyEnumType(data_type=str,
multi_select=True)),
ModelProperty('dataset_id',
data_type=str,
display_name='Dataset ID'),
ModelProperty('discover_id',
data_type=int,
display_name='Discover ID'),
ModelProperty('curation_priority',
data_type=str,
display_name='NIH Priority')
]
model = ds.create_model('SPARC_Dataset', schema=dataset_schema)
try:
ds.get_model('SPARC_Award')
except:
sparc_award_schema = [
ModelProperty('award_id', title=True, display_name='Award ID'),
ModelProperty('award_valid',
data_type=bool,
display_name='Award Valid'),
ModelProperty('title', display_name='Title'),
ModelProperty('description', display_name='Description'),
ModelProperty('principle_investigator', display_name='PI')
]
award_model = ds.create_model('SPARC_Award', schema=sparc_award_schema)
# try:
# ds.get_model('SPARC_Data_Milestone')
# except:
# milestone_schema = [
# ModelProperty('data_description', title=True, display_name='Data Description'),
# ModelProperty('award', data_type=str, display_name='Status'),
# ModelProperty('related_milestone', data_type=int, display_name='Error Index'),
# ModelProperty('owner', data_type=str, display_name='Owner'),
# ModelProperty('owner_email', data_type=str, display_name='Owner email'),
# ModelProperty('sparc_award', data_type=str, display_name='SPARC Award'),
# ModelProperty('number_of_files', data_type=int, display_name='Number of files'),
# ModelProperty('total_size', data_type=str, display_name='Total size'),
# ModelProperty('doi', data_type=str, display_name='DOI'),
# ModelProperty('first_published', data_type=str, display_name='First published'),
# ModelProperty('last_published', data_type=str, display_name='Last published'),
# ModelProperty('blackfynn_id', data_type=str, display_name='Blackfynn ID'),
# ModelProperty('discover_id', data_type=int, display_name='Discover ID'),
# ModelProperty('blackfynn_url', data_type=str, display_name='Blackfynn URL'),
# ModelProperty('discover_url', data_type=str, display_name='Discover URL'),
# ModelProperty('portal_url', data_type=str, display_name='Portal URL'),
# ModelProperty('curation_priority', data_type=str, display_name='NIH Priority')
# ]
# model = ds.create_model('SPARC_Dataset', schema = dataset_schema)
try:
ds.get_model('Update_Run')
except:
process_schema = [
ModelProperty('name', title=True, display_name='Name'),
ModelProperty('status', data_type='date', display_name='Date'),
]
award_model = ds.create_model('Update_Run', schema=process_schema)
def test_model_type_from_dict(json, data_type, format, unit):
decoded = ModelPropertyType._build_from(json)
assert isinstance(decoded, ModelPropertyType)
assert decoded.data_type == data_type
assert decoded.format == format
assert decoded.unit == unit
def test_model_type_conversion(from_type, blackfynn_type, data_type):
property_type = ModelPropertyType._build_from(from_type)
assert property_type.data_type == data_type
assert property_type._blackfynn_type == blackfynn_type
def test_simple_model_properties(dataset):
# Define properties as tuples
model_with_basic_props_1 = dataset.create_model('Basic_Props_1', description='a new description',schema=
[('an_integer', int, 'An Integer', True), ('a_bool', bool), ('a_string', str), ('a_date', datetime.datetime)])
assert dataset.get_model(model_with_basic_props_1.id) == model_with_basic_props_1
# Add a property with a description
model_with_basic_props_1.add_property('a_new_property', float, display_name='Weight', description="some metric")
updated_model_1 = dataset.get_model(model_with_basic_props_1.id)
test_prop = updated_model_1.get_property('a_new_property')
assert test_prop.display_name == 'Weight'
assert test_prop.description == "some metric"
# Define properties as ModelProperty objects
model_with_basic_props_2 = dataset.create_model('Basic_Props_2', description='a new description', schema=[
ModelProperty('name', data_type=str, title=True, required=True),
ModelProperty('age', data_type=int),
ModelProperty('DOB', data_type=datetime.datetime)
])
assert dataset.get_model(model_with_basic_props_2.id) == model_with_basic_props_2
# Add a property
model_with_basic_props_2.add_property('weight2', ModelPropertyType(data_type=float), display_name='Weight')
updated_model_2 = dataset.get_model(model_with_basic_props_2.id)
assert updated_model_2.get_property('weight2').display_name == 'Weight'
assert updated_model_2.get_property('name').required == True
# Define properties as ModelProperty objects with ModelPropertyType data_type
model_with_basic_props_3 = dataset.create_model('Basic_Props_3', description='a new description', schema=[
ModelProperty('name', data_type=ModelPropertyType(data_type=str), title=True),
ModelProperty('age', data_type=ModelPropertyType(data_type=int)),
ModelProperty('DOB', data_type=ModelPropertyType(data_type=str))
])
assert dataset.get_model(model_with_basic_props_3.id) == model_with_basic_props_3
# Add a property
model_with_basic_props_3.add_property('weight3', ModelPropertyType(data_type=float), display_name='Weight')
updated_model_3 = dataset.get_model(model_with_basic_props_3.id)
assert updated_model_3.get_property('weight3').display_name == 'Weight'
# Reverse look up property data types
ModelProperty('name', data_type='string', title=True, required=True)
model_with_basic_props_4 = dataset.create_model('Basic_Props_4', description='a new description', schema=[
ModelProperty('name', data_type='string', title=True, required=True),
ModelProperty('age', data_type='long'),
ModelProperty('DOB', data_type='date')
])
assert dataset.get_model(model_with_basic_props_4.id) == model_with_basic_props_4
# Add a property
model_with_basic_props_4.add_property('weight4', ModelPropertyType(data_type='double'), display_name='Weight')
updated_model_4 = dataset.get_model(model_with_basic_props_4.id)
assert updated_model_4.get_property('weight4').display_name == 'Weight'
assert updated_model_4.get_property('name').required == True
from past.builtins import unicode # Alias of str in Python 3
import pytest
from blackfynn.models import (
DataPackage,
ModelProperty,
ModelPropertyEnumType,
ModelPropertyType,
)
from tests.utils import create_test_dataset, current_ts, get_test_client
@pytest.mark.parametrize('from_type,blackfynn_type,data_type', [
(int, 'long', int),
(ModelPropertyType(data_type=int), 'long', int),
(ModelPropertyType(data_type=unicode), 'string', unicode),
(datetime.date, 'date', datetime.date),
('date', 'date', datetime.datetime),
(ModelPropertyType(data_type=str, format='date'), 'string', str),
])
def test_model_type_conversion(from_type, blackfynn_type, data_type):
property_type = ModelPropertyType._build_from(from_type)
assert property_type.data_type == data_type
assert property_type._blackfynn_type == blackfynn_type
@pytest.mark.parametrize('json,data_type,format,unit', [
('string', unicode, None, None),
({'type':'string', 'format':'date'}, unicode, 'date', None),
({'type': 'double', 'unit': 'kg'}, float, None, 'kg'),
def test_boolean_string():
assert ModelPropertyType(bool)._decode_value("false") == False
assert ModelPropertyType(bool)._decode_value("true") == True
assert ModelPropertyType(bool)._decode_value("nope") == True
def test_simple_model_properties(dataset):
# Define properties as tuples
model_with_basic_props_1 = dataset.create_model(
"Basic_Props_1",
description="a new description",
schema=[
("an_integer", int, "An Integer", True),
("a_bool", bool),
("a_string", str),
("a_date", datetime.datetime),
],
)
assert dataset.get_model(
model_with_basic_props_1.id) == model_with_basic_props_1
# Add a property with a description
model_with_basic_props_1.add_property("a_new_property",
float,
display_name="Weight",
description="some metric")
updated_model_1 = dataset.get_model(model_with_basic_props_1.id)
test_prop = updated_model_1.get_property("a_new_property")
assert test_prop.display_name == "Weight"
assert test_prop.description == "some metric"
# Define properties as ModelProperty objects
model_with_basic_props_2 = dataset.create_model(
"Basic_Props_2",
description="a new description",
schema=[
ModelProperty("name", data_type=str, title=True, required=True),
ModelProperty("age", data_type=int),
ModelProperty("DOB", data_type=datetime.datetime),
],
)
assert dataset.get_model(
model_with_basic_props_2.id) == model_with_basic_props_2
# Add a property
model_with_basic_props_2.add_property("weight2",
ModelPropertyType(data_type=float),
display_name="Weight")
updated_model_2 = dataset.get_model(model_with_basic_props_2.id)
assert updated_model_2.get_property("weight2").display_name == "Weight"
assert updated_model_2.get_property("name").required == True
# Define properties as ModelProperty objects with ModelPropertyType data_type
model_with_basic_props_3 = dataset.create_model(
"Basic_Props_3",
description="a new description",
schema=[
ModelProperty("name",
data_type=ModelPropertyType(data_type=str),
title=True),
ModelProperty("age", data_type=ModelPropertyType(data_type=int)),
ModelProperty("DOB", data_type=ModelPropertyType(data_type=str)),
],
)
assert dataset.get_model(
model_with_basic_props_3.id) == model_with_basic_props_3
# Add a property
model_with_basic_props_3.add_property("weight3",
ModelPropertyType(data_type=float),
display_name="Weight")
updated_model_3 = dataset.get_model(model_with_basic_props_3.id)
assert updated_model_3.get_property("weight3").display_name == "Weight"
# Reverse look up property data types
ModelProperty("name", data_type="string", title=True, required=True)
model_with_basic_props_4 = dataset.create_model(
"Basic_Props_4",
description="a new description",
schema=[
ModelProperty("name",
data_type="string",
title=True,
required=True),
ModelProperty("age", data_type="long"),
ModelProperty("DOB", data_type="date"),
],
)
assert dataset.get_model(
model_with_basic_props_4.id) == model_with_basic_props_4
# Add a property
model_with_basic_props_4.add_property(
"weight4",
ModelPropertyType(data_type="double"),
display_name="Weight")
updated_model_4 = dataset.get_model(model_with_basic_props_4.id)
assert updated_model_4.get_property("weight4").display_name == "Weight"
assert updated_model_4.get_property("name").required == True
from past.builtins import unicode # Alias of str in Python 3
from blackfynn.models import (
DataPackage,
ModelProperty,
ModelPropertyEnumType,
ModelPropertyType,
)
from tests.utils import create_test_dataset, current_ts, get_test_client
@pytest.mark.parametrize(
"from_type,blackfynn_type,data_type",
[
(int, "long", int),
(ModelPropertyType(data_type=int), "long", int),
(ModelPropertyType(data_type=unicode), "string", unicode),
(datetime.date, "date", datetime.date),
("date", "date", datetime.datetime),
(ModelPropertyType(data_type=str, format="date"), "string", str),
],
)
def test_model_type_conversion(from_type, blackfynn_type, data_type):
property_type = ModelPropertyType._build_from(from_type)
assert property_type.data_type == data_type
assert property_type._blackfynn_type == blackfynn_type
@pytest.mark.parametrize(
"json,data_type,format,unit",
[