def create_linked_properties(db: PartitionedDatabase, concept_id: ModelId, id_: RecordId, body: Dict): with db.transaction() as tx: # map of model relationship id -> relationship # allows us to make sure model relationships exist model_relationship_map = { model_relationship.id: model_relationship for model_relationship in db.get_outgoing_model_relationships_tx( tx=tx, from_model=concept_id, one_to_many=False) } # map of model relationship id -> existing record relationship # so we can delete existing relationships such that this behaves as an UPSERT existing_record_relationship_map = { record_relationship.model_relationship_id: record_relationship for record_relationship in db.get_outgoing_record_relationships_tx( tx=tx, from_record=id_, one_to_many=False) } payload: List[Dict[str, str]] = list(body["data"]) # assure each request is or a unique schemaLinkedProperty if len(payload) != len( set(p["schemaLinkedPropertyId"] for p in payload)): raise BadRequest("duplicate model linked properties") to_create: List[CreateRecordRelationship] = [] for item in payload: model_relationship_id = item["schemaLinkedPropertyId"] if model_relationship_id not in model_relationship_map.keys(): raise BadRequest( f"no model linked property exists for {item['schemaLinkedPropertyId']}" ) model_relationship = model_relationship_map[model_relationship_id] if model_relationship_id in existing_record_relationship_map.keys( ): db.delete_outgoing_record_relationship_tx( tx=tx, record=id_, relationship=existing_record_relationship_map[ model_relationship_id], ) to_create.append( CreateRecordRelationship( from_=id_, to=item["to"], model_relationship=model_relationship)) result = db.create_record_relationship_batch_tx(tx=tx, to_create=to_create) data = [to_linked_property(r) for r, _ in result] return {"data": data}
def get_relationships( db: PartitionedDatabase, record_id: RecordId, relationship_type: Optional[str] = None, ) -> List[JsonDict]: with db.transaction() as tx: relationships = db.get_outgoing_record_relationships_tx( tx=tx, from_record=record_id, relationship_name=relationship_type) return [rel.to_dict() for rel in relationships]
def get_linked_properties(db: PartitionedDatabase, concept_id: ModelId, id_: RecordId): # Ignore the ID of the model/concept, as it's not needed to actually # fetch the linked property: with db.transaction() as tx: return [ to_linked_property(rr) for rr in db.get_outgoing_record_relationships_tx( tx=tx, from_record=id_, one_to_many=False) ]
def test_rewrite_ids_and_import(neo4j): """ Test that UUIDs are remapped to the exact correct place with a manually defined remapping. """ dataset_id = 60000 dataset_node_id = "N:dataset:b1154216-d1d7-4484-ad18-81b58fb65484" organization_id = 5 organization_node_id = "N:organization:c905919f-56f5-43ae-9c2a-8d5d542c133b" user_id = 114 user_node_id = "N:user:028058b9-dd8d-4f24-a187-ea56830b379f" db = PartitionedDatabase( db=neo4j, organization_id=OrganizationId(organization_id), dataset_id=DatasetId(dataset_id), user_id=user_node_id, organization_node_id=organization_node_id, dataset_node_id=dataset_node_id, ) REMAPPING = { "0b4b3615-9eaf-425d-9727-bcac29686fd5": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", "7b17c60d-ca2a-4cf5-a4ff-a52bbc32ff17": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", "e507b3ef-ade4-4672-83b4-f3f0774fb282": "cccccccc-cccc-cccc-cccc-cccccccccccc", "bf858cb5-ae51-4fcf-ad74-b1887946f70f": "dddddddd-dddd-dddd-dddd-dddddddddddd", "a99b09f5-caa6-4282-aa0e-cf56bde89254": "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee", "42fa4eb9-51cc-4c59-b550-ac24d6d5024a": "ffffffff-ffff-ffff-ffff-ffffffffffff", "ecb71447-b684-c589-abda-b673c38edefc": "00000000-0000-0000-0000-000000000000", "e2b71447-e29d-11c3-24c6-f2ebffd1486a": "11111111-1111-1111-1111-111111111111", "c8b71de8-cd9c-cc3f-67fe-4e30968d4e50": "22222222-2222-2222-2222-222222222222", "2e754729-684a-4c45-960f-348d68737d4d": "33333333-3333-3333-3333-333333333333", "175ff55b-b44d-4381-bd59-d4dbc0b9c5f0": "44444444-4444-4444-4444-444444444444", "ccf200d3-e77f-4d9e-bed3-f1f28860152f": "55555555-5555-5555-5555-555555555555", "443e141b-f59c-419f-82c1-eed97925b04d": "66666666-6666-6666-6666-666666666666", "d0b71de9-21f9-3557-edda-ad278dd81dc0": "77777777-7777-7777-7777-777777777777", "aeb7476e-55f6-7924-5e43-a83cfa7e4cef": "88888888-8888-8888-8888-888888888888", "fa3daedd-1761-4730-be7d-bb5de8e1261c": "99999999-9999-9999-9999-999999999999", "00b71de7-b42f-1fe9-a83f-824452fe966e": "aaaaaaaa-aaaa-aaaa-aaaa-bbbbbbbbbbbb", "460591a0-8079-4979-a860-c3a4b18a32ad": "aaaaaaaa-aaaa-aaaa-aaaa-cccccccccccc", } def generate_new_id(old_id): new_id = REMAPPING.get(old_id, None) if new_id is None: return old_id return new_id load( dataset=f"{organization_id}/{dataset_id}", bucket="dev-neptune-export-use1", db=db, use_cache=False, smoke_test=False, remap_ids=True, generate_new_id=generate_new_id, ) # Models patient = db.get_model("patient") assert patient == Model( name="patient", display_name="Patient", description="", count=2, id="aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", created_at=iso8601.parse("2019-11-01T20:01:27.027Z"), updated_at=iso8601.parse("2019-11-01T20:01:27.027Z"), template_id=None, ) assert sorted(db.get_properties(patient), key=lambda p: p.index) == [ ModelProperty( id="bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", name="name", display_name="Name", description="", index=0, locked=False, model_title=True, required=False, data_type=dt.String(), default=True, default_value=None, created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", created_at=iso8601.parse("2019-11-01T20:01:37.633Z"), updated_at=iso8601.parse("2019-11-11T15:11:17.383Z"), ), ModelProperty( id="cccccccc-cccc-cccc-cccc-cccccccccccc", name="dob", display_name="DOB", description="", index=1, locked=False, model_title=False, required=False, data_type=dt.Date(), default=False, default_value=None, created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", created_at=iso8601.parse("2019-11-11T15:11:17.383Z"), updated_at=iso8601.parse("2019-11-11T15:11:17.383Z"), ), ] bicycle = db.get_model("bicycle") assert bicycle.name == "bicycle" assert bicycle.display_name == "Bicycle" assert bicycle.id == "dddddddd-dddd-dddd-dddd-dddddddddddd" assert bicycle.count == 1 assert bicycle.template_id == None properties = sorted(db.get_properties(bicycle), key=lambda p: p.index) assert len(properties) == 2 brand = properties[0] assert brand.name == "brand" assert brand.id == "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee" color = properties[1] assert color.name == "color" assert color.data_type == dt.Array( items=dt.String(), enum=["purple", "blue", "orange", "green", "yellow", "red"]) assert color.id == "ffffffff-ffff-ffff-ffff-ffffffffffff" # Records patients = db.get_all_records("patient") alice = Record( id=UUID("00000000-0000-0000-0000-000000000000"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", created_at=iso8601.parse("2019-11-01T20:01:58.537Z"), updated_at=iso8601.parse("2019-11-11T15:37:02.165Z"), values={ "name": "Alice", "dob": neotime.DateTime(year=2004, month=5, day=5, tzinfo=pytz.UTC), }, ) bob = Record( id=UUID("11111111-1111-1111-1111-111111111111"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", created_at=iso8601.parse("2019-11-01T20:02:21.113Z"), updated_at=iso8601.parse("2019-11-01T20:02:21.113Z"), values={ "name": "Bob", # Embedded linked property "mother": RecordStub(id=UUID("00000000-0000-0000-0000-000000000000"), title="Alice"), }, ) assert sorted(patients.results, key=lambda x: x.values["name"]) == [alice, bob] assert db.get_all_records("bicycle").results == [ Record( id=UUID("22222222-2222-2222-2222-222222222222"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", created_at=iso8601.parse("2019-11-05T13:47:02.841Z"), updated_at=iso8601.parse("2019-11-11T15:12:28.042Z"), values={ "brand": "Bianchi", "color": ["red", "blue"] }, ) ] # Model relationships with db.transaction() as tx: assert list( db.get_outgoing_model_relationships_tx(tx, patient, one_to_many=True) ) == [ ModelRelationship( id="33333333-3333-3333-3333-333333333333", type="RIDES", name="rides_c83d5af0-ffd2-11e9-b8f0-1b1d6297ff8c", display_name="Rides", description="", from_="aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", to="dddddddd-dddd-dddd-dddd-dddddddddddd", one_to_many=True, index=None, created_at=iso8601.parse("2019-11-05T13:47:17.981Z"), updated_at=iso8601.parse("2019-11-05T13:47:17.981Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] assert list( db.get_outgoing_model_relationships_tx(tx, bicycle, one_to_many=True) ) == [ ModelRelationship( id="44444444-4444-4444-4444-444444444444", type="BELONGS_TO", name="belongs_to", display_name="Belongs To", description="", from_="dddddddd-dddd-dddd-dddd-dddddddddddd", to="aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", one_to_many=True, index=None, created_at=iso8601.parse("2019-11-21T16:47:36.918Z"), updated_at=iso8601.parse("2019-11-21T16:47:36.918Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] # Model relationship stubs contain no "to" and "from" models, eg. belongs_to assert list(db.get_model_relationship_stubs_tx(tx)) == [ ModelRelationshipStub( id="55555555-5555-5555-5555-555555555555", name="belongs_to", display_name="Belongs To", description="", type="BELONGS_TO", created_at=iso8601.parse("2019-11-05T13:44:38.598Z"), updated_at=iso8601.parse("2019-11-05T13:44:38.598Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] # Duplicate @RELATED_TO relationships are created assert len(db.get_model_relationships_tx(tx, one_to_many=True)) == 2 # Record relationships assert list(db.get_outgoing_record_relationships_tx(tx, alice)) == [ RecordRelationship( id="77777777-7777-7777-7777-777777777777", from_="00000000-0000-0000-0000-000000000000", to="22222222-2222-2222-2222-222222222222", type="RIDES", name="rides_c83d5af0-ffd2-11e9-b8f0-1b1d6297ff8c", model_relationship_id="33333333-3333-3333-3333-333333333333", display_name="Rides", one_to_many=True, created_at=iso8601.parse("2019-11-05T13:47:46.032Z"), updated_at=iso8601.parse("2019-11-05T13:47:46.032Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] assert list( db.get_outgoing_record_relationships_tx( tx, "22222222-2222-2222-2222-222222222222", one_to_many=True) ) == [ RecordRelationship( id="88888888-8888-8888-8888-888888888888", from_="22222222-2222-2222-2222-222222222222", to="11111111-1111-1111-1111-111111111111", type="BELONGS_TO", name="belongs_to", model_relationship_id="44444444-4444-4444-4444-444444444444", display_name="Belongs To", one_to_many=True, created_at=iso8601.parse("2019-11-21T16:47:36.938Z"), updated_at=iso8601.parse("2019-11-21T16:47:36.938Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] # Linked properties assert list( db.get_outgoing_model_relationships_tx(tx, patient, one_to_many=False) ) == [ ModelRelationship( id="66666666-6666-6666-6666-666666666666", type="MOTHER", name="mother", display_name="Mother", description="", from_="aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", to="aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", one_to_many=False, index=1, created_at=iso8601.parse("2019-11-05T13:43:38.341Z"), updated_at=iso8601.parse("2019-11-05T13:43:38.341Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] # Duplicate @RELATED_TO relationships are created for linked properties assert len(db.get_model_relationships_tx(tx, one_to_many=False)) == 1 assert list( db.get_outgoing_record_relationships_tx(tx, bob, one_to_many=False) ) == [ RecordRelationship( id="99999999-9999-9999-9999-999999999999", from_="11111111-1111-1111-1111-111111111111", to="00000000-0000-0000-0000-000000000000", type="MOTHER", model_relationship_id="66666666-6666-6666-6666-666666666666", name="mother", display_name="Mother", one_to_many=False, created_at=iso8601.parse("2019-11-05T13:43:54.116Z"), updated_at=iso8601.parse("2019-11-05T13:43:54.116Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] assert db.get_package_proxies_for_record(alice, limit=10, offset=0) == ( 1, [ PackageProxy( id="aaaaaaaa-aaaa-aaaa-aaaa-bbbbbbbbbbbb", proxy_instance_id="aaaaaaaa-aaaa-aaaa-aaaa-cccccccccccc", package_id=184418, package_node_id= "N:package:b493794a-1c86-4c18-9fb9-dfdf236b1fe3", relationship_type="belongs_to", created_at=iso8601.parse("2019-11-05T13:44:38.748Z"), updated_at=iso8601.parse("2019-11-05T13:44:38.748Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ], ) # Packages link directly to dataset node assert db.count_packages() == 1
def test_fix_multiple_mismatched_model_relationships(neo4j): """ Case when a record relationship has a schemaRelationshipId that does not match the (from)-[relationship]->(to) triple of the relationship, but a compatible model relationship does exist in the dataset. In this case, an extra model relationship (id = `4ea7d9e6-b2ae-487c-981c-8202e37343f9`) connects (bike)-[rides_c83d5af0-ffd2-11e9-b8f0-1b1d6297ff8c]->(bike) and the record relationship (person)-[rides]->(bike) incorrectly uses this schemaRelationshipId. AND Another record relationship, with the same relationship type, also claiming the same schemaRelationshipId, but actually connecting different models. This corrupted data is from old versions of the graph-ingest tool. """ dataset_id = 50000 dataset_node_id = "N:dataset:b1154216-d1d7-4484-ad18-81b58fb65484" organization_id = 5 organization_node_id = "N:organization:c905919f-56f5-43ae-9c2a-8d5d542c133b" user_id = 114 user_node_id = "N:user:028058b9-dd8d-4f24-a187-ea56830b379f" db = PartitionedDatabase( db=neo4j, organization_id=OrganizationId(organization_id), dataset_id=DatasetId(dataset_id), user_id=user_node_id, organization_node_id=organization_node_id, dataset_node_id=dataset_node_id, ) load( dataset=f"{organization_id}/{dataset_id}", bucket="dev-neptune-export-use1", db=db, use_cache=False, smoke_test=False, ) with db.transaction() as tx: assert list( db.get_outgoing_record_relationships_tx( tx, "ecb71447-b684-c589-abda-b673c38edefc") ) == [ RecordRelationship( id="d0b71de9-21f9-3557-edda-ad278dd81dc0", from_="ecb71447-b684-c589-abda-b673c38edefc", to="c8b71de8-cd9c-cc3f-67fe-4e30968d4e50", type="RIDES", name="rides_c83d5af0-ffd2-11e9-b8f0-1b1d6297ff8c", model_relationship_id="2e754729-684a-4c45-960f-348d68737d4d", display_name="Rides", one_to_many=True, created_at=iso8601.parse("2019-11-05T13:47:46.032Z"), updated_at=iso8601.parse("2019-11-05T13:47:46.032Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] assert sorted( db.get_outgoing_record_relationships_tx( tx, "c8b71de8-cd9c-cc3f-67fe-4e30968d4e50", one_to_many=True), key=lambda r: r.type, ) == [ RecordRelationship( id="aeb7476e-55f6-7924-5e43-a83cfa7e4cef", from_="c8b71de8-cd9c-cc3f-67fe-4e30968d4e50", to="e2b71447-e29d-11c3-24c6-f2ebffd1486a", type="BELONGS_TO", name="belongs_to", model_relationship_id="175ff55b-b44d-4381-bd59-d4dbc0b9c5f0", display_name="Belongs To", one_to_many=True, created_at=iso8601.parse("2019-11-21T16:47:36.938Z"), updated_at=iso8601.parse("2019-11-21T16:47:36.938Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ), RecordRelationship( id="83227fcd-ec4d-47e6-a3a9-4c9fbba4a2f3", from_="c8b71de8-cd9c-cc3f-67fe-4e30968d4e50", to="e2b71447-e29d-11c3-24c6-f2ebffd1486a", type="RIDES", name="rides_c83d5af0-ffd2-11e9-b8f0-1b1d6297ff8c", model_relationship_id="92b6a32f-0597-4a7a-944b-27b39998283e", display_name="Rides", one_to_many=True, created_at=iso8601.parse("2019-11-05T13:47:46.032Z"), updated_at=iso8601.parse("2019-11-05T13:47:46.032Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ), ]
def test_fix_nonexistent_model_relationships(neo4j): """ Case when a record relationship has a schemaRelationshipId that does not exist, but a model relationhip (d0b71de9-21f9-3557-edda-ad278dd81dc0) is defined that matches the (from)-[relationship]->(to) triple of the relationship. This corrupted data is from old versions of the graph-ingest tool. """ dataset_id = 30000 dataset_node_id = "N:dataset:b1154216-d1d7-4484-ad18-81b58fb65484" organization_id = 5 organization_node_id = "N:organization:c905919f-56f5-43ae-9c2a-8d5d542c133b" user_id = 114 user_node_id = "N:user:028058b9-dd8d-4f24-a187-ea56830b379f" db = PartitionedDatabase( db=neo4j, organization_id=OrganizationId(organization_id), dataset_id=DatasetId(dataset_id), user_id=user_node_id, organization_node_id=organization_node_id, dataset_node_id=dataset_node_id, ) load( dataset=f"{organization_id}/{dataset_id}", bucket="dev-neptune-export-use1", db=db, use_cache=False, smoke_test=False, ) with db.transaction() as tx: assert list( db.get_outgoing_record_relationships_tx( tx, "ecb71447-b684-c589-abda-b673c38edefc") ) == [ RecordRelationship( id="d0b71de9-21f9-3557-edda-ad278dd81dc0", from_="ecb71447-b684-c589-abda-b673c38edefc", to="c8b71de8-cd9c-cc3f-67fe-4e30968d4e50", type="RIDES", name="rides_c83d5af0-ffd2-11e9-b8f0-1b1d6297ff8c", model_relationship_id="2e754729-684a-4c45-960f-348d68737d4d", display_name="Rides", one_to_many=True, created_at=iso8601.parse("2019-11-05T13:47:46.032Z"), updated_at=iso8601.parse("2019-11-05T13:47:46.032Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] assert list( db.get_outgoing_record_relationships_tx( tx, "c8b71de8-cd9c-cc3f-67fe-4e30968d4e50", one_to_many=True) ) == [ RecordRelationship( id="aeb7476e-55f6-7924-5e43-a83cfa7e4cef", from_="c8b71de8-cd9c-cc3f-67fe-4e30968d4e50", to="e2b71447-e29d-11c3-24c6-f2ebffd1486a", type="BELONGS_TO", name="belongs_to", model_relationship_id="175ff55b-b44d-4381-bd59-d4dbc0b9c5f0", display_name="Belongs To", one_to_many=True, created_at=iso8601.parse("2019-11-21T16:47:36.938Z"), updated_at=iso8601.parse("2019-11-21T16:47:36.938Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ]
def test_loader(neo4j): dataset_id = 29233 dataset_node_id = "N:dataset:b1154216-d1d7-4484-ad18-81b58fb65484" organization_id = 5 organization_node_id = "N:organization:c905919f-56f5-43ae-9c2a-8d5d542c133b" user_id = 114 user_node_id = "N:user:028058b9-dd8d-4f24-a187-ea56830b379f" db = PartitionedDatabase( db=neo4j, organization_id=OrganizationId(organization_id), dataset_id=DatasetId(dataset_id), user_id=user_node_id, organization_node_id=organization_node_id, dataset_node_id=dataset_node_id, ) load( dataset=f"{organization_id}/{dataset_id}", bucket="dev-neptune-export-use1", db=db, use_cache=False, smoke_test=False, ) # Models patient = db.get_model("patient") assert patient == Model( name="patient", display_name="Patient", description="", count=2, id="0b4b3615-9eaf-425d-9727-bcac29686fd5", created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", created_at=iso8601.parse("2019-11-01T20:01:27.027Z"), updated_at=iso8601.parse("2019-11-01T20:01:27.027Z"), template_id=None, ) assert sorted(db.get_properties(patient), key=lambda p: p.index) == [ ModelProperty( id="7b17c60d-ca2a-4cf5-a4ff-a52bbc32ff17", name="name", display_name="Name", description="", index=0, locked=False, model_title=True, required=False, data_type=dt.String(), default=True, default_value=None, created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", created_at=iso8601.parse("2019-11-01T20:01:37.633Z"), updated_at=iso8601.parse("2019-11-11T15:11:17.383Z"), ), ModelProperty( id="e507b3ef-ade4-4672-83b4-f3f0774fb282", name="dob", display_name="DOB", description="", index=1, locked=False, model_title=False, required=False, data_type=dt.Date(), default=False, default_value=None, created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", created_at=iso8601.parse("2019-11-11T15:11:17.383Z"), updated_at=iso8601.parse("2019-11-11T15:11:17.383Z"), ), ] bicycle = db.get_model("bicycle") assert bicycle.name == "bicycle" assert bicycle.display_name == "Bicycle" assert bicycle.id == "bf858cb5-ae51-4fcf-ad74-b1887946f70f" assert bicycle.count == 1 assert bicycle.template_id == None properties = sorted(db.get_properties(bicycle), key=lambda p: p.index) assert len(properties) == 2 brand = properties[0] assert brand.name == "brand" color = properties[1] assert color.name == "color" assert color.data_type == dt.Array( items=dt.String(), enum=["purple", "blue", "orange", "green", "yellow", "red"]) # Records patients = db.get_all_records("patient") alice = Record( id=UUID("ecb71447-b684-c589-abda-b673c38edefc"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", created_at=iso8601.parse("2019-11-01T20:01:58.537Z"), updated_at=iso8601.parse("2019-11-11T15:37:02.165Z"), values={ "name": "Alice", "dob": neotime.DateTime(year=2004, month=5, day=5, tzinfo=pytz.UTC), }, ) bob = Record( id=UUID("e2b71447-e29d-11c3-24c6-f2ebffd1486a"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", created_at=iso8601.parse("2019-11-01T20:02:21.113Z"), updated_at=iso8601.parse("2019-11-01T20:02:21.113Z"), values={ "name": "Bob", # Embedded linked property "mother": RecordStub(id=UUID("ecb71447-b684-c589-abda-b673c38edefc"), title="Alice"), }, ) assert sorted(patients.results, key=lambda x: x.values["name"]) == [alice, bob] assert db.get_all_records("bicycle").results == [ Record( id=UUID("c8b71de8-cd9c-cc3f-67fe-4e30968d4e50"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", created_at=iso8601.parse("2019-11-05T13:47:02.841Z"), updated_at=iso8601.parse("2019-11-11T15:12:28.042Z"), values={ "brand": "Bianchi", "color": ["red", "blue"] }, ) ] # Model relationships with db.transaction() as tx: assert list( db.get_outgoing_model_relationships_tx(tx, patient, one_to_many=True) ) == [ ModelRelationship( id="2e754729-684a-4c45-960f-348d68737d4d", type="RIDES", name="rides_c83d5af0-ffd2-11e9-b8f0-1b1d6297ff8c", display_name="Rides", description="", from_="0b4b3615-9eaf-425d-9727-bcac29686fd5", to="bf858cb5-ae51-4fcf-ad74-b1887946f70f", one_to_many=True, index=None, created_at=iso8601.parse("2019-11-05T13:47:17.981Z"), updated_at=iso8601.parse("2019-11-05T13:47:17.981Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] # This relationship can be created in the Python client with the following: # # >>> patient = ds.models()["patient"] # >>> bike = ds.models()["bicycle"] # >>> bob = patient.get_all()[1] # >>> bianchi = bike.get_all()[0] # >>> bianchi.relate_to(bob, relationship_type="belongs_to") # # This reuses the `belongs_to` name even though that is disallowed through # the frontend. This means that the `belongs_to` CSV contains relationships # between proxy packages and records, *and* between records and records. assert list( db.get_outgoing_model_relationships_tx(tx, bicycle, one_to_many=True) ) == [ ModelRelationship( id="175ff55b-b44d-4381-bd59-d4dbc0b9c5f0", type="BELONGS_TO", name="belongs_to", display_name="Belongs To", description="", from_="bf858cb5-ae51-4fcf-ad74-b1887946f70f", to="0b4b3615-9eaf-425d-9727-bcac29686fd5", one_to_many=True, index=None, created_at=iso8601.parse("2019-11-21T16:47:36.918Z"), updated_at=iso8601.parse("2019-11-21T16:47:36.918Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] # Model relationship stubs contain no "to" and "from" models, eg. belongs_to assert list(db.get_model_relationship_stubs_tx(tx)) == [ ModelRelationshipStub( id="ccf200d3-e77f-4d9e-bed3-f1f28860152f", name="belongs_to", display_name="Belongs To", description="", type="BELONGS_TO", created_at=iso8601.parse("2019-11-05T13:44:38.598Z"), updated_at=iso8601.parse("2019-11-05T13:44:38.598Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] # Duplicate @RELATED_TO relationships are created assert len(db.get_model_relationships_tx(tx, one_to_many=True)) == 2 # Record relationships assert list(db.get_outgoing_record_relationships_tx(tx, alice)) == [ RecordRelationship( id="d0b71de9-21f9-3557-edda-ad278dd81dc0", from_="ecb71447-b684-c589-abda-b673c38edefc", to="c8b71de8-cd9c-cc3f-67fe-4e30968d4e50", type="RIDES", name="rides_c83d5af0-ffd2-11e9-b8f0-1b1d6297ff8c", model_relationship_id="2e754729-684a-4c45-960f-348d68737d4d", display_name="Rides", one_to_many=True, created_at=iso8601.parse("2019-11-05T13:47:46.032Z"), updated_at=iso8601.parse("2019-11-05T13:47:46.032Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] assert list( db.get_outgoing_record_relationships_tx( tx, "c8b71de8-cd9c-cc3f-67fe-4e30968d4e50", one_to_many=True) ) == [ RecordRelationship( id="aeb7476e-55f6-7924-5e43-a83cfa7e4cef", from_="c8b71de8-cd9c-cc3f-67fe-4e30968d4e50", to="e2b71447-e29d-11c3-24c6-f2ebffd1486a", type="BELONGS_TO", name="belongs_to", model_relationship_id="175ff55b-b44d-4381-bd59-d4dbc0b9c5f0", display_name="Belongs To", one_to_many=True, created_at=iso8601.parse("2019-11-21T16:47:36.938Z"), updated_at=iso8601.parse("2019-11-21T16:47:36.938Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] # Linked properties assert list( db.get_outgoing_model_relationships_tx(tx, patient, one_to_many=False) ) == [ ModelRelationship( id="443e141b-f59c-419f-82c1-eed97925b04d", type="MOTHER", name="mother", display_name="Mother", description="", from_="0b4b3615-9eaf-425d-9727-bcac29686fd5", to="0b4b3615-9eaf-425d-9727-bcac29686fd5", one_to_many=False, index=1, created_at=iso8601.parse("2019-11-05T13:43:38.341Z"), updated_at=iso8601.parse("2019-11-05T13:43:38.341Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] # Duplicate @RELATED_TO relationships are created for linked properties assert len(db.get_model_relationships_tx(tx, one_to_many=False)) == 1 assert list( db.get_outgoing_record_relationships_tx(tx, bob, one_to_many=False) ) == [ RecordRelationship( id="fa3daedd-1761-4730-be7d-bb5de8e1261c", from_="e2b71447-e29d-11c3-24c6-f2ebffd1486a", to="ecb71447-b684-c589-abda-b673c38edefc", type="MOTHER", model_relationship_id="443e141b-f59c-419f-82c1-eed97925b04d", name="mother", display_name="Mother", one_to_many=False, created_at=iso8601.parse("2019-11-05T13:43:54.116Z"), updated_at=iso8601.parse("2019-11-05T13:43:54.116Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ] assert db.get_package_proxies_for_record(alice, limit=10, offset=0) == ( 1, [ PackageProxy( id="00b71de7-b42f-1fe9-a83f-824452fe966e", proxy_instance_id="460591a0-8079-4979-a860-c3a4b18a32ad", package_id=184418, package_node_id= "N:package:b493794a-1c86-4c18-9fb9-dfdf236b1fe3", relationship_type="belongs_to", created_at=iso8601.parse("2019-11-05T13:44:38.748Z"), updated_at=iso8601.parse("2019-11-05T13:44:38.748Z"), created_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", updated_by="N:user:06080380-fb56-46eb-8c70-f24112aff878", ) ], ) # Packages link directly to dataset node assert db.count_packages() == 1
def test_rewrite_ids_randomly_and_import(neo4j): """ Test that UUIDs are remapped using the default random remapper. """ dataset_id = 70000 dataset_node_id = "N:dataset:b1154216-d1d7-4484-ad18-81b58fb65484" organization_id = 5 organization_node_id = "N:organization:c905919f-56f5-43ae-9c2a-8d5d542c133b" user_id = 114 user_node_id = "N:user:028058b9-dd8d-4f24-a187-ea56830b379f" db = PartitionedDatabase( db=neo4j, organization_id=OrganizationId(organization_id), dataset_id=DatasetId(dataset_id), user_id=user_node_id, organization_node_id=organization_node_id, dataset_node_id=dataset_node_id, ) load( dataset=f"{organization_id}/{dataset_id}", bucket="dev-neptune-export-use1", db=db, use_cache=False, smoke_test=False, remap_ids=True, ) # Models patient = db.get_model("patient") assert patient is not None assert len(db.get_properties(patient)) == 2 assert patient.id != UUID("0b4b3615-9eaf-425d-9727-bcac29686fd5") bicycle = db.get_model("bicycle") assert bicycle is not None assert len(db.get_properties(bicycle)) == 2 assert bicycle.id != UUID("bf858cb5-ae51-4fcf-ad74-b1887946f70f") # Records patients = db.get_all_records("patient") alice = [r for r in patients if r.values["name"] == "Alice"][0] assert alice.id != "ecb71447-b684-c589-abda-b673c38edefc" bob = [r for r in patients if r.values["name"] == "Bob"][0] assert bob.id != UUID("e2b71447-e29d-11c3-24c6-f2ebffd1486a") assert len(db.get_all_records("bicycle").results) == 1 bianchi = db.get_all_records("bicycle").results[0] assert bianchi.id != UUID("c8b71de8-cd9c-cc3f-67fe-4e30968d4e50") # Model relationships with db.transaction() as tx: assert (len( list( db.get_outgoing_model_relationships_tx( tx, patient, one_to_many=True))) == 1) assert (len( list( db.get_outgoing_model_relationships_tx( tx, bicycle, one_to_many=True))) == 1) # Model relationship stubs contain no "to" and "from" models, eg. belongs_to assert len(list(db.get_model_relationship_stubs_tx(tx))) == 1 # Duplicate @RELATED_TO relationships are created assert len(list(db.get_model_relationships_tx(tx, one_to_many=True))) == 2 # Record relationships assert len(list(db.get_outgoing_record_relationships_tx(tx, alice))) == 1 assert (len( list( db.get_outgoing_record_relationships_tx( tx, bianchi, one_to_many=True))) == 1) # Linked properties assert (len( list( db.get_outgoing_model_relationships_tx( tx, patient, one_to_many=False))) == 1) # Duplicate @RELATED_TO relationships are created for linked properties assert len(list(db.get_model_relationships_tx(tx, one_to_many=False))) == 1 assert (len( list( db.get_outgoing_record_relationships_tx( tx, bob, one_to_many=False))) == 1) assert len( db.get_package_proxies_for_record(alice, limit=10, offset=0)[1]) == 1 # Packages link directly to dataset node assert db.count_packages() == 1