def setUp(self):
DatabaseIDModel.objects.create()
(self.current_id, _) = InstanceIDModel.get_or_create_current_instance()
self.mc = MorangoProfileController('facilitydata')
self.fac1 = FacilityModelFactory(name='school')
self.mc.serialize_into_store()
self.old_rmc = RecordMaxCounter.objects.first()
def setUp(self):
self.profile = "facilitydata"
self.root_scope_def = ScopeDefinition.objects.create(
id="rootcert",
profile=self.profile,
version=1,
primary_scope_param_key="mainpartition",
description="Root cert for ${mainpartition}.",
read_filter_template="",
write_filter_template="",
read_write_filter_template="${mainpartition}",
)
self.subset_scope_def = ScopeDefinition.objects.create(
id="subcert",
profile=self.profile,
version=1,
primary_scope_param_key="",
description=
"Subset cert under ${mainpartition} for ${subpartition}.",
read_filter_template="${mainpartition}",
write_filter_template="${mainpartition}:${subpartition}",
read_write_filter_template="",
)
self.root_cert = Certificate.generate_root_certificate(
self.root_scope_def.id)
self.subset_cert = Certificate(
parent=self.root_cert,
profile=self.profile,
scope_definition=self.subset_scope_def,
scope_version=self.subset_scope_def.version,
scope_params=json.dumps({
"mainpartition": self.root_cert.id,
"subpartition": "abracadabra"
}),
private_key=Key(),
)
self.root_cert.sign_certificate(self.subset_cert)
self.subset_cert.save()
self.unsaved_cert = Certificate(
parent=self.root_cert,
profile=self.profile,
scope_definition=self.subset_scope_def,
scope_version=self.subset_scope_def.version,
scope_params=json.dumps({
"mainpartition": self.root_cert.id,
"subpartition": "other"
}),
public_key=Key(),
)
self.root_cert.sign_certificate(self.unsaved_cert)
self.controller = MorangoProfileController('facilitydata')
self.network_connection = self.controller.create_network_connection(
self.live_server_url)
self.key = SharedKey.get_or_create_shared_key()
def handle_async(self, *args, **options):
# call this in case user directly syncs without migrating database
if not ScopeDefinition.objects.filter():
call_command("loaddata", "scopedefinitions")
controller = MorangoProfileController('facilitydata')
with self.start_progress(total=7) as progress_update:
network_connection = controller.create_network_connection(options['base_url'])
progress_update(1)
options['dataset_id'] = self.get_dataset_id(options['base_url'], options['dataset_id'])
progress_update(1)
client_cert, server_cert = self.get_client_and_server_certs(options['username'], options['password'],
options['dataset_id'], network_connection)
progress_update(1)
sync_client = network_connection.create_sync_session(client_cert, server_cert)
progress_update(1)
# pull from server and push our own data to server
if not options['no_pull']:
sync_client.initiate_pull(Filter(options['dataset_id']))
if not options['no_push']:
sync_client.initiate_push(Filter(options['dataset_id']))
progress_update(1)
self.create_superuser_and_provision_device(options['username'], options['dataset_id'])
progress_update(1)
sync_client.close_sync_session()
progress_update(1)
def setUp(self):
DatabaseIDModel.objects.create()
InstanceIDModel.get_or_create_current_instance()
self.range = 10
self.mc = MorangoProfileController('facilitydata')
self.original_name = "ralphie"
self.new_name = "rafael"
def setUp(self):
(self.current_id, _) = InstanceIDModel.get_or_create_current_instance()
self.range = 10
self.mc = MorangoProfileController('facilitydata')
for i in range(self.range):
self.ident = uuid.uuid4().hex
StoreModelFacilityFactory(pk=self.ident,
serialized=serialized_facility_factory(
self.ident))
def handle_async(self, *args, **options):
# validate url that is passed in
try:
URLValidator()((options['base_url']))
except ValidationError:
print('Base-url is not valid. Please retry command and enter a valid url.')
sys.exit(1)
# call this in case user directly syncs without migrating database
if not ScopeDefinition.objects.filter():
call_command("loaddata", "scopedefinitions")
# ping server at url with info request
info_url = urljoin(options['base_url'], 'api/morango/v1/morangoinfo/1/')
try:
info_resp = requests.get(info_url)
except ConnectionError:
print('Can not connect to server with base-url: {}'.format(options['base_url']))
sys.exit(1)
# if instance_ids are equal, this means device is trying to sync with itself, which we don't allow
if InstanceIDModel.get_or_create_current_instance()[0].id == info_resp.json()['instance_id']:
print('Device can not sync with itself. Please re-check base-url and try again.')
sys.exit(1)
controller = MorangoProfileController('facilitydata')
with self.start_progress(total=7) as progress_update:
network_connection = controller.create_network_connection(options['base_url'])
progress_update(1)
options['dataset_id'] = self.get_dataset_id(options['base_url'], options['dataset_id'])
progress_update(1)
client_cert, server_cert, options['username'] = self.get_client_and_server_certs(options['username'], options['password'],
options['dataset_id'], network_connection)
progress_update(1)
sync_client = network_connection.create_sync_session(client_cert, server_cert)
progress_update(1)
# pull from server and push our own data to server
if not options['no_pull']:
sync_client.initiate_pull(Filter(options['dataset_id']))
if not options['no_push']:
sync_client.initiate_push(Filter(options['dataset_id']))
progress_update(1)
self.create_superuser_and_provision_device(options['username'], options['dataset_id'])
progress_update(1)
sync_client.close_sync_session()
progress_update(1)
class DateTimeTZFieldTestCase(TestCase):
def setUp(self):
self.controller = MorangoProfileController("facilitydata")
InstanceIDModel.get_or_create_current_instance()
def test_deserializing_field(self):
facility = Facility.objects.create(name="hallo")
FacilityUser.objects.create(username="******", facility=facility)
self.controller.serialize_into_store()
Store.objects.update(dirty_bit=True)
try:
self.controller.deserialize_from_store()
except AttributeError as e:
self.fail(e.message)
class DateTimeTZFieldTestCase(TestCase):
def setUp(self):
self.controller = MorangoProfileController('facilitydata')
InstanceIDModel.get_or_create_current_instance()
@unittest.skipIf(os.environ.get('TOX_ENV') == 'postgres', "Skipping testing on postgres because sql is not compatible.")
def test_deserializing_field(self):
facility = Facility.objects.create(name="hallo")
FacilityUser.objects.create(username='******', facility=facility)
self.controller.serialize_into_store()
Store.objects.update(dirty_bit=True)
try:
self.controller.deserialize_from_store()
except AttributeError as e:
self.fail(e.message)
class RecordMaxCounterUpdatesDuringSerialization(TestCase):
def setUp(self):
DatabaseIDModel.objects.create()
(self.current_id, _) = InstanceIDModel.get_or_create_current_instance()
self.mc = MorangoProfileController('facilitydata')
self.fac1 = FacilityModelFactory(name='school')
self.mc.serialize_into_store()
self.old_rmc = RecordMaxCounter.objects.first()
def test_new_rmc_for_existing_model(self):
with mock.patch('platform.platform', return_value='Windows'):
(new_id, _) = InstanceIDModel.get_or_create_current_instance()
Facility.objects.update(name='facility')
self.mc.serialize_into_store()
new_rmc = RecordMaxCounter.objects.get(instance_id=new_id.id,
store_model_id=self.fac1.id)
new_store_record = Store.objects.get(id=self.fac1.id)
self.assertEqual(new_rmc.counter, new_store_record.last_saved_counter)
self.assertEqual(new_rmc.instance_id,
new_store_record.last_saved_instance)
def test_update_rmc_for_existing_model(self):
Facility.objects.update(name='facility')
self.mc.serialize_into_store()
# there should only be 1 RecordMaxCounter for a specific instance_id and a specific model (unique_together)
self.assertEqual(
RecordMaxCounter.objects.filter(
instance_id=self.current_id.id,
store_model_id=self.fac1.id).count(), 1)
new_rmc = RecordMaxCounter.objects.get(instance_id=self.current_id.id,
store_model_id=self.fac1.id)
new_store_record = Store.objects.get(id=self.fac1.id)
self.assertEqual(self.old_rmc.counter + 1, new_rmc.counter)
self.assertEqual(new_rmc.counter, new_store_record.last_saved_counter)
self.assertEqual(new_rmc.instance_id,
new_store_record.last_saved_instance)
def test_new_rmc_for_non_existent_model(self):
with mock.patch('platform.platform', return_value='Windows'):
(new_id, _) = InstanceIDModel.get_or_create_current_instance()
new_fac = FacilityModelFactory(name='college')
self.mc.serialize_into_store()
new_rmc = RecordMaxCounter.objects.get(instance_id=new_id.id,
store_model_id=new_fac.id)
new_store_record = Store.objects.get(id=new_fac.id)
self.assertNotEqual(new_id.id, self.current_id.id)
self.assertEqual(new_store_record.last_saved_instance,
new_rmc.instance_id)
self.assertEqual(new_store_record.last_saved_counter, new_rmc.counter)
class PostDeleteSignalsTestCase(TestCase):
def setUp(self):
InstanceIDModel.get_or_create_current_instance()
[FacilityModelFactory() for _ in range(10)]
self.mc = MorangoProfileController('facilitydata')
self.mc.serialize_into_store()
def test_deleted_flag_gets_set(self):
facility = Facility.objects.first()
deleted_id = facility.id
facility.delete()
self.assertTrue(DeletedModels.objects.filter(id=deleted_id))
def test_cascading_delete(self):
facility = Facility.objects.first()
child = FacilityModelFactory(parent=facility)
deleted_child_id = child.id
facility.delete()
self.assertTrue(DeletedModels.objects.filter(id=deleted_child_id))
class SelfReferentialFKDeserializationTestCase(TestCase):
def setUp(self):
(self.current_id, _) = InstanceIDModel.get_or_create_current_instance()
self.mc = MorangoProfileController('facilitydata')
def test_self_ref_fk(self):
self.assertEqual(_self_referential_fk(Facility), 'parent_id')
self.assertEqual(_self_referential_fk(MyUser), None)
def test_delete_model_in_store_deletes_models_in_app(self):
root = FacilityModelFactory()
child1 = FacilityModelFactory(parent=root)
child2 = FacilityModelFactory(parent=root)
self.mc.serialize_into_store()
# simulate a node being deleted and synced
Store.objects.filter(id=child2.id).update(deleted=True)
Store.objects.update(dirty_bit=True)
grandchild1 = FacilityModelFactory(parent=child2)
grandchild2 = FacilityModelFactory(parent=child2)
self.mc.deserialize_from_store()
# ensure tree structure in app layer is correct
child1 = Facility.objects.filter(id=child1.id)
self.assertTrue(child1.exists())
self.assertEqual(child1[0].parent_id, root.id)
self.assertFalse(Facility.objects.filter(id=child2.id).exists())
self.assertFalse(Facility.objects.filter(id=grandchild1.id).exists())
self.assertFalse(Facility.objects.filter(id=grandchild2.id).exists())
def test_models_created_successfully(self):
root = FacilityModelFactory()
child1 = FacilityModelFactory(parent=root)
child2 = FacilityModelFactory(parent=root)
self.mc.serialize_into_store()
Facility.objects.all().delete()
DeletedModels.objects.all().delete()
Store.objects.update(dirty_bit=True, deleted=False)
self.mc.deserialize_from_store()
# ensure tree structure in app layer is correct
self.assertTrue(Facility.objects.filter(id=root.id).exists())
child1 = Facility.objects.filter(id=child1.id)
self.assertTrue(child1.exists())
self.assertEqual(child1[0].parent_id, root.id)
child2 = Facility.objects.filter(id=child2.id)
self.assertTrue(child2.exists())
self.assertEqual(child2[0].parent_id, root.id)
def create_dummy_store_data():
data = {}
DatabaseIDModel.objects.create()
(data['group1_id'], _) = InstanceIDModel.get_or_create_current_instance() # counter is at 0
# create controllers for app/store/buffer operations
data['mc'] = MorangoProfileController('facilitydata')
data['sc'] = SyncClient(None, 'host')
session = SyncSession.objects.create(id=uuid.uuid4().hex, profile="facilitydata", last_activity_timestamp=timezone.now())
data['sc'].current_transfer_session = TransferSession.objects.create(id=uuid.uuid4().hex, sync_session=session, push=True, last_activity_timestamp=timezone.now())
data['mc'].serialize_into_store() # counter is at 1
# create group of facilities and first serialization
data['group1_c1'] = [FacilityFactory() for _ in range(5)]
data['mc'].serialize_into_store() # counter is at 2
# create group of facilities and second serialization
data['group1_c2'] = [FacilityFactory() for _ in range(5)]
# create users and logs associated with user
data['user1'] = MyUser.objects.create(username='******')
data['user1_sumlogs'] = [SummaryLog.objects.create(user=data['user1']) for _ in range(5)]
data['mc'].serialize_into_store() # counter is at 3
# create new instance id and group of facilities
with mock.patch('platform.platform', return_value='plataforma'):
(data['group2_id'], _) = InstanceIDModel.get_or_create_current_instance() # new counter is at 0
data['mc'].serialize_into_store() # new counter is at 1
data['group2_c1'] = [FacilityFactory() for _ in range(5)]
# create users and logs associated with user
data['user2'] = MyUser.objects.create(username='******')
data['user2_sumlogs'] = [SummaryLog.objects.create(user=data['user2']) for _ in range(5)]
data['user2_interlogs'] = [InteractionLog.objects.create(user=data['user2']) for _ in range(5)]
data['user3'] = MyUser.objects.create(username='******')
data['user3_sumlogs'] = [SummaryLog.objects.create(user=data['user3']) for _ in range(5)]
data['user3_interlogs'] = [InteractionLog.objects.create(user=data['user3']) for _ in range(5)]
data['mc'].serialize_into_store() # new counter is at 2
data['user4'] = MyUser.objects.create(username='******', _morango_partition='badpartition')
data['mc'].serialize_into_store() # new counter is at 3
return data
def setUp(self):
settings.MORANGO_DESERIALIZE_AFTER_DEQUEUING = False
self.data = {}
DatabaseIDModel.objects.create()
(self.current_id, _) = InstanceIDModel.get_or_create_current_instance()
# create controllers for app/store/buffer operations
self.data['mc'] = MorangoProfileController('facilitydata')
self.data['sc'] = SyncClient(None, 'host')
session = SyncSession.objects.create(
id=uuid.uuid4().hex,
profile="",
last_activity_timestamp=timezone.now())
self.data[
'sc'].current_transfer_session = TransferSession.objects.create(
id=uuid.uuid4().hex,
sync_session=session,
push=True,
last_activity_timestamp=timezone.now())
self.data.update(
create_buffer_and_store_dummy_data(
self.data['sc'].current_transfer_session.id))
class NetworkSyncConnectionTestCase(TestCase):
def setUp(self):
self.profile = "facilitydata"
self.root_scope_def = ScopeDefinition.objects.create(
id="rootcert",
profile=self.profile,
version=1,
primary_scope_param_key="mainpartition",
description="Root cert for ${mainpartition}.",
read_filter_template="",
write_filter_template="",
read_write_filter_template="${mainpartition}",
)
self.subset_scope_def = ScopeDefinition.objects.create(
id="subcert",
profile=self.profile,
version=1,
primary_scope_param_key="",
description=
"Subset cert under ${mainpartition} for ${subpartition}.",
read_filter_template="${mainpartition}",
write_filter_template="${mainpartition}:${subpartition}",
read_write_filter_template="",
)
self.root_cert = Certificate.generate_root_certificate(
self.root_scope_def.id)
self.subset_cert = Certificate(
parent=self.root_cert,
profile=self.profile,
scope_definition=self.subset_scope_def,
scope_version=self.subset_scope_def.version,
scope_params=json.dumps({
"mainpartition": self.root_cert.id,
"subpartition": "abracadabra"
}),
private_key=Key(),
)
self.root_cert.sign_certificate(self.subset_cert)
self.subset_cert.save()
self.controller = MorangoProfileController('facilitydata')
self.network_connection = self.controller.create_network_connection(
'127.0.0.1')
@mock_patch_decorator
def test_creating_sync_session_successful(self):
self.assertEqual(SyncSession.objects.filter(active=True).count(), 0)
NetworkSyncConnection._request.return_value.json.return_value = {
'signature': 'sig',
'local_fsic': '{}'
}
self.network_connection.create_sync_session(self.subset_cert,
self.root_cert)
self.assertEqual(SyncSession.objects.filter(active=True).count(), 1)
@mock_patch_decorator
def test_creating_sync_session_cert_fails_to_verify(self):
Certificate.verify.return_value = False
with self.assertRaises(CertificateSignatureInvalid):
self.network_connection.create_sync_session(
self.subset_cert, self.root_cert)
@mock_patch_decorator
def test_get_remote_certs(self):
# mock certs being returned by server
certs = self.subset_cert.get_ancestors(include_self=True)
cert_serialized = json.dumps(
CertificateSerializer(certs, many=True).data)
NetworkSyncConnection._request.return_value.json.return_value = json.loads(
cert_serialized)
# we want to see if the models are created (not saved) successfully
remote_certs = self.network_connection.get_remote_certificates('abc')
self.assertSetEqual(set(certs), set(remote_certs))
@mock_patch_decorator
def test_csr(self):
# mock a "signed" cert being returned by server
cert_serialized = json.dumps(
CertificateSerializer(self.subset_cert).data)
NetworkSyncConnection._request.return_value.json.return_value = json.loads(
cert_serialized)
self.subset_cert.delete()
# we only want to make sure the "signed" cert is saved
with mock.patch.object(Key,
"get_private_key_string",
return_value=self.subset_cert.private_key.
get_private_key_string()):
self.network_connection.certificate_signing_request(
self.root_cert, '', '')
self.assertTrue(
Certificate.objects.filter(
id=json.loads(cert_serialized)['id']).exists())
@mock_patch_decorator
def test_get_cert_chain(self):
# mock a cert chain being returned by server
certs = self.subset_cert.get_ancestors(include_self=True)
original_cert_count = certs.count()
cert_serialized = json.dumps(
CertificateSerializer(certs, many=True).data)
NetworkSyncConnection._request.return_value.json.return_value = json.loads(
cert_serialized)
Certificate.objects.all().delete()
# we only want to make sure the cert chain is saved
self.network_connection._get_certificate_chain(certs[1])
self.assertEqual(Certificate.objects.count(), original_cert_count)
def setUp(self):
(self.current_id, _) = InstanceIDModel.get_or_create_current_instance()
self.mc = MorangoProfileController('facilitydata')
def handle_async(self, *args, **options):
# validate url that is passed in
try:
URLValidator()((options["base_url"]))
except ValidationError:
raise CommandError(
"Base URL is not valid. Please retry command and enter a valid URL."
)
# call this in case user directly syncs without migrating database
if not ScopeDefinition.objects.filter():
call_command("loaddata", "scopedefinitions")
controller = MorangoProfileController("facilitydata")
with self.start_progress(total=7) as progress_update:
try:
network_connection = controller.create_network_connection(
options["base_url"])
except ConnectionError:
raise CommandError(
"Can not connect to server with base URL: {}".format(
options["base_url"]))
# if instance_ids are equal, this means device is trying to sync with itself, which we don't allow
if (InstanceIDModel.get_or_create_current_instance()[0].id ==
network_connection.server_info["instance_id"]):
raise CommandError(
"Device can not sync with itself. Please recheck base URL and try again."
)
progress_update(1)
options["dataset_id"] = self.get_dataset_id(
options["base_url"], options["dataset_id"])
progress_update(1)
client_cert, server_cert, options[
"username"] = self.get_client_and_server_certs(
options["username"],
options["password"],
options["dataset_id"],
network_connection,
)
progress_update(1)
sync_client = network_connection.create_sync_session(
client_cert, server_cert, chunk_size=options["chunk_size"])
progress_update(1)
# pull from server and push our own data to server
if not options["no_pull"]:
sync_client.initiate_pull(Filter(options["dataset_id"]))
if not options["no_push"]:
sync_client.initiate_push(Filter(options["dataset_id"]))
progress_update(1)
self.create_superuser_and_provision_device(options["username"],
options["dataset_id"])
progress_update(1)
sync_client.close_sync_session()
progress_update(1)
class SerializeIntoStoreTestCase(TestCase):
def setUp(self):
InstanceIDModel.get_or_create_current_instance()
self.range = 10
self.mc = MorangoProfileController('facilitydata')
self.original_name = "ralphie"
self.new_name = "rafael"
def test_all_models_get_serialized(self):
[FacilityModelFactory() for _ in range(self.range)]
self.mc.serialize_into_store()
self.assertEqual(len(Store.objects.all()), self.range)
def test_no_models_get_serialized(self):
# set dirty bit off on new models created
[
FacilityModelFactory.build().save(update_dirty_bit_to=False)
for _ in range(self.range)
]
# only models with dirty bit on should be serialized
self.mc.serialize_into_store()
self.assertFalse(Store.objects.exists())
def test_dirty_bit_gets_set(self):
[FacilityModelFactory() for _ in range(self.range)]
# dirty bit should be on
for facility in Facility.objects.all():
self.assertTrue(facility._morango_dirty_bit)
self.mc.serialize_into_store()
# dirty bit should have been toggled off
for facility in Facility.objects.all():
self.assertFalse(facility._morango_dirty_bit)
def test_store_models_get_updated(self):
FacilityModelFactory(name=self.original_name)
self.mc.serialize_into_store()
store_facility = Store.objects.first()
deserialized_model = json.loads(store_facility.serialized)
self.assertEqual(deserialized_model['name'], self.original_name)
Facility.objects.update(name=self.new_name)
self.mc.serialize_into_store()
store_facility = Store.objects.first()
deserialized_model = json.loads(store_facility.serialized)
self.assertEqual(deserialized_model['name'], self.new_name)
def test_last_saved_counter_updates(self):
FacilityModelFactory(name=self.original_name)
self.mc.serialize_into_store()
old_counter = Store.objects.first().last_saved_counter
Facility.objects.all().update(name=self.new_name)
self.mc.serialize_into_store()
new_counter = Store.objects.first().last_saved_counter
self.assertEqual(old_counter + 1, new_counter)
def test_last_saved_instance_updates(self):
FacilityModelFactory(name=self.original_name)
self.mc.serialize_into_store()
old_instance_id = Store.objects.first().last_saved_instance
with mock.patch('platform.platform', return_value='Windows'):
(new_id, _) = InstanceIDModel.get_or_create_current_instance()
Facility.objects.all().update(name=self.new_name)
self.mc.serialize_into_store()
new_instance_id = Store.objects.first().last_saved_instance
self.assertNotEqual(old_instance_id, new_instance_id)
self.assertEqual(new_instance_id, new_id.id)
def test_extra_fields_dont_get_overwritten(self):
serialized = """{"username": "******", "height": 6.0, "weight": 100}"""
MyUser.objects.create(username='******')
self.mc.serialize_into_store()
Store.objects.update(serialized=serialized)
MyUser.objects.update(username='******')
self.mc.serialize_into_store()
serialized = json.loads(Store.objects.first().serialized)
self.assertIn('height', serialized)
def test_updates_store_deleted_flag(self):
fac = FacilityModelFactory()
fac_id = fac.id
self.mc.serialize_into_store()
self.assertFalse(Store.objects.get(pk=fac_id).deleted)
fac.delete()
self.assertTrue(DeletedModels.objects.exists())
self.mc.serialize_into_store()
self.assertFalse(DeletedModels.objects.exists())
self.assertTrue(Store.objects.get(pk=fac_id).deleted)
def test_cascading_delete_updates_store_deleted_flag(self):
fac = FacilityModelFactory()
child = FacilityModelFactory(parent_id=fac.id)
child_id = child.id
self.mc.serialize_into_store()
self.assertFalse(Store.objects.get(pk=child_id).deleted)
fac.delete()
self.mc.serialize_into_store()
self.assertTrue(Store.objects.get(pk=child_id).deleted)
def test_conflicting_data_appended(self):
self.maxDiff = None
serialized = json.dumps({"username": "******"})
conflicting = []
user = MyUser.objects.create(username="******")
self.mc.serialize_into_store()
# add serialized fields to conflicting data
conflicting.insert(0, serialized)
conflicting.insert(0, json.dumps(user.serialize()))
# set store record and app record dirty bits to true to force serialization merge conflict
Store.objects.update(conflicting_serialized_data=serialized,
dirty_bit=True)
user.username = "******"
user.save(update_dirty_bit_to=True)
self.mc.serialize_into_store()
# assert we have placed serialized object into store's serialized field
st = Store.objects.get(id=user.id)
self.assertEqual(json.loads(st.serialized), user.serialize())
# assert store serialized field is moved to conflicting data
conflicting_serialized_data = st.conflicting_serialized_data.split(
'\n')
for x in range(len(conflicting)):
self.assertEqual(conflicting[x], conflicting_serialized_data[x])
def test_filtered_serialization_single_filter(self):
fac = FacilityModelFactory()
user = MyUser.objects.create(username='******')
log = SummaryLog.objects.create(user=user)
self.mc.serialize_into_store(filter=Filter(user._morango_partition))
self.assertFalse(Store.objects.filter(id=fac.id).exists())
self.assertTrue(Store.objects.filter(id=user.id).exists())
self.assertTrue(Store.objects.filter(id=log.id).exists())
def test_filtered_serialization_multiple_filter(self):
fac = FacilityModelFactory()
user = MyUser.objects.create(username='******')
user2 = MyUser.objects.create(username='******')
log = SummaryLog.objects.create(user=user)
self.mc.serialize_into_store(filter=Filter(user._morango_partition +
"\n" +
user2._morango_partition))
self.assertFalse(Store.objects.filter(id=fac.id).exists())
self.assertTrue(Store.objects.filter(id=user2.id).exists())
self.assertTrue(Store.objects.filter(id=user.id).exists())
self.assertTrue(Store.objects.filter(id=log.id).exists())
def test_self_ref_fk_class_adds_value_to_store(self):
root = FacilityModelFactory()
child = FacilityModelFactory(parent=root)
self.mc.serialize_into_store()
self.assertEqual(Store.objects.get(id=child.id)._self_ref_fk, root.id)
def test_regular_class_leaves_value_blank_in_store(self):
log = SummaryLog.objects.create(user=MyUser.objects.create(
username='******'))
self.mc.serialize_into_store()
self.assertEqual(Store.objects.get(id=log.id)._self_ref_fk, '')
def test_previously_deleted_store_flag_resets(self):
# create and delete object
user = MyUser.objects.create(username='******')
self.mc.serialize_into_store()
MyUser.objects.all().delete()
self.mc.serialize_into_store()
self.assertTrue(Store.objects.get(id=user.id).deleted)
# recreate object with same id
user = MyUser.objects.create(username='******')
# ensure deleted flag is updated after recreation
self.mc.serialize_into_store()
self.assertFalse(Store.objects.get(id=user.id).deleted)
def setUp(self):
DatabaseIDModel.objects.create()
InstanceIDModel.get_or_create_current_instance()
[FacilityModelFactory() for _ in range(10)]
self.mc = MorangoProfileController('facilitydata')
self.mc.serialize_into_store()
class DeserializationFromStoreIntoAppTestCase(TestCase):
def setUp(self):
(self.current_id, _) = InstanceIDModel.get_or_create_current_instance()
self.range = 10
self.mc = MorangoProfileController('facilitydata')
for i in range(self.range):
self.ident = uuid.uuid4().hex
StoreModelFacilityFactory(pk=self.ident,
serialized=serialized_facility_factory(
self.ident))
def test_dirty_store_records_are_deserialized(self):
self.assertFalse(Facility.objects.all().exists())
self.mc.deserialize_from_store()
self.assertEqual(len(Facility.objects.all()), self.range)
def test_clean_store_records_do_not_get_deserialized(self):
self.assertFalse(Facility.objects.exists())
Store.objects.update(dirty_bit=False)
self.mc.deserialize_from_store()
self.assertFalse(Facility.objects.exists())
def test_deleted_models_do_not_get_deserialized(self):
Store.objects.update_or_create(defaults={'deleted': True},
id=self.ident)
self.mc.deserialize_from_store()
self.assertFalse(Facility.objects.filter(id=self.ident).exists())
def test_deleted_models_deletes_them_in_app(self):
# put models in app layer
self.mc.deserialize_from_store()
# deleted flag on store should delete model in app layer
Store.objects.update_or_create(defaults={
'deleted': True,
'dirty_bit': True
},
id=self.ident)
self.mc.deserialize_from_store()
self.assertFalse(Facility.objects.filter(id=self.ident).exists())
def test_update_app_with_newer_data_from_store(self):
name = 'test'
fac = FacilityModelFactory(id=self.ident, name=name)
fac.save(update_dirty_bit_to=False)
self.assertEqual(fac.name, name)
self.mc.deserialize_from_store()
fac = Facility.objects.get(id=self.ident)
self.assertNotEqual(fac.name, name)
def test_handle_extra_field_deserialization(self):
# modify a store record by adding extra serialized field
store_model = Store.objects.get(id=self.ident)
serialized = json.loads(store_model.serialized)
serialized.update({'wacky': True})
store_model.serialized = json.dumps(serialized)
store_model.save()
# deserialize records
self.mc.deserialize_from_store()
# by this point no errors should have occurred but we check list of fields anyways
fac = Facility.objects.get(id=self.ident)
self.assertNotIn('wacky', fac.__dict__)
def test_store_dirty_bit_resets(self):
self.assertTrue(Store.objects.filter(dirty_bit=True))
self.mc.deserialize_from_store()
self.assertFalse(Store.objects.filter(dirty_bit=True))
def test_record_with_dirty_bit_off_doesnt_deserialize(self):
st = Store.objects.first()
st.dirty_bit = False
st.save()
self.mc.deserialize_from_store()
self.assertFalse(Facility.objects.filter(id=st.id).exists())
def test_broken_fk_adds_to_deleted_models(self):
serialized = """{"user_id": "40de9a3fded95d7198f200c78e559353"}"""
StoreModelFacilityFactory(id=uuid.uuid4().hex,
serialized=serialized,
model_name="contentsummarylog")
with mock.patch('morango.models.SyncableModel._update_deleted_models'
) as mock_update:
self.mc.deserialize_from_store()
self.assertTrue(mock_update.called)
class DeserializationFromStoreIntoAppTestCase(TestCase):
def setUp(self):
DatabaseIDModel.objects.create()
(self.current_id, _) = InstanceIDModel.get_or_create_current_instance()
self.range = 10
self.mc = MorangoProfileController('facilitydata')
for i in range(self.range):
self.ident = uuid.uuid4().hex
StoreModelFacilityFactory(pk=self.ident,
serialized=serialized_facility_factory(
self.ident))
def test_dirty_store_records_are_deserialized(self):
self.assertFalse(Facility.objects.all().exists())
self.mc.deserialize_from_store()
self.assertEqual(len(Facility.objects.all()), self.range)
def test_clean_store_records_do_not_get_deserialized(self):
self.assertFalse(Facility.objects.exists())
Store.objects.update(dirty_bit=False)
self.mc.deserialize_from_store()
self.assertFalse(Facility.objects.exists())
def test_deleted_models_do_not_get_deserialized(self):
Store.objects.update_or_create(defaults={'deleted': True},
id=self.ident)
self.mc.deserialize_from_store()
self.assertFalse(Facility.objects.filter(id=self.ident).exists())
def test_deleted_models_deletes_them_in_app(self):
# put models in app layer
self.mc.deserialize_from_store()
# deleted flag on store should delete model in app layer
Store.objects.update_or_create(defaults={
'deleted': True,
'dirty_bit': True
},
id=self.ident)
self.mc.deserialize_from_store()
self.assertFalse(Facility.objects.filter(id=self.ident).exists())
def test_update_app_with_newer_data_from_store(self):
name = 'test'
fac = FacilityModelFactory(id=self.ident, name=name)
fac.save(update_dirty_bit_to=False)
self.assertEqual(fac.name, name)
self.mc.deserialize_from_store()
fac = Facility.objects.get(id=self.ident)
self.assertNotEqual(fac.name, name)
def test_handle_extra_field_deserialization(self):
# modify a store record by adding extra serialized field
store_model = Store.objects.get(id=self.ident)
serialized = json.loads(store_model.serialized)
serialized.update({'wacky': True})
store_model.serialized = json.dumps(serialized)
store_model.save()
# deserialize records
self.mc.deserialize_from_store()
# by this point no errors should have occurred but we check list of fields anyways
fac = Facility.objects.get(id=self.ident)
self.assertNotIn('wacky', fac.__dict__)
def test_broken_foreign_key_deserialization(self):
# add fake foreign key
store_model = Store.objects.get(id=self.ident)
serialized = json.loads(store_model.serialized)
serialized.update({'parent_id': '4d53c8e72b8bea87a393910ff0dcb212'})
store_model.serialized = json.dumps(serialized)
store_model.save()
# deserialize records
self.mc.deserialize_from_store()
self.assertTrue(
DeletedModels.objects.filter(id=store_model.id).exists())
def test_store_dirty_bit_resets(self):
self.assertTrue(Store.objects.filter(dirty_bit=True))
self.mc.deserialize_from_store()
self.assertFalse(Store.objects.filter(dirty_bit=True))
def test_record_with_dirty_bit_off_doesnt_deserialize(self):
st = Store.objects.first()
st.dirty_bit = False
st.save()
self.mc.deserialize_from_store()
self.assertFalse(Facility.objects.filter(id=st.id).exists())
class NetworkSyncConnectionTestCase(LiveServerTestCase):
def setUp(self):
self.profile = "facilitydata"
self.root_scope_def = ScopeDefinition.objects.create(
id="rootcert",
profile=self.profile,
version=1,
primary_scope_param_key="mainpartition",
description="Root cert for ${mainpartition}.",
read_filter_template="",
write_filter_template="",
read_write_filter_template="${mainpartition}",
)
self.subset_scope_def = ScopeDefinition.objects.create(
id="subcert",
profile=self.profile,
version=1,
primary_scope_param_key="",
description=
"Subset cert under ${mainpartition} for ${subpartition}.",
read_filter_template="${mainpartition}",
write_filter_template="${mainpartition}:${subpartition}",
read_write_filter_template="",
)
self.root_cert = Certificate.generate_root_certificate(
self.root_scope_def.id)
self.subset_cert = Certificate(
parent=self.root_cert,
profile=self.profile,
scope_definition=self.subset_scope_def,
scope_version=self.subset_scope_def.version,
scope_params=json.dumps({
"mainpartition": self.root_cert.id,
"subpartition": "abracadabra"
}),
private_key=Key(),
)
self.root_cert.sign_certificate(self.subset_cert)
self.subset_cert.save()
self.unsaved_cert = Certificate(
parent=self.root_cert,
profile=self.profile,
scope_definition=self.subset_scope_def,
scope_version=self.subset_scope_def.version,
scope_params=json.dumps({
"mainpartition": self.root_cert.id,
"subpartition": "other"
}),
public_key=Key(),
)
self.root_cert.sign_certificate(self.unsaved_cert)
self.controller = MorangoProfileController('facilitydata')
self.network_connection = self.controller.create_network_connection(
self.live_server_url)
self.key = SharedKey.get_or_create_shared_key()
@mock.patch.object(SyncSession.objects, 'create', return_value=None)
def test_creating_sync_session_successful(self, mock_object):
self.assertEqual(SyncSession.objects.filter(active=True).count(), 0)
self.network_connection.create_sync_session(self.subset_cert,
self.root_cert)
self.assertEqual(SyncSession.objects.filter(active=True).count(), 1)
@mock.patch.object(NetworkSyncConnection, '_create_sync_session')
@mock.patch.object(Certificate, 'verify', return_value=False)
def test_creating_sync_session_cert_fails_to_verify(
self, mock_verify, mock_create):
mock_create.return_value.json.return_value = {}
with self.assertRaises(CertificateSignatureInvalid):
self.network_connection.create_sync_session(
self.subset_cert, self.root_cert)
def test_get_remote_certs(self):
certs = self.subset_cert.get_ancestors(include_self=True)
remote_certs = self.network_connection.get_remote_certificates(
self.root_cert.id)
self.assertSetEqual(set(certs), set(remote_certs))
@mock.patch.object(NetworkSyncConnection, '_request')
def test_csr(self, mock_request):
# mock a "signed" cert being returned by server
cert_serialized = json.dumps(
CertificateSerializer(self.subset_cert).data)
mock_request.return_value.json.return_value = json.loads(
cert_serialized)
self.subset_cert.delete()
# we only want to make sure the "signed" cert is saved
with mock.patch.object(Key,
"get_private_key_string",
return_value=self.subset_cert.private_key.
get_private_key_string()):
self.network_connection.certificate_signing_request(
self.root_cert, '', '')
self.assertTrue(
Certificate.objects.filter(
id=json.loads(cert_serialized)['id']).exists())
@override_settings(ALLOW_CERTIFICATE_PUSHING=True)
def test_push_signed_client_certificate_chain(self):
cert = self.network_connection.push_signed_client_certificate_chain(
self.root_cert, self.subset_scope_def.id, {
"mainpartition": self.root_cert.id,
"subpartition": "abracadabra"
})
self.assertEqual(cert.private_key, None)
self.assertTrue(Certificate.objects.filter(id=cert.id).exists())
@override_settings(ALLOW_CERTIFICATE_PUSHING=True)
def test_push_signed_client_certificate_chain_publickey_error(self):
with mock.patch.object(NetworkSyncConnection, '_get_public_key'):
NetworkSyncConnection._get_public_key.return_value.json.return_value = [
{
'public_key': Key().get_public_key_string()
}
]
with self.assertRaises(HTTPError) as e:
self.network_connection.push_signed_client_certificate_chain(
self.root_cert, self.subset_scope_def.id, {
"mainpartition": self.root_cert.id,
"subpartition": "abracadabra"
})
self.assertEqual(e.exception.response.status_code, 400)
@override_settings(ALLOW_CERTIFICATE_PUSHING=True)
def test_push_signed_client_certificate_chain_bad_cert(self):
with self.assertRaises(HTTPError) as e:
self.network_connection.push_signed_client_certificate_chain(
self.root_cert, self.subset_scope_def.id,
{"bad": "scope_params"})
self.assertEqual(e.exception.response.status_code, 400)
@override_settings(ALLOW_CERTIFICATE_PUSHING=True)
@mock.patch.object(NetworkSyncConnection, '_get_nonce')
def test_push_signed_client_certificate_chain_nonce_error(
self, mock_nonce):
mock_nonce.return_value.json.return_value = {'id': uuid.uuid4().hex}
with self.assertRaises(HTTPError) as e:
self.network_connection.push_signed_client_certificate_chain(
self.root_cert, self.subset_scope_def.id, {
"mainpartition": self.root_cert.id,
"subpartition": "abracadabra"
})
self.assertEqual(e.exception.response.status_code, 403)
def test_push_signed_client_certificate_chain_not_allowed(self):
with self.assertRaises(MorangoServerDoesNotAllowNewCertPush) as e:
self.network_connection.push_signed_client_certificate_chain(
self.root_cert, self.subset_scope_def.id, {
"mainpartition": self.root_cert.id,
"subpartition": "abracadabra"
})
self.assertEqual(e.exception.response.status_code, 403)
def test_get_cert_chain(self):
response = self.network_connection._get_certificate_chain(
self.subset_cert)
data = response.json()
self.assertEqual(len(data), Certificate.objects.count())
self.assertEqual(data[0]['id'], self.root_cert.id)
self.assertEqual(data[1]['id'], self.subset_cert.id)
class DeserializationFromStoreIntoAppTestCase(TestCase):
def setUp(self):
(self.current_id, _) = InstanceIDModel.get_or_create_current_instance()
self.range = 10
self.mc = MorangoProfileController('facilitydata')
for i in range(self.range):
self.ident = uuid.uuid4().hex
StoreModelFacilityFactory(pk=self.ident,
serialized=serialized_facility_factory(
self.ident))
def test_dirty_store_records_are_deserialized(self):
self.assertFalse(Facility.objects.all().exists())
self.mc.deserialize_from_store()
self.assertEqual(len(Facility.objects.all()), self.range)
def test_clean_store_records_do_not_get_deserialized(self):
self.assertFalse(Facility.objects.exists())
Store.objects.update(dirty_bit=False)
self.mc.deserialize_from_store()
self.assertFalse(Facility.objects.exists())
def test_deleted_models_do_not_get_deserialized(self):
Store.objects.update_or_create(defaults={'deleted': True},
id=self.ident)
self.mc.deserialize_from_store()
self.assertFalse(Facility.objects.filter(id=self.ident).exists())
def test_deleted_models_deletes_them_in_app(self):
# put models in app layer
self.mc.deserialize_from_store()
# deleted flag on store should delete model in app layer
Store.objects.update_or_create(defaults={
'deleted': True,
'dirty_bit': True
},
id=self.ident)
self.mc.deserialize_from_store()
self.assertFalse(Facility.objects.filter(id=self.ident).exists())
def test_update_app_with_newer_data_from_store(self):
name = 'test'
fac = FacilityModelFactory(id=self.ident, name=name)
fac.save(update_dirty_bit_to=False)
self.assertEqual(fac.name, name)
self.mc.deserialize_from_store()
fac = Facility.objects.get(id=self.ident)
self.assertNotEqual(fac.name, name)
def test_handle_extra_field_deserialization(self):
# modify a store record by adding extra serialized field
store_model = Store.objects.get(id=self.ident)
serialized = json.loads(store_model.serialized)
serialized.update({'wacky': True})
store_model.serialized = json.dumps(serialized)
store_model.save()
# deserialize records
self.mc.deserialize_from_store()
# by this point no errors should have occurred but we check list of fields anyways
fac = Facility.objects.get(id=self.ident)
self.assertNotIn('wacky', fac.__dict__)
def test_store_dirty_bit_resets(self):
self.assertTrue(Store.objects.filter(dirty_bit=True))
self.mc.deserialize_from_store()
self.assertFalse(Store.objects.filter(dirty_bit=True))
def test_record_with_dirty_bit_off_doesnt_deserialize(self):
st = Store.objects.first()
st.dirty_bit = False
st.save()
self.mc.deserialize_from_store()
self.assertFalse(Facility.objects.filter(id=st.id).exists())
def test_broken_fk_leaves_store_dirty_bit(self):
serialized = """{"user_id": "40de9a3fded95d7198f200c78e559353", "id": "bd205b5ee5bc42da85925d24c61341a8"}"""
st = StoreModelFacilityFactory(id=uuid.uuid4().hex,
serialized=serialized,
model_name="contentsummarylog")
self.mc.deserialize_from_store()
st.refresh_from_db()
self.assertTrue(st.dirty_bit)
def test_invalid_model_leaves_store_dirty_bit(self):
user = MyUser(username='******' * 21)
st = StoreModelFacilityFactory(model_name="user",
id=uuid.uuid4().hex,
serialized=json.dumps(user.serialize()))
self.mc.deserialize_from_store()
st.refresh_from_db()
self.assertTrue(st.dirty_bit)
def test_deleted_model_propagates_to_store_record(self):
"""
It could be the case that we have two store records, one that is deleted and the other that has a fk pointing to the deleted record.
When we deserialize, we want to ensure that the record with the fk pointer also gets the deleted flag set, while also not
deserializing the data into a model.
"""
# user will be deleted
user = MyUser(username='******')
user.save(update_dirty_bit_to=False)
# log may be synced in from other device
log = SummaryLog(user_id=user.id)
log.id = log.calculate_uuid()
StoreModelFacilityFactory(model_name="user",
id=user.id,
serialized=json.dumps(user.serialize()),
deleted=True)
StoreModelFacilityFactory(model_name="contentsummarylog",
id=log.id,
serialized=json.dumps(log.serialize()))
# make sure delete propagates to store due to deleted foreign key
self.mc.deserialize_from_store()
# have to serialize to update deleted models
self.mc.serialize_into_store()
self.assertFalse(SummaryLog.objects.filter(id=log.id).exists())
self.assertTrue(Store.objects.get(id=log.id).deleted)
def test_hard_deleted_model_propagates_to_store_record(self):
"""
It could be the case that we have two store records, one that is hard deleted and the other that has a fk pointing to the hard deleted record.
When we deserialize, we want to ensure that the record with the fk pointer also gets the hard deleted flag set, while also not
deserializing the data into a model.
"""
# user will be deleted
user = MyUser(username='******')
user.save(update_dirty_bit_to=False)
# log may be synced in from other device
log = SummaryLog(user_id=user.id)
log.id = log.calculate_uuid()
StoreModelFacilityFactory(model_name="user",
id=user.id,
serialized=json.dumps(user.serialize()),
deleted=True,
hard_deleted=True)
StoreModelFacilityFactory(model_name="contentsummarylog",
id=log.id,
serialized=json.dumps(log.serialize()))
# make sure delete propagates to store due to deleted foreign key
self.mc.deserialize_from_store()
# have to serialize to update deleted models
self.mc.serialize_into_store()
self.assertFalse(SummaryLog.objects.filter(id=log.id).exists())
self.assertTrue(Store.objects.get(id=log.id).hard_deleted)
def test_regular_model_deserialization(self):
# deserialization should be able to handle multiple records
user = MyUser(username='******', password='******')
user2 = MyUser(username='******', password='******')
user.save(update_dirty_bit_to=False)
user2.save(update_dirty_bit_to=False)
user.username = '******'
user2.username = '******'
StoreModelFacilityFactory(id=user.id,
serialized=json.dumps(user.serialize()),
model_name="user")
StoreModelFacilityFactory(id=user2.id,
serialized=json.dumps(user2.serialize()),
model_name="user")
self.mc.deserialize_from_store()
self.assertFalse(MyUser.objects.filter(username='******').exists())
self.assertFalse(MyUser.objects.filter(username='******').exists())
self.assertTrue(MyUser.objects.filter(username='******').exists())
self.assertTrue(MyUser.objects.filter(username='******').exists())
def setUp(self):
self.controller = MorangoProfileController('facilitydata')
InstanceIDModel.get_or_create_current_instance()
def handle_async(self, *args, **options):
# validate url that is passed in
try:
URLValidator()((options['base_url']))
except ValidationError:
print(
'Base-url is not valid. Please retry command and enter a valid url.'
)
sys.exit(1)
# call this in case user directly syncs without migrating database
if not ScopeDefinition.objects.filter():
call_command("loaddata", "scopedefinitions")
# ping server at url with info request
info_url = urljoin(options['base_url'],
'api/morango/v1/morangoinfo/1/')
try:
info_resp = requests.get(info_url)
except ConnectionError:
print('Can not connect to server with base-url: {}'.format(
options['base_url']))
sys.exit(1)
# if instance_ids are equal, this means device is trying to sync with itself, which we don't allow
if InstanceIDModel.get_or_create_current_instance(
)[0].id == info_resp.json()['instance_id']:
print(
'Device can not sync with itself. Please re-check base-url and try again.'
)
sys.exit(1)
controller = MorangoProfileController('facilitydata')
with self.start_progress(total=7) as progress_update:
network_connection = controller.create_network_connection(
options['base_url'])
progress_update(1)
options['dataset_id'] = self.get_dataset_id(
options['base_url'], options['dataset_id'])
progress_update(1)
client_cert, server_cert, options[
'username'] = self.get_client_and_server_certs(
options['username'], options['password'],
options['dataset_id'], network_connection)
progress_update(1)
sync_client = network_connection.create_sync_session(
client_cert, server_cert)
progress_update(1)
# pull from server and push our own data to server
if not options['no_pull']:
sync_client.initiate_pull(Filter(options['dataset_id']))
if not options['no_push']:
sync_client.initiate_push(Filter(options['dataset_id']))
progress_update(1)
self.create_superuser_and_provision_device(options['username'],
options['dataset_id'])
progress_update(1)
sync_client.close_sync_session()
progress_update(1)
def handle_async(self, *args, **options):
baseurl = options["baseurl"]
facility_id = options["facility"]
chunk_size = options["chunk_size"]
# This handles the case for when we want to pull in facility data for our empty kolibri instance
if not facility_id:
self._fullfacilitysync(baseurl)
facility = get_facility(
facility_id=facility_id, noninteractive=options["noninteractive"]
)
# if url is not pointing to portal server, do P2P syncing
self._fullfacilitysync(baseurl, facility=facility, chunk_size=chunk_size)
# data portal syncing
self.stdout.write("Syncing has been initiated (this may take a while)...")
controller = MorangoProfileController("facilitydata")
with self.start_progress(total=5) as progress_update:
try:
network_connection = controller.create_network_connection(baseurl)
except ConnectionError as e:
raise CommandError(e)
progress_update(1)
# get client certificate
client_cert = (
facility.dataset.get_owned_certificates()
.filter(scope_definition_id=FULL_FACILITY)
.first()
)
if not client_cert:
raise CommandError(
"This device does not own a certificate for Facility: {}".format(
facility.name
)
)
# push certificate up to portal server
scope_params = json.loads(client_cert.scope_params)
server_cert = network_connection.push_signed_client_certificate_chain(
local_parent_cert=client_cert,
scope_definition_id=FULL_FACILITY,
scope_params=scope_params,
)
progress_update(1)
# we should now be able to push our facility data
sync_client = network_connection.create_sync_session(
client_cert, server_cert, chunk_size=chunk_size
)
progress_update(1)
sync_client.initiate_push(Filter(scope_params["dataset_id"]))
progress_update(1)
sync_client.close_sync_session()
progress_update(1)
self.stdout.write("Syncing has been completed.")