def setUp(self): DatabaseIDModel.objects.create() (self.current_id, _) = InstanceIDModel.get_or_create_current_instance() self.mc = MorangoProfileController('facilitydata') self.fac1 = FacilityModelFactory(name='school') self.mc.serialize_into_store() self.old_rmc = RecordMaxCounter.objects.first()
def setUp(self): self.profile = "facilitydata" self.root_scope_def = ScopeDefinition.objects.create( id="rootcert", profile=self.profile, version=1, primary_scope_param_key="mainpartition", description="Root cert for ${mainpartition}.", read_filter_template="", write_filter_template="", read_write_filter_template="${mainpartition}", ) self.subset_scope_def = ScopeDefinition.objects.create( id="subcert", profile=self.profile, version=1, primary_scope_param_key="", description= "Subset cert under ${mainpartition} for ${subpartition}.", read_filter_template="${mainpartition}", write_filter_template="${mainpartition}:${subpartition}", read_write_filter_template="", ) self.root_cert = Certificate.generate_root_certificate( self.root_scope_def.id) self.subset_cert = Certificate( parent=self.root_cert, profile=self.profile, scope_definition=self.subset_scope_def, scope_version=self.subset_scope_def.version, scope_params=json.dumps({ "mainpartition": self.root_cert.id, "subpartition": "abracadabra" }), private_key=Key(), ) self.root_cert.sign_certificate(self.subset_cert) self.subset_cert.save() self.unsaved_cert = Certificate( parent=self.root_cert, profile=self.profile, scope_definition=self.subset_scope_def, scope_version=self.subset_scope_def.version, scope_params=json.dumps({ "mainpartition": self.root_cert.id, "subpartition": "other" }), public_key=Key(), ) self.root_cert.sign_certificate(self.unsaved_cert) self.controller = MorangoProfileController('facilitydata') self.network_connection = self.controller.create_network_connection( self.live_server_url) self.key = SharedKey.get_or_create_shared_key()
def handle_async(self, *args, **options): # call this in case user directly syncs without migrating database if not ScopeDefinition.objects.filter(): call_command("loaddata", "scopedefinitions") controller = MorangoProfileController('facilitydata') with self.start_progress(total=7) as progress_update: network_connection = controller.create_network_connection(options['base_url']) progress_update(1) options['dataset_id'] = self.get_dataset_id(options['base_url'], options['dataset_id']) progress_update(1) client_cert, server_cert = self.get_client_and_server_certs(options['username'], options['password'], options['dataset_id'], network_connection) progress_update(1) sync_client = network_connection.create_sync_session(client_cert, server_cert) progress_update(1) # pull from server and push our own data to server if not options['no_pull']: sync_client.initiate_pull(Filter(options['dataset_id'])) if not options['no_push']: sync_client.initiate_push(Filter(options['dataset_id'])) progress_update(1) self.create_superuser_and_provision_device(options['username'], options['dataset_id']) progress_update(1) sync_client.close_sync_session() progress_update(1)
def setUp(self): DatabaseIDModel.objects.create() InstanceIDModel.get_or_create_current_instance() self.range = 10 self.mc = MorangoProfileController('facilitydata') self.original_name = "ralphie" self.new_name = "rafael"
def setUp(self): (self.current_id, _) = InstanceIDModel.get_or_create_current_instance() self.range = 10 self.mc = MorangoProfileController('facilitydata') for i in range(self.range): self.ident = uuid.uuid4().hex StoreModelFacilityFactory(pk=self.ident, serialized=serialized_facility_factory( self.ident))
def handle_async(self, *args, **options): # validate url that is passed in try: URLValidator()((options['base_url'])) except ValidationError: print('Base-url is not valid. Please retry command and enter a valid url.') sys.exit(1) # call this in case user directly syncs without migrating database if not ScopeDefinition.objects.filter(): call_command("loaddata", "scopedefinitions") # ping server at url with info request info_url = urljoin(options['base_url'], 'api/morango/v1/morangoinfo/1/') try: info_resp = requests.get(info_url) except ConnectionError: print('Can not connect to server with base-url: {}'.format(options['base_url'])) sys.exit(1) # if instance_ids are equal, this means device is trying to sync with itself, which we don't allow if InstanceIDModel.get_or_create_current_instance()[0].id == info_resp.json()['instance_id']: print('Device can not sync with itself. Please re-check base-url and try again.') sys.exit(1) controller = MorangoProfileController('facilitydata') with self.start_progress(total=7) as progress_update: network_connection = controller.create_network_connection(options['base_url']) progress_update(1) options['dataset_id'] = self.get_dataset_id(options['base_url'], options['dataset_id']) progress_update(1) client_cert, server_cert, options['username'] = self.get_client_and_server_certs(options['username'], options['password'], options['dataset_id'], network_connection) progress_update(1) sync_client = network_connection.create_sync_session(client_cert, server_cert) progress_update(1) # pull from server and push our own data to server if not options['no_pull']: sync_client.initiate_pull(Filter(options['dataset_id'])) if not options['no_push']: sync_client.initiate_push(Filter(options['dataset_id'])) progress_update(1) self.create_superuser_and_provision_device(options['username'], options['dataset_id']) progress_update(1) sync_client.close_sync_session() progress_update(1)
class DateTimeTZFieldTestCase(TestCase): def setUp(self): self.controller = MorangoProfileController("facilitydata") InstanceIDModel.get_or_create_current_instance() def test_deserializing_field(self): facility = Facility.objects.create(name="hallo") FacilityUser.objects.create(username="******", facility=facility) self.controller.serialize_into_store() Store.objects.update(dirty_bit=True) try: self.controller.deserialize_from_store() except AttributeError as e: self.fail(e.message)
class DateTimeTZFieldTestCase(TestCase): def setUp(self): self.controller = MorangoProfileController('facilitydata') InstanceIDModel.get_or_create_current_instance() @unittest.skipIf(os.environ.get('TOX_ENV') == 'postgres', "Skipping testing on postgres because sql is not compatible.") def test_deserializing_field(self): facility = Facility.objects.create(name="hallo") FacilityUser.objects.create(username='******', facility=facility) self.controller.serialize_into_store() Store.objects.update(dirty_bit=True) try: self.controller.deserialize_from_store() except AttributeError as e: self.fail(e.message)
class RecordMaxCounterUpdatesDuringSerialization(TestCase): def setUp(self): DatabaseIDModel.objects.create() (self.current_id, _) = InstanceIDModel.get_or_create_current_instance() self.mc = MorangoProfileController('facilitydata') self.fac1 = FacilityModelFactory(name='school') self.mc.serialize_into_store() self.old_rmc = RecordMaxCounter.objects.first() def test_new_rmc_for_existing_model(self): with mock.patch('platform.platform', return_value='Windows'): (new_id, _) = InstanceIDModel.get_or_create_current_instance() Facility.objects.update(name='facility') self.mc.serialize_into_store() new_rmc = RecordMaxCounter.objects.get(instance_id=new_id.id, store_model_id=self.fac1.id) new_store_record = Store.objects.get(id=self.fac1.id) self.assertEqual(new_rmc.counter, new_store_record.last_saved_counter) self.assertEqual(new_rmc.instance_id, new_store_record.last_saved_instance) def test_update_rmc_for_existing_model(self): Facility.objects.update(name='facility') self.mc.serialize_into_store() # there should only be 1 RecordMaxCounter for a specific instance_id and a specific model (unique_together) self.assertEqual( RecordMaxCounter.objects.filter( instance_id=self.current_id.id, store_model_id=self.fac1.id).count(), 1) new_rmc = RecordMaxCounter.objects.get(instance_id=self.current_id.id, store_model_id=self.fac1.id) new_store_record = Store.objects.get(id=self.fac1.id) self.assertEqual(self.old_rmc.counter + 1, new_rmc.counter) self.assertEqual(new_rmc.counter, new_store_record.last_saved_counter) self.assertEqual(new_rmc.instance_id, new_store_record.last_saved_instance) def test_new_rmc_for_non_existent_model(self): with mock.patch('platform.platform', return_value='Windows'): (new_id, _) = InstanceIDModel.get_or_create_current_instance() new_fac = FacilityModelFactory(name='college') self.mc.serialize_into_store() new_rmc = RecordMaxCounter.objects.get(instance_id=new_id.id, store_model_id=new_fac.id) new_store_record = Store.objects.get(id=new_fac.id) self.assertNotEqual(new_id.id, self.current_id.id) self.assertEqual(new_store_record.last_saved_instance, new_rmc.instance_id) self.assertEqual(new_store_record.last_saved_counter, new_rmc.counter)
class PostDeleteSignalsTestCase(TestCase): def setUp(self): InstanceIDModel.get_or_create_current_instance() [FacilityModelFactory() for _ in range(10)] self.mc = MorangoProfileController('facilitydata') self.mc.serialize_into_store() def test_deleted_flag_gets_set(self): facility = Facility.objects.first() deleted_id = facility.id facility.delete() self.assertTrue(DeletedModels.objects.filter(id=deleted_id)) def test_cascading_delete(self): facility = Facility.objects.first() child = FacilityModelFactory(parent=facility) deleted_child_id = child.id facility.delete() self.assertTrue(DeletedModels.objects.filter(id=deleted_child_id))
class SelfReferentialFKDeserializationTestCase(TestCase): def setUp(self): (self.current_id, _) = InstanceIDModel.get_or_create_current_instance() self.mc = MorangoProfileController('facilitydata') def test_self_ref_fk(self): self.assertEqual(_self_referential_fk(Facility), 'parent_id') self.assertEqual(_self_referential_fk(MyUser), None) def test_delete_model_in_store_deletes_models_in_app(self): root = FacilityModelFactory() child1 = FacilityModelFactory(parent=root) child2 = FacilityModelFactory(parent=root) self.mc.serialize_into_store() # simulate a node being deleted and synced Store.objects.filter(id=child2.id).update(deleted=True) Store.objects.update(dirty_bit=True) grandchild1 = FacilityModelFactory(parent=child2) grandchild2 = FacilityModelFactory(parent=child2) self.mc.deserialize_from_store() # ensure tree structure in app layer is correct child1 = Facility.objects.filter(id=child1.id) self.assertTrue(child1.exists()) self.assertEqual(child1[0].parent_id, root.id) self.assertFalse(Facility.objects.filter(id=child2.id).exists()) self.assertFalse(Facility.objects.filter(id=grandchild1.id).exists()) self.assertFalse(Facility.objects.filter(id=grandchild2.id).exists()) def test_models_created_successfully(self): root = FacilityModelFactory() child1 = FacilityModelFactory(parent=root) child2 = FacilityModelFactory(parent=root) self.mc.serialize_into_store() Facility.objects.all().delete() DeletedModels.objects.all().delete() Store.objects.update(dirty_bit=True, deleted=False) self.mc.deserialize_from_store() # ensure tree structure in app layer is correct self.assertTrue(Facility.objects.filter(id=root.id).exists()) child1 = Facility.objects.filter(id=child1.id) self.assertTrue(child1.exists()) self.assertEqual(child1[0].parent_id, root.id) child2 = Facility.objects.filter(id=child2.id) self.assertTrue(child2.exists()) self.assertEqual(child2[0].parent_id, root.id)
def create_dummy_store_data(): data = {} DatabaseIDModel.objects.create() (data['group1_id'], _) = InstanceIDModel.get_or_create_current_instance() # counter is at 0 # create controllers for app/store/buffer operations data['mc'] = MorangoProfileController('facilitydata') data['sc'] = SyncClient(None, 'host') session = SyncSession.objects.create(id=uuid.uuid4().hex, profile="facilitydata", last_activity_timestamp=timezone.now()) data['sc'].current_transfer_session = TransferSession.objects.create(id=uuid.uuid4().hex, sync_session=session, push=True, last_activity_timestamp=timezone.now()) data['mc'].serialize_into_store() # counter is at 1 # create group of facilities and first serialization data['group1_c1'] = [FacilityFactory() for _ in range(5)] data['mc'].serialize_into_store() # counter is at 2 # create group of facilities and second serialization data['group1_c2'] = [FacilityFactory() for _ in range(5)] # create users and logs associated with user data['user1'] = MyUser.objects.create(username='******') data['user1_sumlogs'] = [SummaryLog.objects.create(user=data['user1']) for _ in range(5)] data['mc'].serialize_into_store() # counter is at 3 # create new instance id and group of facilities with mock.patch('platform.platform', return_value='plataforma'): (data['group2_id'], _) = InstanceIDModel.get_or_create_current_instance() # new counter is at 0 data['mc'].serialize_into_store() # new counter is at 1 data['group2_c1'] = [FacilityFactory() for _ in range(5)] # create users and logs associated with user data['user2'] = MyUser.objects.create(username='******') data['user2_sumlogs'] = [SummaryLog.objects.create(user=data['user2']) for _ in range(5)] data['user2_interlogs'] = [InteractionLog.objects.create(user=data['user2']) for _ in range(5)] data['user3'] = MyUser.objects.create(username='******') data['user3_sumlogs'] = [SummaryLog.objects.create(user=data['user3']) for _ in range(5)] data['user3_interlogs'] = [InteractionLog.objects.create(user=data['user3']) for _ in range(5)] data['mc'].serialize_into_store() # new counter is at 2 data['user4'] = MyUser.objects.create(username='******', _morango_partition='badpartition') data['mc'].serialize_into_store() # new counter is at 3 return data
def setUp(self): settings.MORANGO_DESERIALIZE_AFTER_DEQUEUING = False self.data = {} DatabaseIDModel.objects.create() (self.current_id, _) = InstanceIDModel.get_or_create_current_instance() # create controllers for app/store/buffer operations self.data['mc'] = MorangoProfileController('facilitydata') self.data['sc'] = SyncClient(None, 'host') session = SyncSession.objects.create( id=uuid.uuid4().hex, profile="", last_activity_timestamp=timezone.now()) self.data[ 'sc'].current_transfer_session = TransferSession.objects.create( id=uuid.uuid4().hex, sync_session=session, push=True, last_activity_timestamp=timezone.now()) self.data.update( create_buffer_and_store_dummy_data( self.data['sc'].current_transfer_session.id))
class NetworkSyncConnectionTestCase(TestCase): def setUp(self): self.profile = "facilitydata" self.root_scope_def = ScopeDefinition.objects.create( id="rootcert", profile=self.profile, version=1, primary_scope_param_key="mainpartition", description="Root cert for ${mainpartition}.", read_filter_template="", write_filter_template="", read_write_filter_template="${mainpartition}", ) self.subset_scope_def = ScopeDefinition.objects.create( id="subcert", profile=self.profile, version=1, primary_scope_param_key="", description= "Subset cert under ${mainpartition} for ${subpartition}.", read_filter_template="${mainpartition}", write_filter_template="${mainpartition}:${subpartition}", read_write_filter_template="", ) self.root_cert = Certificate.generate_root_certificate( self.root_scope_def.id) self.subset_cert = Certificate( parent=self.root_cert, profile=self.profile, scope_definition=self.subset_scope_def, scope_version=self.subset_scope_def.version, scope_params=json.dumps({ "mainpartition": self.root_cert.id, "subpartition": "abracadabra" }), private_key=Key(), ) self.root_cert.sign_certificate(self.subset_cert) self.subset_cert.save() self.controller = MorangoProfileController('facilitydata') self.network_connection = self.controller.create_network_connection( '127.0.0.1') @mock_patch_decorator def test_creating_sync_session_successful(self): self.assertEqual(SyncSession.objects.filter(active=True).count(), 0) NetworkSyncConnection._request.return_value.json.return_value = { 'signature': 'sig', 'local_fsic': '{}' } self.network_connection.create_sync_session(self.subset_cert, self.root_cert) self.assertEqual(SyncSession.objects.filter(active=True).count(), 1) @mock_patch_decorator def test_creating_sync_session_cert_fails_to_verify(self): Certificate.verify.return_value = False with self.assertRaises(CertificateSignatureInvalid): self.network_connection.create_sync_session( self.subset_cert, self.root_cert) @mock_patch_decorator def test_get_remote_certs(self): # mock certs being returned by server certs = self.subset_cert.get_ancestors(include_self=True) cert_serialized = json.dumps( CertificateSerializer(certs, many=True).data) NetworkSyncConnection._request.return_value.json.return_value = json.loads( cert_serialized) # we want to see if the models are created (not saved) successfully remote_certs = self.network_connection.get_remote_certificates('abc') self.assertSetEqual(set(certs), set(remote_certs)) @mock_patch_decorator def test_csr(self): # mock a "signed" cert being returned by server cert_serialized = json.dumps( CertificateSerializer(self.subset_cert).data) NetworkSyncConnection._request.return_value.json.return_value = json.loads( cert_serialized) self.subset_cert.delete() # we only want to make sure the "signed" cert is saved with mock.patch.object(Key, "get_private_key_string", return_value=self.subset_cert.private_key. get_private_key_string()): self.network_connection.certificate_signing_request( self.root_cert, '', '') self.assertTrue( Certificate.objects.filter( id=json.loads(cert_serialized)['id']).exists()) @mock_patch_decorator def test_get_cert_chain(self): # mock a cert chain being returned by server certs = self.subset_cert.get_ancestors(include_self=True) original_cert_count = certs.count() cert_serialized = json.dumps( CertificateSerializer(certs, many=True).data) NetworkSyncConnection._request.return_value.json.return_value = json.loads( cert_serialized) Certificate.objects.all().delete() # we only want to make sure the cert chain is saved self.network_connection._get_certificate_chain(certs[1]) self.assertEqual(Certificate.objects.count(), original_cert_count)
def setUp(self): (self.current_id, _) = InstanceIDModel.get_or_create_current_instance() self.mc = MorangoProfileController('facilitydata')
def handle_async(self, *args, **options): # validate url that is passed in try: URLValidator()((options["base_url"])) except ValidationError: raise CommandError( "Base URL is not valid. Please retry command and enter a valid URL." ) # call this in case user directly syncs without migrating database if not ScopeDefinition.objects.filter(): call_command("loaddata", "scopedefinitions") controller = MorangoProfileController("facilitydata") with self.start_progress(total=7) as progress_update: try: network_connection = controller.create_network_connection( options["base_url"]) except ConnectionError: raise CommandError( "Can not connect to server with base URL: {}".format( options["base_url"])) # if instance_ids are equal, this means device is trying to sync with itself, which we don't allow if (InstanceIDModel.get_or_create_current_instance()[0].id == network_connection.server_info["instance_id"]): raise CommandError( "Device can not sync with itself. Please recheck base URL and try again." ) progress_update(1) options["dataset_id"] = self.get_dataset_id( options["base_url"], options["dataset_id"]) progress_update(1) client_cert, server_cert, options[ "username"] = self.get_client_and_server_certs( options["username"], options["password"], options["dataset_id"], network_connection, ) progress_update(1) sync_client = network_connection.create_sync_session( client_cert, server_cert, chunk_size=options["chunk_size"]) progress_update(1) # pull from server and push our own data to server if not options["no_pull"]: sync_client.initiate_pull(Filter(options["dataset_id"])) if not options["no_push"]: sync_client.initiate_push(Filter(options["dataset_id"])) progress_update(1) self.create_superuser_and_provision_device(options["username"], options["dataset_id"]) progress_update(1) sync_client.close_sync_session() progress_update(1)
class SerializeIntoStoreTestCase(TestCase): def setUp(self): InstanceIDModel.get_or_create_current_instance() self.range = 10 self.mc = MorangoProfileController('facilitydata') self.original_name = "ralphie" self.new_name = "rafael" def test_all_models_get_serialized(self): [FacilityModelFactory() for _ in range(self.range)] self.mc.serialize_into_store() self.assertEqual(len(Store.objects.all()), self.range) def test_no_models_get_serialized(self): # set dirty bit off on new models created [ FacilityModelFactory.build().save(update_dirty_bit_to=False) for _ in range(self.range) ] # only models with dirty bit on should be serialized self.mc.serialize_into_store() self.assertFalse(Store.objects.exists()) def test_dirty_bit_gets_set(self): [FacilityModelFactory() for _ in range(self.range)] # dirty bit should be on for facility in Facility.objects.all(): self.assertTrue(facility._morango_dirty_bit) self.mc.serialize_into_store() # dirty bit should have been toggled off for facility in Facility.objects.all(): self.assertFalse(facility._morango_dirty_bit) def test_store_models_get_updated(self): FacilityModelFactory(name=self.original_name) self.mc.serialize_into_store() store_facility = Store.objects.first() deserialized_model = json.loads(store_facility.serialized) self.assertEqual(deserialized_model['name'], self.original_name) Facility.objects.update(name=self.new_name) self.mc.serialize_into_store() store_facility = Store.objects.first() deserialized_model = json.loads(store_facility.serialized) self.assertEqual(deserialized_model['name'], self.new_name) def test_last_saved_counter_updates(self): FacilityModelFactory(name=self.original_name) self.mc.serialize_into_store() old_counter = Store.objects.first().last_saved_counter Facility.objects.all().update(name=self.new_name) self.mc.serialize_into_store() new_counter = Store.objects.first().last_saved_counter self.assertEqual(old_counter + 1, new_counter) def test_last_saved_instance_updates(self): FacilityModelFactory(name=self.original_name) self.mc.serialize_into_store() old_instance_id = Store.objects.first().last_saved_instance with mock.patch('platform.platform', return_value='Windows'): (new_id, _) = InstanceIDModel.get_or_create_current_instance() Facility.objects.all().update(name=self.new_name) self.mc.serialize_into_store() new_instance_id = Store.objects.first().last_saved_instance self.assertNotEqual(old_instance_id, new_instance_id) self.assertEqual(new_instance_id, new_id.id) def test_extra_fields_dont_get_overwritten(self): serialized = """{"username": "******", "height": 6.0, "weight": 100}""" MyUser.objects.create(username='******') self.mc.serialize_into_store() Store.objects.update(serialized=serialized) MyUser.objects.update(username='******') self.mc.serialize_into_store() serialized = json.loads(Store.objects.first().serialized) self.assertIn('height', serialized) def test_updates_store_deleted_flag(self): fac = FacilityModelFactory() fac_id = fac.id self.mc.serialize_into_store() self.assertFalse(Store.objects.get(pk=fac_id).deleted) fac.delete() self.assertTrue(DeletedModels.objects.exists()) self.mc.serialize_into_store() self.assertFalse(DeletedModels.objects.exists()) self.assertTrue(Store.objects.get(pk=fac_id).deleted) def test_cascading_delete_updates_store_deleted_flag(self): fac = FacilityModelFactory() child = FacilityModelFactory(parent_id=fac.id) child_id = child.id self.mc.serialize_into_store() self.assertFalse(Store.objects.get(pk=child_id).deleted) fac.delete() self.mc.serialize_into_store() self.assertTrue(Store.objects.get(pk=child_id).deleted) def test_conflicting_data_appended(self): self.maxDiff = None serialized = json.dumps({"username": "******"}) conflicting = [] user = MyUser.objects.create(username="******") self.mc.serialize_into_store() # add serialized fields to conflicting data conflicting.insert(0, serialized) conflicting.insert(0, json.dumps(user.serialize())) # set store record and app record dirty bits to true to force serialization merge conflict Store.objects.update(conflicting_serialized_data=serialized, dirty_bit=True) user.username = "******" user.save(update_dirty_bit_to=True) self.mc.serialize_into_store() # assert we have placed serialized object into store's serialized field st = Store.objects.get(id=user.id) self.assertEqual(json.loads(st.serialized), user.serialize()) # assert store serialized field is moved to conflicting data conflicting_serialized_data = st.conflicting_serialized_data.split( '\n') for x in range(len(conflicting)): self.assertEqual(conflicting[x], conflicting_serialized_data[x]) def test_filtered_serialization_single_filter(self): fac = FacilityModelFactory() user = MyUser.objects.create(username='******') log = SummaryLog.objects.create(user=user) self.mc.serialize_into_store(filter=Filter(user._morango_partition)) self.assertFalse(Store.objects.filter(id=fac.id).exists()) self.assertTrue(Store.objects.filter(id=user.id).exists()) self.assertTrue(Store.objects.filter(id=log.id).exists()) def test_filtered_serialization_multiple_filter(self): fac = FacilityModelFactory() user = MyUser.objects.create(username='******') user2 = MyUser.objects.create(username='******') log = SummaryLog.objects.create(user=user) self.mc.serialize_into_store(filter=Filter(user._morango_partition + "\n" + user2._morango_partition)) self.assertFalse(Store.objects.filter(id=fac.id).exists()) self.assertTrue(Store.objects.filter(id=user2.id).exists()) self.assertTrue(Store.objects.filter(id=user.id).exists()) self.assertTrue(Store.objects.filter(id=log.id).exists()) def test_self_ref_fk_class_adds_value_to_store(self): root = FacilityModelFactory() child = FacilityModelFactory(parent=root) self.mc.serialize_into_store() self.assertEqual(Store.objects.get(id=child.id)._self_ref_fk, root.id) def test_regular_class_leaves_value_blank_in_store(self): log = SummaryLog.objects.create(user=MyUser.objects.create( username='******')) self.mc.serialize_into_store() self.assertEqual(Store.objects.get(id=log.id)._self_ref_fk, '') def test_previously_deleted_store_flag_resets(self): # create and delete object user = MyUser.objects.create(username='******') self.mc.serialize_into_store() MyUser.objects.all().delete() self.mc.serialize_into_store() self.assertTrue(Store.objects.get(id=user.id).deleted) # recreate object with same id user = MyUser.objects.create(username='******') # ensure deleted flag is updated after recreation self.mc.serialize_into_store() self.assertFalse(Store.objects.get(id=user.id).deleted)
def setUp(self): DatabaseIDModel.objects.create() InstanceIDModel.get_or_create_current_instance() [FacilityModelFactory() for _ in range(10)] self.mc = MorangoProfileController('facilitydata') self.mc.serialize_into_store()
class DeserializationFromStoreIntoAppTestCase(TestCase): def setUp(self): (self.current_id, _) = InstanceIDModel.get_or_create_current_instance() self.range = 10 self.mc = MorangoProfileController('facilitydata') for i in range(self.range): self.ident = uuid.uuid4().hex StoreModelFacilityFactory(pk=self.ident, serialized=serialized_facility_factory( self.ident)) def test_dirty_store_records_are_deserialized(self): self.assertFalse(Facility.objects.all().exists()) self.mc.deserialize_from_store() self.assertEqual(len(Facility.objects.all()), self.range) def test_clean_store_records_do_not_get_deserialized(self): self.assertFalse(Facility.objects.exists()) Store.objects.update(dirty_bit=False) self.mc.deserialize_from_store() self.assertFalse(Facility.objects.exists()) def test_deleted_models_do_not_get_deserialized(self): Store.objects.update_or_create(defaults={'deleted': True}, id=self.ident) self.mc.deserialize_from_store() self.assertFalse(Facility.objects.filter(id=self.ident).exists()) def test_deleted_models_deletes_them_in_app(self): # put models in app layer self.mc.deserialize_from_store() # deleted flag on store should delete model in app layer Store.objects.update_or_create(defaults={ 'deleted': True, 'dirty_bit': True }, id=self.ident) self.mc.deserialize_from_store() self.assertFalse(Facility.objects.filter(id=self.ident).exists()) def test_update_app_with_newer_data_from_store(self): name = 'test' fac = FacilityModelFactory(id=self.ident, name=name) fac.save(update_dirty_bit_to=False) self.assertEqual(fac.name, name) self.mc.deserialize_from_store() fac = Facility.objects.get(id=self.ident) self.assertNotEqual(fac.name, name) def test_handle_extra_field_deserialization(self): # modify a store record by adding extra serialized field store_model = Store.objects.get(id=self.ident) serialized = json.loads(store_model.serialized) serialized.update({'wacky': True}) store_model.serialized = json.dumps(serialized) store_model.save() # deserialize records self.mc.deserialize_from_store() # by this point no errors should have occurred but we check list of fields anyways fac = Facility.objects.get(id=self.ident) self.assertNotIn('wacky', fac.__dict__) def test_store_dirty_bit_resets(self): self.assertTrue(Store.objects.filter(dirty_bit=True)) self.mc.deserialize_from_store() self.assertFalse(Store.objects.filter(dirty_bit=True)) def test_record_with_dirty_bit_off_doesnt_deserialize(self): st = Store.objects.first() st.dirty_bit = False st.save() self.mc.deserialize_from_store() self.assertFalse(Facility.objects.filter(id=st.id).exists()) def test_broken_fk_adds_to_deleted_models(self): serialized = """{"user_id": "40de9a3fded95d7198f200c78e559353"}""" StoreModelFacilityFactory(id=uuid.uuid4().hex, serialized=serialized, model_name="contentsummarylog") with mock.patch('morango.models.SyncableModel._update_deleted_models' ) as mock_update: self.mc.deserialize_from_store() self.assertTrue(mock_update.called)
class DeserializationFromStoreIntoAppTestCase(TestCase): def setUp(self): DatabaseIDModel.objects.create() (self.current_id, _) = InstanceIDModel.get_or_create_current_instance() self.range = 10 self.mc = MorangoProfileController('facilitydata') for i in range(self.range): self.ident = uuid.uuid4().hex StoreModelFacilityFactory(pk=self.ident, serialized=serialized_facility_factory( self.ident)) def test_dirty_store_records_are_deserialized(self): self.assertFalse(Facility.objects.all().exists()) self.mc.deserialize_from_store() self.assertEqual(len(Facility.objects.all()), self.range) def test_clean_store_records_do_not_get_deserialized(self): self.assertFalse(Facility.objects.exists()) Store.objects.update(dirty_bit=False) self.mc.deserialize_from_store() self.assertFalse(Facility.objects.exists()) def test_deleted_models_do_not_get_deserialized(self): Store.objects.update_or_create(defaults={'deleted': True}, id=self.ident) self.mc.deserialize_from_store() self.assertFalse(Facility.objects.filter(id=self.ident).exists()) def test_deleted_models_deletes_them_in_app(self): # put models in app layer self.mc.deserialize_from_store() # deleted flag on store should delete model in app layer Store.objects.update_or_create(defaults={ 'deleted': True, 'dirty_bit': True }, id=self.ident) self.mc.deserialize_from_store() self.assertFalse(Facility.objects.filter(id=self.ident).exists()) def test_update_app_with_newer_data_from_store(self): name = 'test' fac = FacilityModelFactory(id=self.ident, name=name) fac.save(update_dirty_bit_to=False) self.assertEqual(fac.name, name) self.mc.deserialize_from_store() fac = Facility.objects.get(id=self.ident) self.assertNotEqual(fac.name, name) def test_handle_extra_field_deserialization(self): # modify a store record by adding extra serialized field store_model = Store.objects.get(id=self.ident) serialized = json.loads(store_model.serialized) serialized.update({'wacky': True}) store_model.serialized = json.dumps(serialized) store_model.save() # deserialize records self.mc.deserialize_from_store() # by this point no errors should have occurred but we check list of fields anyways fac = Facility.objects.get(id=self.ident) self.assertNotIn('wacky', fac.__dict__) def test_broken_foreign_key_deserialization(self): # add fake foreign key store_model = Store.objects.get(id=self.ident) serialized = json.loads(store_model.serialized) serialized.update({'parent_id': '4d53c8e72b8bea87a393910ff0dcb212'}) store_model.serialized = json.dumps(serialized) store_model.save() # deserialize records self.mc.deserialize_from_store() self.assertTrue( DeletedModels.objects.filter(id=store_model.id).exists()) def test_store_dirty_bit_resets(self): self.assertTrue(Store.objects.filter(dirty_bit=True)) self.mc.deserialize_from_store() self.assertFalse(Store.objects.filter(dirty_bit=True)) def test_record_with_dirty_bit_off_doesnt_deserialize(self): st = Store.objects.first() st.dirty_bit = False st.save() self.mc.deserialize_from_store() self.assertFalse(Facility.objects.filter(id=st.id).exists())
class NetworkSyncConnectionTestCase(LiveServerTestCase): def setUp(self): self.profile = "facilitydata" self.root_scope_def = ScopeDefinition.objects.create( id="rootcert", profile=self.profile, version=1, primary_scope_param_key="mainpartition", description="Root cert for ${mainpartition}.", read_filter_template="", write_filter_template="", read_write_filter_template="${mainpartition}", ) self.subset_scope_def = ScopeDefinition.objects.create( id="subcert", profile=self.profile, version=1, primary_scope_param_key="", description= "Subset cert under ${mainpartition} for ${subpartition}.", read_filter_template="${mainpartition}", write_filter_template="${mainpartition}:${subpartition}", read_write_filter_template="", ) self.root_cert = Certificate.generate_root_certificate( self.root_scope_def.id) self.subset_cert = Certificate( parent=self.root_cert, profile=self.profile, scope_definition=self.subset_scope_def, scope_version=self.subset_scope_def.version, scope_params=json.dumps({ "mainpartition": self.root_cert.id, "subpartition": "abracadabra" }), private_key=Key(), ) self.root_cert.sign_certificate(self.subset_cert) self.subset_cert.save() self.unsaved_cert = Certificate( parent=self.root_cert, profile=self.profile, scope_definition=self.subset_scope_def, scope_version=self.subset_scope_def.version, scope_params=json.dumps({ "mainpartition": self.root_cert.id, "subpartition": "other" }), public_key=Key(), ) self.root_cert.sign_certificate(self.unsaved_cert) self.controller = MorangoProfileController('facilitydata') self.network_connection = self.controller.create_network_connection( self.live_server_url) self.key = SharedKey.get_or_create_shared_key() @mock.patch.object(SyncSession.objects, 'create', return_value=None) def test_creating_sync_session_successful(self, mock_object): self.assertEqual(SyncSession.objects.filter(active=True).count(), 0) self.network_connection.create_sync_session(self.subset_cert, self.root_cert) self.assertEqual(SyncSession.objects.filter(active=True).count(), 1) @mock.patch.object(NetworkSyncConnection, '_create_sync_session') @mock.patch.object(Certificate, 'verify', return_value=False) def test_creating_sync_session_cert_fails_to_verify( self, mock_verify, mock_create): mock_create.return_value.json.return_value = {} with self.assertRaises(CertificateSignatureInvalid): self.network_connection.create_sync_session( self.subset_cert, self.root_cert) def test_get_remote_certs(self): certs = self.subset_cert.get_ancestors(include_self=True) remote_certs = self.network_connection.get_remote_certificates( self.root_cert.id) self.assertSetEqual(set(certs), set(remote_certs)) @mock.patch.object(NetworkSyncConnection, '_request') def test_csr(self, mock_request): # mock a "signed" cert being returned by server cert_serialized = json.dumps( CertificateSerializer(self.subset_cert).data) mock_request.return_value.json.return_value = json.loads( cert_serialized) self.subset_cert.delete() # we only want to make sure the "signed" cert is saved with mock.patch.object(Key, "get_private_key_string", return_value=self.subset_cert.private_key. get_private_key_string()): self.network_connection.certificate_signing_request( self.root_cert, '', '') self.assertTrue( Certificate.objects.filter( id=json.loads(cert_serialized)['id']).exists()) @override_settings(ALLOW_CERTIFICATE_PUSHING=True) def test_push_signed_client_certificate_chain(self): cert = self.network_connection.push_signed_client_certificate_chain( self.root_cert, self.subset_scope_def.id, { "mainpartition": self.root_cert.id, "subpartition": "abracadabra" }) self.assertEqual(cert.private_key, None) self.assertTrue(Certificate.objects.filter(id=cert.id).exists()) @override_settings(ALLOW_CERTIFICATE_PUSHING=True) def test_push_signed_client_certificate_chain_publickey_error(self): with mock.patch.object(NetworkSyncConnection, '_get_public_key'): NetworkSyncConnection._get_public_key.return_value.json.return_value = [ { 'public_key': Key().get_public_key_string() } ] with self.assertRaises(HTTPError) as e: self.network_connection.push_signed_client_certificate_chain( self.root_cert, self.subset_scope_def.id, { "mainpartition": self.root_cert.id, "subpartition": "abracadabra" }) self.assertEqual(e.exception.response.status_code, 400) @override_settings(ALLOW_CERTIFICATE_PUSHING=True) def test_push_signed_client_certificate_chain_bad_cert(self): with self.assertRaises(HTTPError) as e: self.network_connection.push_signed_client_certificate_chain( self.root_cert, self.subset_scope_def.id, {"bad": "scope_params"}) self.assertEqual(e.exception.response.status_code, 400) @override_settings(ALLOW_CERTIFICATE_PUSHING=True) @mock.patch.object(NetworkSyncConnection, '_get_nonce') def test_push_signed_client_certificate_chain_nonce_error( self, mock_nonce): mock_nonce.return_value.json.return_value = {'id': uuid.uuid4().hex} with self.assertRaises(HTTPError) as e: self.network_connection.push_signed_client_certificate_chain( self.root_cert, self.subset_scope_def.id, { "mainpartition": self.root_cert.id, "subpartition": "abracadabra" }) self.assertEqual(e.exception.response.status_code, 403) def test_push_signed_client_certificate_chain_not_allowed(self): with self.assertRaises(MorangoServerDoesNotAllowNewCertPush) as e: self.network_connection.push_signed_client_certificate_chain( self.root_cert, self.subset_scope_def.id, { "mainpartition": self.root_cert.id, "subpartition": "abracadabra" }) self.assertEqual(e.exception.response.status_code, 403) def test_get_cert_chain(self): response = self.network_connection._get_certificate_chain( self.subset_cert) data = response.json() self.assertEqual(len(data), Certificate.objects.count()) self.assertEqual(data[0]['id'], self.root_cert.id) self.assertEqual(data[1]['id'], self.subset_cert.id)
class DeserializationFromStoreIntoAppTestCase(TestCase): def setUp(self): (self.current_id, _) = InstanceIDModel.get_or_create_current_instance() self.range = 10 self.mc = MorangoProfileController('facilitydata') for i in range(self.range): self.ident = uuid.uuid4().hex StoreModelFacilityFactory(pk=self.ident, serialized=serialized_facility_factory( self.ident)) def test_dirty_store_records_are_deserialized(self): self.assertFalse(Facility.objects.all().exists()) self.mc.deserialize_from_store() self.assertEqual(len(Facility.objects.all()), self.range) def test_clean_store_records_do_not_get_deserialized(self): self.assertFalse(Facility.objects.exists()) Store.objects.update(dirty_bit=False) self.mc.deserialize_from_store() self.assertFalse(Facility.objects.exists()) def test_deleted_models_do_not_get_deserialized(self): Store.objects.update_or_create(defaults={'deleted': True}, id=self.ident) self.mc.deserialize_from_store() self.assertFalse(Facility.objects.filter(id=self.ident).exists()) def test_deleted_models_deletes_them_in_app(self): # put models in app layer self.mc.deserialize_from_store() # deleted flag on store should delete model in app layer Store.objects.update_or_create(defaults={ 'deleted': True, 'dirty_bit': True }, id=self.ident) self.mc.deserialize_from_store() self.assertFalse(Facility.objects.filter(id=self.ident).exists()) def test_update_app_with_newer_data_from_store(self): name = 'test' fac = FacilityModelFactory(id=self.ident, name=name) fac.save(update_dirty_bit_to=False) self.assertEqual(fac.name, name) self.mc.deserialize_from_store() fac = Facility.objects.get(id=self.ident) self.assertNotEqual(fac.name, name) def test_handle_extra_field_deserialization(self): # modify a store record by adding extra serialized field store_model = Store.objects.get(id=self.ident) serialized = json.loads(store_model.serialized) serialized.update({'wacky': True}) store_model.serialized = json.dumps(serialized) store_model.save() # deserialize records self.mc.deserialize_from_store() # by this point no errors should have occurred but we check list of fields anyways fac = Facility.objects.get(id=self.ident) self.assertNotIn('wacky', fac.__dict__) def test_store_dirty_bit_resets(self): self.assertTrue(Store.objects.filter(dirty_bit=True)) self.mc.deserialize_from_store() self.assertFalse(Store.objects.filter(dirty_bit=True)) def test_record_with_dirty_bit_off_doesnt_deserialize(self): st = Store.objects.first() st.dirty_bit = False st.save() self.mc.deserialize_from_store() self.assertFalse(Facility.objects.filter(id=st.id).exists()) def test_broken_fk_leaves_store_dirty_bit(self): serialized = """{"user_id": "40de9a3fded95d7198f200c78e559353", "id": "bd205b5ee5bc42da85925d24c61341a8"}""" st = StoreModelFacilityFactory(id=uuid.uuid4().hex, serialized=serialized, model_name="contentsummarylog") self.mc.deserialize_from_store() st.refresh_from_db() self.assertTrue(st.dirty_bit) def test_invalid_model_leaves_store_dirty_bit(self): user = MyUser(username='******' * 21) st = StoreModelFacilityFactory(model_name="user", id=uuid.uuid4().hex, serialized=json.dumps(user.serialize())) self.mc.deserialize_from_store() st.refresh_from_db() self.assertTrue(st.dirty_bit) def test_deleted_model_propagates_to_store_record(self): """ It could be the case that we have two store records, one that is deleted and the other that has a fk pointing to the deleted record. When we deserialize, we want to ensure that the record with the fk pointer also gets the deleted flag set, while also not deserializing the data into a model. """ # user will be deleted user = MyUser(username='******') user.save(update_dirty_bit_to=False) # log may be synced in from other device log = SummaryLog(user_id=user.id) log.id = log.calculate_uuid() StoreModelFacilityFactory(model_name="user", id=user.id, serialized=json.dumps(user.serialize()), deleted=True) StoreModelFacilityFactory(model_name="contentsummarylog", id=log.id, serialized=json.dumps(log.serialize())) # make sure delete propagates to store due to deleted foreign key self.mc.deserialize_from_store() # have to serialize to update deleted models self.mc.serialize_into_store() self.assertFalse(SummaryLog.objects.filter(id=log.id).exists()) self.assertTrue(Store.objects.get(id=log.id).deleted) def test_hard_deleted_model_propagates_to_store_record(self): """ It could be the case that we have two store records, one that is hard deleted and the other that has a fk pointing to the hard deleted record. When we deserialize, we want to ensure that the record with the fk pointer also gets the hard deleted flag set, while also not deserializing the data into a model. """ # user will be deleted user = MyUser(username='******') user.save(update_dirty_bit_to=False) # log may be synced in from other device log = SummaryLog(user_id=user.id) log.id = log.calculate_uuid() StoreModelFacilityFactory(model_name="user", id=user.id, serialized=json.dumps(user.serialize()), deleted=True, hard_deleted=True) StoreModelFacilityFactory(model_name="contentsummarylog", id=log.id, serialized=json.dumps(log.serialize())) # make sure delete propagates to store due to deleted foreign key self.mc.deserialize_from_store() # have to serialize to update deleted models self.mc.serialize_into_store() self.assertFalse(SummaryLog.objects.filter(id=log.id).exists()) self.assertTrue(Store.objects.get(id=log.id).hard_deleted) def test_regular_model_deserialization(self): # deserialization should be able to handle multiple records user = MyUser(username='******', password='******') user2 = MyUser(username='******', password='******') user.save(update_dirty_bit_to=False) user2.save(update_dirty_bit_to=False) user.username = '******' user2.username = '******' StoreModelFacilityFactory(id=user.id, serialized=json.dumps(user.serialize()), model_name="user") StoreModelFacilityFactory(id=user2.id, serialized=json.dumps(user2.serialize()), model_name="user") self.mc.deserialize_from_store() self.assertFalse(MyUser.objects.filter(username='******').exists()) self.assertFalse(MyUser.objects.filter(username='******').exists()) self.assertTrue(MyUser.objects.filter(username='******').exists()) self.assertTrue(MyUser.objects.filter(username='******').exists())
def setUp(self): self.controller = MorangoProfileController('facilitydata') InstanceIDModel.get_or_create_current_instance()
def handle_async(self, *args, **options): # validate url that is passed in try: URLValidator()((options['base_url'])) except ValidationError: print( 'Base-url is not valid. Please retry command and enter a valid url.' ) sys.exit(1) # call this in case user directly syncs without migrating database if not ScopeDefinition.objects.filter(): call_command("loaddata", "scopedefinitions") # ping server at url with info request info_url = urljoin(options['base_url'], 'api/morango/v1/morangoinfo/1/') try: info_resp = requests.get(info_url) except ConnectionError: print('Can not connect to server with base-url: {}'.format( options['base_url'])) sys.exit(1) # if instance_ids are equal, this means device is trying to sync with itself, which we don't allow if InstanceIDModel.get_or_create_current_instance( )[0].id == info_resp.json()['instance_id']: print( 'Device can not sync with itself. Please re-check base-url and try again.' ) sys.exit(1) controller = MorangoProfileController('facilitydata') with self.start_progress(total=7) as progress_update: network_connection = controller.create_network_connection( options['base_url']) progress_update(1) options['dataset_id'] = self.get_dataset_id( options['base_url'], options['dataset_id']) progress_update(1) client_cert, server_cert, options[ 'username'] = self.get_client_and_server_certs( options['username'], options['password'], options['dataset_id'], network_connection) progress_update(1) sync_client = network_connection.create_sync_session( client_cert, server_cert) progress_update(1) # pull from server and push our own data to server if not options['no_pull']: sync_client.initiate_pull(Filter(options['dataset_id'])) if not options['no_push']: sync_client.initiate_push(Filter(options['dataset_id'])) progress_update(1) self.create_superuser_and_provision_device(options['username'], options['dataset_id']) progress_update(1) sync_client.close_sync_session() progress_update(1)
def handle_async(self, *args, **options): baseurl = options["baseurl"] facility_id = options["facility"] chunk_size = options["chunk_size"] # This handles the case for when we want to pull in facility data for our empty kolibri instance if not facility_id: self._fullfacilitysync(baseurl) facility = get_facility( facility_id=facility_id, noninteractive=options["noninteractive"] ) # if url is not pointing to portal server, do P2P syncing self._fullfacilitysync(baseurl, facility=facility, chunk_size=chunk_size) # data portal syncing self.stdout.write("Syncing has been initiated (this may take a while)...") controller = MorangoProfileController("facilitydata") with self.start_progress(total=5) as progress_update: try: network_connection = controller.create_network_connection(baseurl) except ConnectionError as e: raise CommandError(e) progress_update(1) # get client certificate client_cert = ( facility.dataset.get_owned_certificates() .filter(scope_definition_id=FULL_FACILITY) .first() ) if not client_cert: raise CommandError( "This device does not own a certificate for Facility: {}".format( facility.name ) ) # push certificate up to portal server scope_params = json.loads(client_cert.scope_params) server_cert = network_connection.push_signed_client_certificate_chain( local_parent_cert=client_cert, scope_definition_id=FULL_FACILITY, scope_params=scope_params, ) progress_update(1) # we should now be able to push our facility data sync_client = network_connection.create_sync_session( client_cert, server_cert, chunk_size=chunk_size ) progress_update(1) sync_client.initiate_push(Filter(scope_params["dataset_id"])) progress_update(1) sync_client.close_sync_session() progress_update(1) self.stdout.write("Syncing has been completed.")