def test_instance_id_caching(self, *args):
"""
Ensure that the cache works but that clearing it works as well.
"""
with EnvironmentVarGuard() as env:
env["MORANGO_SYSTEM_ID"] = "oldmagicsysid"
old_instance, created = InstanceIDModel.get_or_create_current_instance(clear_cache=True)
self.assertTrue(created)
env["MORANGO_SYSTEM_ID"] = "newmagicsysid"
cached_instance, created = InstanceIDModel.get_or_create_current_instance()
self.assertFalse(created)
uncached_instance, created = InstanceIDModel.get_or_create_current_instance(clear_cache=True)
self.assertTrue(created)
recached_instance, created = InstanceIDModel.get_or_create_current_instance()
self.assertFalse(created)
self.assertEqual(old_instance.id, cached_instance.id)
self.assertNotEqual(old_instance.id, uncached_instance.id)
self.assertEqual(uncached_instance.id, recached_instance.id)
def setUp(self):
(self.current_id, _) = InstanceIDModel.get_or_create_current_instance()
self.range = 10
self.mc = MorangoProfileController("facilitydata")
for i in range(self.range):
self.ident = uuid.uuid4().hex
StoreModelFacilityFactory(
pk=self.ident, serialized=serialized_facility_factory(self.ident)
)
def setUp(self):
self.session = SyncSession.objects.create(
id=uuid.uuid4().hex,
profile="facilitydata",
last_activity_timestamp=timezone.now(),
)
self.transfer_session = TransferSession.objects.create(
id=uuid.uuid4().hex,
sync_session=self.session,
filter="partition",
push=True,
last_activity_timestamp=timezone.now(),
records_total=3,
)
self.chunk_size = 3
self.conn = NetworkSyncConnection(base_url=self.live_server_url)
self.syncclient = self.build_client(BaseSyncClient)
InstanceIDModel.get_or_create_current_instance()
def retrieve(self, request, pk=None):
(id_model, _) = InstanceIDModel.get_or_create_current_instance()
m_info = {
"instance_hash": id_model.get_proquint(),
"instance_id": id_model.id,
"system_os": platform.system(),
"version": morango.__version__,
"capabilities": CAPABILITIES,
}
return response.Response(m_info)
def test_same_node_id(self):
with mock.patch(
"uuid.getnode", return_value=67002173923623
): # fake (random) address
(IDModel, _) = InstanceIDModel.get_or_create_current_instance()
ident = IDModel.id
with mock.patch(
"uuid.getnode", return_value=69002173923623
): # fake (random) address
(IDModel, _) = InstanceIDModel.get_or_create_current_instance()
with mock.patch(
"uuid.getnode", return_value=67002173923623
): # fake (random) address
(IDModel, _) = InstanceIDModel.get_or_create_current_instance()
self.assertFalse(
InstanceIDModel.objects.exclude(id=ident).filter(current=True).exists()
)
self.assertTrue(InstanceIDModel.objects.get(id=ident).current)
def test_last_saved_instance_updates(self):
FacilityModelFactory(name=self.original_name)
self.mc.serialize_into_store()
old_instance_id = Store.objects.first().last_saved_instance
with EnvironmentVarGuard() as env:
env['MORANGO_SYSTEM_ID'] = 'new_sys_id'
(new_id, _) = InstanceIDModel.get_or_create_current_instance(clear_cache=True)
Facility.objects.all().update(name=self.new_name)
self.mc.serialize_into_store()
new_instance_id = Store.objects.first().last_saved_instance
self.assertNotEqual(old_instance_id, new_instance_id)
self.assertEqual(new_instance_id, new_id.id)
def test_new_rmc_for_existing_model(self):
with EnvironmentVarGuard() as env:
env['MORANGO_SYSTEM_ID'] = 'new_sys_id'
(new_id, _) = InstanceIDModel.get_or_create_current_instance(clear_cache=True)
Facility.objects.update(name="facility")
self.mc.serialize_into_store()
new_rmc = RecordMaxCounter.objects.get(
instance_id=new_id.id, store_model_id=self.fac1.id
)
new_store_record = Store.objects.get(id=self.fac1.id)
self.assertEqual(new_rmc.counter, new_store_record.last_saved_counter)
self.assertEqual(new_rmc.instance_id, new_store_record.last_saved_instance)
def test_consistent_0_5_instance_id(self, *args):
"""
If this test fails, it means we've changed the way Instance IDs are calculated in an undesirable way.
"""
with EnvironmentVarGuard() as env:
env["MORANGO_SYSTEM_ID"] = "magicsysid"
DatabaseIDModel.objects.all().update(current=False)
database_id = DatabaseIDModel.objects.create(
id="7fe445b75cea11858c00fb97bdee8878", current=True
).id
self.assertEqual(get_0_5_system_id(), "54940f560a55bbf7d86b")
self.assertEqual(get_0_5_mac_address(), "804f4c20d3b2b5a29b95")
instance, _ = InstanceIDModel.get_or_create_current_instance(clear_cache=True)
self.assertEqual(instance.id, "e45c06595d820f4581e0c82930359592")
def test_consistent_with_0_4_instance_id_calculation(self, *args):
"""
This test ensures that we don't accidentally make changes that impact how we calculate
the instance ID, in a way that would cause instance IDs to change when they shouldn't.
"""
from morango.models.utils import _get_database_path
sys.version = "2.7.333"
DatabaseIDModel.objects.all().update(current=False)
database_id = DatabaseIDModel.objects.create(
id="6fe445b75cea11858c00fb97bdee8878", current=True
).id
node_id = hashlib.sha1(
"{}:{}".format(database_id, 24359248572014).encode("utf-8")
).hexdigest()[:20]
target = {
"platform": "Windows 3.1",
"hostname": "myhost",
"sysversion": "2.7.333",
"node_id": node_id,
"database_id": database_id,
"db_path": _get_database_path(),
}
result = get_0_4_system_parameters(database_id)
self.assertEqual(target, result)
calculated_id = _calculate_0_4_uuid(result)
self.assertEqual(calculated_id, "4480fda04236975d0895c0048b767647")
InstanceIDModel.objects.all().delete()
InstanceIDModel.objects.create(current=True, id=calculated_id, **result)
instance, _ = InstanceIDModel.get_or_create_current_instance()
self.assertEqual(calculated_id, instance.id)
def setUp(self):
settings.MORANGO_DESERIALIZE_AFTER_DEQUEUING = False
self.data = {}
DatabaseIDModel.objects.create()
(self.current_id, _) = InstanceIDModel.get_or_create_current_instance()
# create controllers for app/store/buffer operations
self.data["mc"] = MorangoProfileController("facilitydata")
self.data["sc"] = BaseSyncClient(None, "host")
session = SyncSession.objects.create(
id=uuid.uuid4().hex,
profile="",
last_activity_timestamp=timezone.now())
self.data[
"sc"].current_transfer_session = TransferSession.objects.create(
id=uuid.uuid4().hex,
sync_session=session,
push=True,
last_activity_timestamp=timezone.now(),
)
self.data.update(
create_buffer_and_store_dummy_data(
self.data["sc"].current_transfer_session.id))
def test_envvar_overrides(self, *args):
with EnvironmentVarGuard() as env:
env["MORANGO_SYSTEM_ID"] = "magicsysid"
env["MORANGO_NODE_ID"] = "magicnodeid"
DatabaseIDModel.objects.all().update(current=False)
database_id = DatabaseIDModel.objects.create(
id="7fe445b75cea11858c00fb97bdee8878", current=True
).id
system_id = get_0_5_system_id()
node_id = get_0_5_mac_address()
self.assertEqual(system_id, "54940f560a55bbf7d86b")
self.assertEqual(node_id, "9ed21d0fb4dacfa4009d")
instance, _ = InstanceIDModel.get_or_create_current_instance(clear_cache=True)
self.assertEqual(instance.id, "9033c0cec24d8a8d906dcba416f77625")
expected_id = sha2_uuid(database_id, system_id, node_id)
self.assertEqual(instance.id, expected_id)
def test_only_one_current_instance_ID(self):
with mock.patch("platform.platform", return_value="platform"):
InstanceIDModel.get_or_create_current_instance()
self.assertEqual(len(InstanceIDModel.objects.filter(current=True)), 1)
def test_creating_same_instance_ID_model(self):
firstIDModel = InstanceIDModel.objects.first()
(secondIDModel, _) = InstanceIDModel.get_or_create_current_instance()
self.assertEqual(firstIDModel, secondIDModel)
self.assertEqual(InstanceIDModel.objects.count(), 1)
def setUp(self):
InstanceIDModel.get_or_create_current_instance()
def setUp(self):
(self.current_id, _) = InstanceIDModel.get_or_create_current_instance()
self.mc = MorangoProfileController("facilitydata")
self.fac1 = FacilityModelFactory(name="school")
self.mc.serialize_into_store()
self.old_rmc = RecordMaxCounter.objects.first()
def setUp(self):
InstanceIDModel.get_or_create_current_instance()
self.range = 10
self.mc = MorangoProfileController("facilitydata")
self.original_name = "ralphie"
self.new_name = "rafael"
def setUp(self):
(self.current_id, _) = InstanceIDModel.get_or_create_current_instance()
self.mc = MorangoProfileController("facilitydata")
def create_sync_session(self, client_cert, server_cert, chunk_size=500):
# if server cert does not exist locally, retrieve it from server
if not Certificate.objects.filter(id=server_cert.id).exists():
cert_chain_response = self._get_certificate_chain(
params={"ancestors_of": server_cert.id})
# upon receiving cert chain from server, we attempt to save the chain into our records
Certificate.save_certificate_chain(cert_chain_response.json(),
expected_last_id=server_cert.id)
# request the server for a one-time-use nonce
nonce_resp = self._get_nonce()
nonce = nonce_resp.json()["id"]
# if no hostname then url is actually an ip
url = urlparse(self.base_url)
hostname = url.hostname or self.base_url
port = url.port or (80 if url.scheme == "http" else 443)
# prepare the data to send in the syncsession creation request
data = {
"id":
uuid.uuid4().hex,
"server_certificate_id":
server_cert.id,
"client_certificate_id":
client_cert.id,
"profile":
client_cert.profile,
"certificate_chain":
json.dumps(
CertificateSerializer(
client_cert.get_ancestors(include_self=True),
many=True).data),
"connection_path":
self.base_url,
"instance":
json.dumps(
InstanceIDSerializer(
InstanceIDModel.get_or_create_current_instance()[0]).data),
"nonce":
nonce,
"client_ip":
_get_client_ip_for_server(hostname, port),
"server_ip":
_get_server_ip(hostname),
}
# sign the nonce/ID combo to attach to the request
message = "{nonce}:{id}".format(**data)
data["signature"] = client_cert.sign(message)
# Sync Session creation request
session_resp = self._create_sync_session(data)
# check that the nonce/id were properly signed by the server cert
if not server_cert.verify(message,
session_resp.json().get("signature")):
raise CertificateSignatureInvalid()
# build the data to be used for creating our own syncsession
data = {
"id":
data["id"],
"start_timestamp":
timezone.now(),
"last_activity_timestamp":
timezone.now(),
"active":
True,
"is_server":
False,
"client_certificate":
client_cert,
"server_certificate":
server_cert,
"profile":
client_cert.profile,
"connection_kind":
"network",
"connection_path":
self.base_url,
"client_ip":
data["client_ip"],
"server_ip":
data["server_ip"],
"client_instance":
json.dumps(
InstanceIDSerializer(
InstanceIDModel.get_or_create_current_instance()[0]).data),
"server_instance":
session_resp.json().get("server_instance") or "{}",
}
sync_session = SyncSession.objects.create(**data)
return SyncSessionClient(self, sync_session, chunk_size=chunk_size)
def create_dummy_store_data():
data = {}
DatabaseIDModel.objects.create()
data["group1_id"] = InstanceIDModel.get_or_create_current_instance()[
0] # counter is at 0
# create controllers for app/store/buffer operations
data["mc"] = MorangoProfileController("facilitydata")
data["sc"] = BaseSyncClient(None, "host")
session = SyncSession.objects.create(
id=uuid.uuid4().hex,
profile="facilitydata",
last_activity_timestamp=timezone.now(),
)
data["sc"].current_transfer_session = TransferSession.objects.create(
id=uuid.uuid4().hex,
sync_session=session,
push=True,
last_activity_timestamp=timezone.now(),
)
data["mc"].serialize_into_store() # counter is at 1
# create group of facilities and first serialization
data["group1_c1"] = [FacilityFactory() for _ in range(5)]
data["mc"].serialize_into_store() # counter is at 2
# create group of facilities and second serialization
data["group1_c2"] = [FacilityFactory() for _ in range(5)]
# create users and logs associated with user
data["user1"] = MyUser.objects.create(username="******")
data["user1_sumlogs"] = [
SummaryLog.objects.create(user=data["user1"]) for _ in range(5)
]
data["mc"].serialize_into_store() # counter is at 3
# create new instance id and group of facilities
with EnvironmentVarGuard() as env:
env["MORANGO_SYSTEM_ID"] = "new_sys_id"
data["group2_id"] = InstanceIDModel.get_or_create_current_instance(
clear_cache=True)[0] # new counter is at 0
data["mc"].serialize_into_store() # new counter is at 1
data["group2_c1"] = [FacilityFactory() for _ in range(5)]
# create users and logs associated with user
data["user2"] = MyUser.objects.create(username="******")
data["user2_sumlogs"] = [
SummaryLog.objects.create(user=data["user2"]) for _ in range(5)
]
data["user2_interlogs"] = [
InteractionLog.objects.create(user=data["user2"]) for _ in range(5)
]
data["user3"] = MyUser.objects.create(username="******")
data["user3_sumlogs"] = [
SummaryLog.objects.create(user=data["user3"]) for _ in range(5)
]
data["user3_interlogs"] = [
InteractionLog.objects.create(user=data["user3"]) for _ in range(5)
]
data["mc"].serialize_into_store() # new counter is at 2
data["user4"] = MyUser.objects.create(
username="******", _morango_partition="badpartition")
data["mc"].serialize_into_store() # new counter is at 3
return data
def test_dequeue_into_store(self):
_dequeue_into_store(self.data["sc"].current_transfer_session)
# ensure a record with different transfer session id is not affected
self.assertTrue(
Buffer.objects.filter(
transfer_session_id=self.data["tfs_id"]).exists())
self.assertFalse(Store.objects.filter(id=self.data["model6"]).exists())
self.assertFalse(
RecordMaxCounter.objects.filter(
store_model_id=self.data["model6"],
instance_id__in=self.data["model6_rmcb_ids"],
).exists())
# ensure reverse fast forward records are not modified
self.assertNotEqual(
Store.objects.get(id=self.data["model1"]).serialized, "buffer")
self.assertFalse(
RecordMaxCounter.objects.filter(
instance_id=self.data["model1_rmcb_ids"][1]).exists())
# ensure records with merge conflicts are modified
self.assertEqual(
Store.objects.get(
id=self.data["model2"]).conflicting_serialized_data,
"buffer\nstore",
) # conflicting field is overwritten
self.assertEqual(
Store.objects.get(
id=self.data["model5"]).conflicting_serialized_data,
"buffer\nstore",
)
self.assertTrue(
RecordMaxCounter.objects.filter(
instance_id=self.data["model2_rmcb_ids"][1]).exists())
self.assertTrue(
RecordMaxCounter.objects.filter(
instance_id=self.data["model5_rmcb_ids"][1]).exists())
self.assertEqual(
Store.objects.get(id=self.data["model2"]).last_saved_instance,
InstanceIDModel.get_or_create_current_instance()[0].id,
)
self.assertEqual(
Store.objects.get(id=self.data["model5"]).last_saved_instance,
InstanceIDModel.get_or_create_current_instance()[0].id,
)
# ensure fast forward records are modified
self.assertEqual(
Store.objects.get(id=self.data["model3"]).serialized,
"buffer") # serialized field is overwritten
self.assertTrue(
RecordMaxCounter.objects.filter(
instance_id=self.data["model3_rmcb_ids"][1]).exists())
self.assertEqual(
Store.objects.get(id=self.data["model3"]).last_saved_instance,
self.data["model3_rmcb_ids"][1],
) # last_saved_by is updated
self.assertEqual(
RecordMaxCounter.objects.get(
instance_id=self.data["model3_rmcb_ids"][0],
store_model_id=self.data["model3"],
).counter,
3,
)
# ensure all buffer and rmcb records were deleted for this transfer session id
self.assertFalse(
Buffer.objects.filter(transfer_session_id=self.data["sc"].
current_transfer_session.id).exists())
self.assertFalse(
RecordMaxCounterBuffer.objects.filter(
transfer_session_id=self.data["sc"].current_transfer_session.id
).exists())
def setUp(self):
InstanceIDModel.get_or_create_current_instance()
[FacilityModelFactory() for _ in range(10)]
self.mc = MorangoProfileController('facilitydata')
self.mc.serialize_into_store()
