def handle_async(self, *args, **options):
# call this in case user directly syncs without migrating database
if not ScopeDefinition.objects.filter():
call_command("loaddata", "scopedefinitions")
controller = MorangoProfileController('facilitydata')
with self.start_progress(total=7) as progress_update:
network_connection = controller.create_network_connection(options['base_url'])
progress_update(1)
options['dataset_id'] = self.get_dataset_id(options['base_url'], options['dataset_id'])
progress_update(1)
client_cert, server_cert = self.get_client_and_server_certs(options['username'], options['password'],
options['dataset_id'], network_connection)
progress_update(1)
sync_client = network_connection.create_sync_session(client_cert, server_cert)
progress_update(1)
# pull from server and push our own data to server
if not options['no_pull']:
sync_client.initiate_pull(Filter(options['dataset_id']))
if not options['no_push']:
sync_client.initiate_push(Filter(options['dataset_id']))
progress_update(1)
self.create_superuser_and_provision_device(options['username'], options['dataset_id'])
progress_update(1)
sync_client.close_sync_session()
progress_update(1)
def _queue_into_buffer(transfersession):
"""
Takes a chunk of data from the store to be put into the buffer to be sent to another morango instance.
"""
last_saved_by_conditions = []
filter_prefixes = Filter(transfersession.filter)
server_fsic = json.loads(transfersession.server_fsic)
client_fsic = json.loads(transfersession.client_fsic)
if transfersession.push:
fsics = _fsic_queuing_calc(client_fsic, server_fsic)
else:
fsics = _fsic_queuing_calc(server_fsic, client_fsic)
# if fsics are identical or receiving end has newer data, then there is nothing to queue
if not fsics:
return
# create condition for all push FSICs where instance_ids are equal, but internal counters are higher than FSICs counters
for instance, counter in six.iteritems(fsics):
last_saved_by_conditions += ["(last_saved_instance = '{0}' AND last_saved_counter > {1})".format(instance, counter)]
if fsics:
last_saved_by_conditions = [_join_with_logical_operator(last_saved_by_conditions, 'OR')]
partition_conditions = []
# create condition for filtering by partitions
for prefix in filter_prefixes:
partition_conditions += ["partition LIKE '{}%'".format(prefix)]
if filter_prefixes:
partition_conditions = [_join_with_logical_operator(partition_conditions, 'OR')]
# combine conditions
fsic_and_partition_conditions = _join_with_logical_operator(last_saved_by_conditions + partition_conditions, 'AND')
# filter by profile
where_condition = _join_with_logical_operator([fsic_and_partition_conditions, "profile = '{}'".format(transfersession.sync_session.profile)], 'AND')
# execute raw sql to take all records that match condition, to be put into buffer for transfer
with connection.cursor() as cursor:
queue_buffer = """INSERT INTO {outgoing_buffer}
(model_uuid, serialized, deleted, last_saved_instance, last_saved_counter, hard_deleted,
model_name, profile, partition, source_id, conflicting_serialized_data, transfer_session_id, _self_ref_fk)
SELECT id, serialized, deleted, last_saved_instance, last_saved_counter, hard_deleted, model_name, profile, partition, source_id, conflicting_serialized_data, '{transfer_session_id}', _self_ref_fk
FROM {store} WHERE {condition}""".format(outgoing_buffer=Buffer._meta.db_table,
transfer_session_id=transfersession.id,
condition=where_condition,
store=Store._meta.db_table)
cursor.execute(queue_buffer)
# take all record max counters that are foreign keyed onto store models, which were queued into the buffer
queue_rmc_buffer = """INSERT INTO {outgoing_rmcb}
(instance_id, counter, transfer_session_id, model_uuid)
SELECT instance_id, counter, '{transfer_session_id}', store_model_id
FROM {record_max_counter} AS rmc
INNER JOIN {outgoing_buffer} AS buffer ON rmc.store_model_id = buffer.model_uuid
WHERE buffer.transfer_session_id = '{transfer_session_id}'
""".format(outgoing_rmcb=RecordMaxCounterBuffer._meta.db_table,
transfer_session_id=transfersession.id,
record_max_counter=RecordMaxCounter._meta.db_table,
outgoing_buffer=Buffer._meta.db_table)
cursor.execute(queue_rmc_buffer)
def test_filtered_serialization_single_filter(self):
fac = FacilityModelFactory()
user = MyUser.objects.create(username='******')
log = SummaryLog.objects.create(user=user)
self.mc.serialize_into_store(filter=Filter(user._morango_partition))
self.assertFalse(Store.objects.filter(id=fac.id).exists())
self.assertTrue(Store.objects.filter(id=user.id).exists())
self.assertTrue(Store.objects.filter(id=log.id).exists())
def _create_transfer_session(self, push, filter):
# build data for creating transfer session on server side
data = {
'id': uuid.uuid4().hex,
'filter': str(filter),
'push': push,
'sync_session_id': self.sync_session.id,
}
data['last_activity_timestamp'] = timezone.now()
self.current_transfer_session = TransferSession.objects.create(**data)
data.pop('last_activity_timestamp')
if push:
# before pushing, we want to serialize the most recent data and update database max counters
if getattr(settings, 'MORANGO_SERIALIZE_BEFORE_QUEUING', True):
_serialize_into_store(
self.current_transfer_session.sync_session.profile,
filter=Filter(self.current_transfer_session.filter))
data['client_fsic'] = json.dumps(
DatabaseMaxCounter.calculate_filter_max_counters(filter))
self.current_transfer_session.client_fsic = data['client_fsic']
# save transfersession locally before creating transfersession server side
self.current_transfer_session.save()
# create transfer session on server side
transfer_resp = self.sync_connection._create_transfer_session(data)
self.current_transfer_session.server_fsic = transfer_resp.json().get(
'server_fsic') or '{}'
if not push:
self.current_transfer_session.records_total = transfer_resp.json(
).get('records_total')
self.current_transfer_session.save()
def handle_async(self, *args, **options):
baseurl = options["baseurl"]
facility_id = options["facility"]
chunk_size = options["chunk_size"]
# This handles the case for when we want to pull in facility data for our empty kolibri instance
if not facility_id:
self._fullfacilitysync(baseurl)
facility = get_facility(
facility_id=facility_id, noninteractive=options["noninteractive"]
)
# if url is not pointing to portal server, do P2P syncing
self._fullfacilitysync(baseurl, facility=facility, chunk_size=chunk_size)
# data portal syncing
self.stdout.write("Syncing has been initiated (this may take a while)...")
controller = MorangoProfileController("facilitydata")
with self.start_progress(total=5) as progress_update:
try:
network_connection = controller.create_network_connection(baseurl)
except ConnectionError as e:
raise CommandError(e)
progress_update(1)
# get client certificate
client_cert = (
facility.dataset.get_owned_certificates()
.filter(scope_definition_id=FULL_FACILITY)
.first()
)
if not client_cert:
raise CommandError(
"This device does not own a certificate for Facility: {}".format(
facility.name
)
)
# push certificate up to portal server
scope_params = json.loads(client_cert.scope_params)
server_cert = network_connection.push_signed_client_certificate_chain(
local_parent_cert=client_cert,
scope_definition_id=FULL_FACILITY,
scope_params=scope_params,
)
progress_update(1)
# we should now be able to push our facility data
sync_client = network_connection.create_sync_session(
client_cert, server_cert, chunk_size=chunk_size
)
progress_update(1)
sync_client.initiate_push(Filter(scope_params["dataset_id"]))
progress_update(1)
sync_client.close_sync_session()
progress_update(1)
self.stdout.write("Syncing has been completed.")
def handle_async(self, *args, **options):
# validate url that is passed in
try:
URLValidator()((options['base_url']))
except ValidationError:
print(
'Base-url is not valid. Please retry command and enter a valid url.'
)
sys.exit(1)
# call this in case user directly syncs without migrating database
if not ScopeDefinition.objects.filter():
call_command("loaddata", "scopedefinitions")
# ping server at url with info request
info_url = urljoin(options['base_url'],
'api/morango/v1/morangoinfo/1/')
try:
info_resp = requests.get(info_url)
except ConnectionError:
print('Can not connect to server with base-url: {}'.format(
options['base_url']))
sys.exit(1)
# if instance_ids are equal, this means device is trying to sync with itself, which we don't allow
if InstanceIDModel.get_or_create_current_instance(
)[0].id == info_resp.json()['instance_id']:
print(
'Device can not sync with itself. Please re-check base-url and try again.'
)
sys.exit(1)
controller = MorangoProfileController('facilitydata')
with self.start_progress(total=7) as progress_update:
network_connection = controller.create_network_connection(
options['base_url'])
progress_update(1)
options['dataset_id'] = self.get_dataset_id(
options['base_url'], options['dataset_id'])
progress_update(1)
client_cert, server_cert, options[
'username'] = self.get_client_and_server_certs(
options['username'], options['password'],
options['dataset_id'], network_connection)
progress_update(1)
sync_client = network_connection.create_sync_session(
client_cert, server_cert)
progress_update(1)
# pull from server and push our own data to server
if not options['no_pull']:
sync_client.initiate_pull(Filter(options['dataset_id']))
if not options['no_push']:
sync_client.initiate_push(Filter(options['dataset_id']))
progress_update(1)
self.create_superuser_and_provision_device(options['username'],
options['dataset_id'])
progress_update(1)
sync_client.close_sync_session()
progress_update(1)
def handle_async(self, *args, **options):
# validate url that is passed in
try:
URLValidator()((options["base_url"]))
except ValidationError:
raise CommandError(
"Base URL is not valid. Please retry command and enter a valid URL."
)
# call this in case user directly syncs without migrating database
if not ScopeDefinition.objects.filter():
call_command("loaddata", "scopedefinitions")
controller = MorangoProfileController("facilitydata")
with self.start_progress(total=7) as progress_update:
try:
network_connection = controller.create_network_connection(
options["base_url"])
except ConnectionError:
raise CommandError(
"Can not connect to server with base URL: {}".format(
options["base_url"]))
# if instance_ids are equal, this means device is trying to sync with itself, which we don't allow
if (InstanceIDModel.get_or_create_current_instance()[0].id ==
network_connection.server_info["instance_id"]):
raise CommandError(
"Device can not sync with itself. Please recheck base URL and try again."
)
progress_update(1)
options["dataset_id"] = self.get_dataset_id(
options["base_url"], options["dataset_id"])
progress_update(1)
client_cert, server_cert, options[
"username"] = self.get_client_and_server_certs(
options["username"],
options["password"],
options["dataset_id"],
network_connection,
)
progress_update(1)
sync_client = network_connection.create_sync_session(
client_cert, server_cert, chunk_size=options["chunk_size"])
progress_update(1)
# pull from server and push our own data to server
if not options["no_pull"]:
sync_client.initiate_pull(Filter(options["dataset_id"]))
if not options["no_push"]:
sync_client.initiate_push(Filter(options["dataset_id"]))
progress_update(1)
self.create_superuser_and_provision_device(options["username"],
options["dataset_id"])
progress_update(1)
sync_client.close_sync_session()
progress_update(1)