def test_locations_migration(self): checkpoint = MigrationCheckpoint( domain=TEST_DOMAIN, start_date=datetime.utcnow(), date=datetime.utcnow(), api='product', limit=100, offset=0 ) location_api = ApiSyncObject( 'location_facility', self.endpoint.get_locations, self.api_object.location_sync, filters=dict(type='facility') ) synchronization(location_api, checkpoint, None, 100, 0) self.assertEqual('location_facility', checkpoint.api) self.assertEqual(100, checkpoint.limit) self.assertEqual(0, checkpoint.offset) self.assertEqual(5, len(list(Location.by_domain(TEST_DOMAIN)))) self.assertEqual(5, SQLLocation.objects.filter(domain=TEST_DOMAIN).count()) sql_location = SQLLocation.objects.get(domain=TEST_DOMAIN, site_code='DM520053') self.assertEqual('FACILITY', sql_location.location_type.name) self.assertIsNotNone(sql_location.supply_point_id) sql_location2 = SQLLocation.objects.get(domain=TEST_DOMAIN, site_code='region-dodoma') self.assertEqual('REGION', sql_location2.location_type.name) self.assertIsNone(sql_location2.supply_point_id)
def test_webusers_migration(self): checkpoint = MigrationCheckpoint(domain=TEST_DOMAIN, start_date=datetime.utcnow(), date=datetime.utcnow(), api='product', limit=100, offset=0) location_api = ApiSyncObject('webuser', self.endpoint.get_webusers, self.api_object.web_user_sync) synchronization(location_api, checkpoint, None, 100, 0) self.assertEqual('webuser', checkpoint.api) self.assertEqual(100, checkpoint.limit) self.assertEqual(0, checkpoint.offset) self.assertEqual(5, len(list(WebUser.by_domain(TEST_DOMAIN))))
def bootstrap_domain(api_object, **kwargs): domain = api_object.domain endpoint = api_object.endpoint start_date = datetime.today() # get the last saved checkpoint from a prior migration and various config options try: checkpoint = MigrationCheckpoint.objects.get(domain=domain) api = checkpoint.api date = checkpoint.date limit = 100 offset = checkpoint.offset if not checkpoint.start_date: checkpoint.start_date = start_date checkpoint.save() else: start_date = checkpoint.start_date except MigrationCheckpoint.DoesNotExist: # bootstrap static domain data api_object.prepare_commtrack_config() checkpoint = MigrationCheckpoint() checkpoint.domain = domain checkpoint.start_date = start_date api = 'product' date = None limit = 100 offset = 0 api_object.set_default_backend() api_object.prepare_custom_fields() api_object.create_or_edit_roles() apis = api_object.apis try: apis_from_checkpoint = itertools.dropwhile(lambda x: x.name != api, apis) for api_object in apis_from_checkpoint: if date and api_object.migrate_once: continue api_object.add_date_filter(date) synchronization(api_object, checkpoint, date, limit, offset, **kwargs) limit = 100 offset = 0 save_checkpoint(checkpoint, 'product', 100, 0, checkpoint.start_date, False) checkpoint.start_date = None checkpoint.save() except ConnectionError as e: logging.error(e)
def bootstrap_domain(config, endpoint, extensions=None, **kwargs): domain = config.domain start_date = datetime.today() endpoint = endpoint.from_config(config) try: checkpoint = MigrationCheckpoint.objects.get(domain=domain) api = checkpoint.api date = checkpoint.date limit = checkpoint.limit offset = checkpoint.offset if not checkpoint.start_date: checkpoint.start_date = start_date checkpoint.save() else: start_date = checkpoint.start_date except MigrationCheckpoint.DoesNotExist: checkpoint = MigrationCheckpoint() checkpoint.domain = domain checkpoint.start_date = start_date api = 'product' date = None limit = 1000 offset = 0 apis = [ ('product', partial(products_sync, domain, endpoint, checkpoint, date=date, **kwargs)), ('location_facility', partial(locations_sync, domain, endpoint, checkpoint, date=date, filters=dict(date_updated__gte=date, type='facility'), **kwargs)), ('location_district', partial(locations_sync, domain, endpoint, checkpoint, date=date, filters=dict(date_updated__gte=date, type='district'), **kwargs)), ('location_region', partial(locations_sync, domain, endpoint, checkpoint, date=date, filters=dict(date_updated__gte=date, type='region'), **kwargs)), ('webuser', partial(webusers_sync, domain, endpoint, checkpoint, date=date, filters=dict(user__date_joined__gte=date))), ('smsuser', partial(smsusers_sync, domain, endpoint, checkpoint, date=date, filters=dict(date_updated__gte=date))) ] try: apis_from_checkpoint = itertools.dropwhile(lambda x: x[0] != api, apis) for api in apis_from_checkpoint: kwargs = dict(limit=limit, offset=offset) extension = extensions.get(api[0]) if extensions else None if extension: kwargs['extension'] = extension api[1](**kwargs) limit = 1000 offset = 0 save_checkpoint(checkpoint, 'product', 1000, 0, start_date, False) checkpoint.start_date = None checkpoint.save() except ConnectionError as e: logging.error(e)
def test_products_migration(self): checkpoint = MigrationCheckpoint( domain=TEST_DOMAIN, start_date=datetime.utcnow(), date=datetime.utcnow(), api='product', limit=100, offset=0 ) product_api = ApiSyncObject( 'product', self.endpoint.get_products, self.api_object.product_sync ) synchronization(product_api, checkpoint, None, 100, 0) self.assertEqual('product', checkpoint.api) self.assertEqual(100, checkpoint.limit) self.assertEqual(0, checkpoint.offset) self.assertEqual(6, len(list(Prod.by_domain(TEST_DOMAIN))))
def bootstrap_domain(api_object, **kwargs): domain = api_object.domain start_date = datetime.today() # get the last saved checkpoint from a prior migration and various config options try: checkpoint = MigrationCheckpoint.objects.get(domain=domain) api = checkpoint.api date = checkpoint.date limit = 100 offset = checkpoint.offset if not checkpoint.start_date: checkpoint.start_date = start_date checkpoint.save() else: start_date = checkpoint.start_date except MigrationCheckpoint.DoesNotExist: # bootstrap static domain data api_object.prepare_commtrack_config() checkpoint = MigrationCheckpoint() checkpoint.domain = domain checkpoint.start_date = start_date api = "product" date = None limit = 100 offset = 0 api_object.set_default_backend() api_object.prepare_custom_fields() api_object.create_or_edit_roles() apis = api_object.apis try: apis_from_checkpoint = itertools.dropwhile(lambda x: x.name != api, apis) for api in apis_from_checkpoint: if date and api.migrate_once: continue api.add_date_filter(date) synchronization(api, checkpoint, date, limit, offset, **kwargs) limit = 100 offset = 0 save_checkpoint(checkpoint, "product", 100, 0, checkpoint.start_date, False) checkpoint.start_date = None checkpoint.save() from custom.ilsgateway.tasks import balance_migration_task balance_migration_task.delay(api_object.domain, api_object.endpoint) except ConnectionError as e: logging.error(e)
def bootstrap_domain(api_object, **kwargs): domain = api_object.domain endpoint = api_object.endpoint start_date = datetime.today() try: checkpoint = MigrationCheckpoint.objects.get(domain=domain) api = checkpoint.api date = checkpoint.date limit = 100 offset = checkpoint.offset if not checkpoint.start_date: checkpoint.start_date = start_date checkpoint.save() else: start_date = checkpoint.start_date except MigrationCheckpoint.DoesNotExist: api_object.prepare_commtrack_config() checkpoint = MigrationCheckpoint() checkpoint.domain = domain checkpoint.start_date = start_date api = 'product' date = None limit = 100 offset = 0 api_object.prepare_custom_fields() synchronize_domain = partial(synchronization, checkpoint=checkpoint, date=date) apis = [ ('product', partial( synchronize_domain, get_objects_function=endpoint.get_products, sync_function=api_object.product_sync )), ('location_facility', partial( synchronize_domain, get_objects_function=endpoint.get_locations, sync_function=api_object.location_sync, fetch_groups=True, filters=dict(date_updated__gte=date, type='facility', is_active=True) )), ('location_district', partial( synchronize_domain, get_objects_function=endpoint.get_locations, sync_function=api_object.location_sync, fetch_groups=True, filters=dict(date_updated__gte=date, type='district', is_active=True) )), ('location_region', partial( synchronize_domain, get_objects_function=endpoint.get_locations, sync_function=api_object.location_sync, fetch_groups=True, filters=dict(date_updated__gte=date, type='region', is_active=True) )), ('webuser', partial( synchronize_domain, get_objects_function=endpoint.get_webusers, sync_function=api_object.web_user_sync, filters=dict(user__date_joined__gte=date) )), ('smsuser', partial( synchronize_domain, get_objects_function=endpoint.get_smsusers, sync_function=api_object.sms_user_sync, filters=dict(date_updated__gte=date) )) ] try: apis_from_checkpoint = itertools.dropwhile(lambda x: x[0] != api, apis) for (api_name, api_function) in apis_from_checkpoint: api_function(name=api_name, limit=limit, offset=offset, **kwargs) limit = 100 offset = 0 save_checkpoint(checkpoint, 'product', 100, 0, start_date, False) checkpoint.start_date = None checkpoint.save() except ConnectionError as e: logging.error(e)