def test_locations_migration(self):
checkpoint = MigrationCheckpoint(
domain=TEST_DOMAIN,
start_date=datetime.utcnow(),
date=datetime.utcnow(),
api='product',
limit=100,
offset=0
)
location_api = ApiSyncObject(
'location_facility',
self.endpoint.get_locations,
self.api_object.location_sync,
filters=dict(type='facility')
)
synchronization(location_api, checkpoint, None, 100, 0)
self.assertEqual('location_facility', checkpoint.api)
self.assertEqual(100, checkpoint.limit)
self.assertEqual(0, checkpoint.offset)
self.assertEqual(5, len(list(Location.by_domain(TEST_DOMAIN))))
self.assertEqual(5, SQLLocation.objects.filter(domain=TEST_DOMAIN).count())
sql_location = SQLLocation.objects.get(domain=TEST_DOMAIN, site_code='DM520053')
self.assertEqual('FACILITY', sql_location.location_type.name)
self.assertIsNotNone(sql_location.supply_point_id)
sql_location2 = SQLLocation.objects.get(domain=TEST_DOMAIN, site_code='region-dodoma')
self.assertEqual('REGION', sql_location2.location_type.name)
self.assertIsNone(sql_location2.supply_point_id)
def test_webusers_migration(self):
checkpoint = MigrationCheckpoint(domain=TEST_DOMAIN,
start_date=datetime.utcnow(),
date=datetime.utcnow(),
api='product',
limit=100,
offset=0)
location_api = ApiSyncObject('webuser', self.endpoint.get_webusers,
self.api_object.web_user_sync)
synchronization(location_api, checkpoint, None, 100, 0)
self.assertEqual('webuser', checkpoint.api)
self.assertEqual(100, checkpoint.limit)
self.assertEqual(0, checkpoint.offset)
self.assertEqual(5, len(list(WebUser.by_domain(TEST_DOMAIN))))
def bootstrap_domain(api_object, **kwargs):
domain = api_object.domain
endpoint = api_object.endpoint
start_date = datetime.today()
# get the last saved checkpoint from a prior migration and various config options
try:
checkpoint = MigrationCheckpoint.objects.get(domain=domain)
api = checkpoint.api
date = checkpoint.date
limit = 100
offset = checkpoint.offset
if not checkpoint.start_date:
checkpoint.start_date = start_date
checkpoint.save()
else:
start_date = checkpoint.start_date
except MigrationCheckpoint.DoesNotExist:
# bootstrap static domain data
api_object.prepare_commtrack_config()
checkpoint = MigrationCheckpoint()
checkpoint.domain = domain
checkpoint.start_date = start_date
api = 'product'
date = None
limit = 100
offset = 0
api_object.set_default_backend()
api_object.prepare_custom_fields()
api_object.create_or_edit_roles()
apis = api_object.apis
try:
apis_from_checkpoint = itertools.dropwhile(lambda x: x.name != api,
apis)
for api_object in apis_from_checkpoint:
if date and api_object.migrate_once:
continue
api_object.add_date_filter(date)
synchronization(api_object, checkpoint, date, limit, offset,
**kwargs)
limit = 100
offset = 0
save_checkpoint(checkpoint, 'product', 100, 0, checkpoint.start_date,
False)
checkpoint.start_date = None
checkpoint.save()
except ConnectionError as e:
logging.error(e)
def bootstrap_domain(config, endpoint, extensions=None, **kwargs):
domain = config.domain
start_date = datetime.today()
endpoint = endpoint.from_config(config)
try:
checkpoint = MigrationCheckpoint.objects.get(domain=domain)
api = checkpoint.api
date = checkpoint.date
limit = checkpoint.limit
offset = checkpoint.offset
if not checkpoint.start_date:
checkpoint.start_date = start_date
checkpoint.save()
else:
start_date = checkpoint.start_date
except MigrationCheckpoint.DoesNotExist:
checkpoint = MigrationCheckpoint()
checkpoint.domain = domain
checkpoint.start_date = start_date
api = 'product'
date = None
limit = 1000
offset = 0
apis = [
('product', partial(products_sync, domain, endpoint, checkpoint, date=date, **kwargs)),
('location_facility', partial(locations_sync, domain, endpoint, checkpoint, date=date,
filters=dict(date_updated__gte=date, type='facility'), **kwargs)),
('location_district', partial(locations_sync, domain, endpoint, checkpoint, date=date,
filters=dict(date_updated__gte=date, type='district'), **kwargs)),
('location_region', partial(locations_sync, domain, endpoint, checkpoint, date=date,
filters=dict(date_updated__gte=date, type='region'), **kwargs)),
('webuser', partial(webusers_sync, domain, endpoint, checkpoint, date=date,
filters=dict(user__date_joined__gte=date))),
('smsuser', partial(smsusers_sync, domain, endpoint, checkpoint, date=date,
filters=dict(date_updated__gte=date)))
]
try:
apis_from_checkpoint = itertools.dropwhile(lambda x: x[0] != api, apis)
for api in apis_from_checkpoint:
kwargs = dict(limit=limit, offset=offset)
extension = extensions.get(api[0]) if extensions else None
if extension:
kwargs['extension'] = extension
api[1](**kwargs)
limit = 1000
offset = 0
save_checkpoint(checkpoint, 'product', 1000, 0, start_date, False)
checkpoint.start_date = None
checkpoint.save()
except ConnectionError as e:
logging.error(e)
def test_products_migration(self):
checkpoint = MigrationCheckpoint(
domain=TEST_DOMAIN,
start_date=datetime.utcnow(),
date=datetime.utcnow(),
api='product',
limit=100,
offset=0
)
product_api = ApiSyncObject(
'product',
self.endpoint.get_products,
self.api_object.product_sync
)
synchronization(product_api, checkpoint, None, 100, 0)
self.assertEqual('product', checkpoint.api)
self.assertEqual(100, checkpoint.limit)
self.assertEqual(0, checkpoint.offset)
self.assertEqual(6, len(list(Prod.by_domain(TEST_DOMAIN))))
def bootstrap_domain(api_object, **kwargs):
domain = api_object.domain
start_date = datetime.today()
# get the last saved checkpoint from a prior migration and various config options
try:
checkpoint = MigrationCheckpoint.objects.get(domain=domain)
api = checkpoint.api
date = checkpoint.date
limit = 100
offset = checkpoint.offset
if not checkpoint.start_date:
checkpoint.start_date = start_date
checkpoint.save()
else:
start_date = checkpoint.start_date
except MigrationCheckpoint.DoesNotExist:
# bootstrap static domain data
api_object.prepare_commtrack_config()
checkpoint = MigrationCheckpoint()
checkpoint.domain = domain
checkpoint.start_date = start_date
api = "product"
date = None
limit = 100
offset = 0
api_object.set_default_backend()
api_object.prepare_custom_fields()
api_object.create_or_edit_roles()
apis = api_object.apis
try:
apis_from_checkpoint = itertools.dropwhile(lambda x: x.name != api, apis)
for api in apis_from_checkpoint:
if date and api.migrate_once:
continue
api.add_date_filter(date)
synchronization(api, checkpoint, date, limit, offset, **kwargs)
limit = 100
offset = 0
save_checkpoint(checkpoint, "product", 100, 0, checkpoint.start_date, False)
checkpoint.start_date = None
checkpoint.save()
from custom.ilsgateway.tasks import balance_migration_task
balance_migration_task.delay(api_object.domain, api_object.endpoint)
except ConnectionError as e:
logging.error(e)
def bootstrap_domain(api_object, **kwargs):
domain = api_object.domain
endpoint = api_object.endpoint
start_date = datetime.today()
try:
checkpoint = MigrationCheckpoint.objects.get(domain=domain)
api = checkpoint.api
date = checkpoint.date
limit = 100
offset = checkpoint.offset
if not checkpoint.start_date:
checkpoint.start_date = start_date
checkpoint.save()
else:
start_date = checkpoint.start_date
except MigrationCheckpoint.DoesNotExist:
api_object.prepare_commtrack_config()
checkpoint = MigrationCheckpoint()
checkpoint.domain = domain
checkpoint.start_date = start_date
api = 'product'
date = None
limit = 100
offset = 0
api_object.prepare_custom_fields()
synchronize_domain = partial(synchronization, checkpoint=checkpoint, date=date)
apis = [
('product', partial(
synchronize_domain,
get_objects_function=endpoint.get_products,
sync_function=api_object.product_sync
)),
('location_facility', partial(
synchronize_domain,
get_objects_function=endpoint.get_locations,
sync_function=api_object.location_sync,
fetch_groups=True,
filters=dict(date_updated__gte=date, type='facility', is_active=True)
)),
('location_district', partial(
synchronize_domain,
get_objects_function=endpoint.get_locations,
sync_function=api_object.location_sync,
fetch_groups=True,
filters=dict(date_updated__gte=date, type='district', is_active=True)
)),
('location_region', partial(
synchronize_domain,
get_objects_function=endpoint.get_locations,
sync_function=api_object.location_sync,
fetch_groups=True,
filters=dict(date_updated__gte=date, type='region', is_active=True)
)),
('webuser', partial(
synchronize_domain,
get_objects_function=endpoint.get_webusers,
sync_function=api_object.web_user_sync,
filters=dict(user__date_joined__gte=date)
)),
('smsuser', partial(
synchronize_domain,
get_objects_function=endpoint.get_smsusers,
sync_function=api_object.sms_user_sync,
filters=dict(date_updated__gte=date)
))
]
try:
apis_from_checkpoint = itertools.dropwhile(lambda x: x[0] != api, apis)
for (api_name, api_function) in apis_from_checkpoint:
api_function(name=api_name, limit=limit, offset=offset, **kwargs)
limit = 100
offset = 0
save_checkpoint(checkpoint, 'product', 100, 0, start_date, False)
checkpoint.start_date = None
checkpoint.save()
except ConnectionError as e:
logging.error(e)
