def test_locations_migration(self):
checkpoint = MigrationCheckpoint(
domain=TEST_DOMAIN,
start_date=datetime.utcnow(),
date=datetime.utcnow(),
api='product',
limit=100,
offset=0
)
location_api = ApiSyncObject(
'location_facility',
self.endpoint.get_locations,
self.api_object.location_sync,
filters=dict(type='facility')
)
synchronization(location_api, checkpoint, None, 100, 0)
self.assertEqual('location_facility', checkpoint.api)
self.assertEqual(100, checkpoint.limit)
self.assertEqual(0, checkpoint.offset)
self.assertEqual(5, len(list(Location.by_domain(TEST_DOMAIN))))
self.assertEqual(5, SQLLocation.objects.filter(domain=TEST_DOMAIN).count())
sql_location = SQLLocation.objects.get(domain=TEST_DOMAIN, site_code='DM520053')
self.assertEqual('FACILITY', sql_location.location_type.name)
self.assertIsNotNone(sql_location.supply_point_id)
sql_location2 = SQLLocation.objects.get(domain=TEST_DOMAIN, site_code='region-dodoma')
self.assertEqual('REGION', sql_location2.location_type.name)
self.assertIsNone(sql_location2.supply_point_id)
def test_locations_migration(self):
checkpoint = MigrationCheckpoint(
domain=TEST_DOMAIN,
start_date=datetime.utcnow(),
date=datetime.utcnow(),
api='product',
limit=100,
offset=0
)
location_api = ApiSyncObject(
'location_facility',
self.endpoint.get_locations,
self.api_object.location_sync,
filters=dict(type='facility')
)
synchronization(location_api, checkpoint, None, 100, 0)
self.assertEqual('location_facility', checkpoint.api)
self.assertEqual(100, checkpoint.limit)
self.assertEqual(0, checkpoint.offset)
self.assertEqual(6, len(list(Location.by_domain(TEST_DOMAIN))))
self.assertEqual(6, SQLLocation.objects.filter(domain=TEST_DOMAIN).count())
sql_location = SQLLocation.objects.get(domain=TEST_DOMAIN, site_code='DM520053')
self.assertEqual('FACILITY', sql_location.location_type.name)
self.assertIsNotNone(sql_location.supply_point_id)
sql_location2 = SQLLocation.objects.get(domain=TEST_DOMAIN, site_code='region-dodoma')
self.assertEqual('REGION', sql_location2.location_type.name)
self.assertIsNone(sql_location2.supply_point_id)
def stock_data_task(api_object):
# checkpoint logic
start_date = datetime.today()
default_api = api_object.apis[0][0]
checkpoint, _ = StockDataCheckpoint.objects.get_or_create(
domain=api_object.domain,
defaults={
"api": default_api,
"date": None,
"limit": 1000,
"offset": 0,
"location": None,
"start_date": start_date,
},
)
if not checkpoint.api:
checkpoint.api = default_api
if not checkpoint.start_date:
checkpoint.start_date = start_date
checkpoint.save()
if not api_object.all_stock_data:
facilities = api_object.test_facilities
else:
facilities = api_object.get_ids()
if checkpoint.location:
external_id = api_object.get_last_processed_location(checkpoint)
if external_id:
facilities = list(itertools.dropwhile(lambda x: int(x) != int(external_id), facilities))
process_facility_task(api_object, facilities[0], start_from=checkpoint.api)
facilities = facilities[1:]
if not checkpoint.date or checkpoint.location:
# use subtasks only during initial migration
facilities_chunked_list = chunked(facilities, 5)
for chunk in facilities_chunked_list:
api_object.process_data(process_facility_task, chunk)
else:
offset = checkpoint.offset
for stock_api in itertools.dropwhile(lambda x: x.name != checkpoint.api, api_object.get_stock_apis_objects()):
stock_api.add_date_filter(checkpoint.date, checkpoint.start_date)
synchronization(
stock_api,
checkpoint,
checkpoint.date,
1000,
offset,
params={"domain": api_object.domain},
domain=api_object.domain,
atomic=True,
)
offset = 0
checkpoint = StockDataCheckpoint.objects.get(domain=api_object.domain)
save_stock_data_checkpoint(checkpoint, default_api, 1000, 0, checkpoint.start_date, None, False)
checkpoint.start_date = None
checkpoint.save()
def stock_data_task(api_object):
# checkpoint logic
start_date = datetime.today()
default_api = api_object.apis[0][0]
checkpoint, _ = StockDataCheckpoint.objects.get_or_create(domain=api_object.domain, defaults={
'api': default_api,
'date': None,
'limit': 1000,
'offset': 0,
'location': None,
'start_date': start_date
})
if not checkpoint.api:
checkpoint.api = default_api
if not checkpoint.start_date:
checkpoint.start_date = start_date
checkpoint.save()
if not api_object.all_stock_data:
facilities = api_object.test_facilities
else:
facilities = api_object.get_ids()
if checkpoint.location:
external_id = api_object.get_last_processed_location(checkpoint)
if external_id:
facilities = list(itertools.dropwhile(lambda x: int(x) != int(external_id), facilities))
process_facility_task(api_object, facilities[0], start_from=checkpoint.api)
facilities = facilities[1:]
if not checkpoint.date or checkpoint.location:
# use subtasks only during initial migration
facilities_chunked_list = chunked(facilities, 5)
for chunk in facilities_chunked_list:
api_object.process_data(process_facility_task, chunk)
else:
offset = checkpoint.offset
for stock_api in itertools.dropwhile(
lambda x: x.name != checkpoint.api, api_object.get_stock_apis_objects()
):
stock_api.add_date_filter(checkpoint.date, checkpoint.start_date)
synchronization(
stock_api,
checkpoint,
checkpoint.date,
1000,
offset,
params={'domain': api_object.domain},
domain=api_object.domain
)
offset = 0
checkpoint = StockDataCheckpoint.objects.get(domain=api_object.domain)
save_stock_data_checkpoint(checkpoint, default_api, 1000, 0, checkpoint.start_date, None, False)
checkpoint.start_date = None
checkpoint.save()
def test_webusers_migration(self):
checkpoint = MigrationCheckpoint(
domain=TEST_DOMAIN, start_date=datetime.utcnow(), date=datetime.utcnow(), api="product", limit=100, offset=0
)
location_api = ApiSyncObject("webuser", self.endpoint.get_webusers, self.api_object.web_user_sync)
synchronization(location_api, checkpoint, None, 100, 0)
self.assertEqual("webuser", checkpoint.api)
self.assertEqual(100, checkpoint.limit)
self.assertEqual(0, checkpoint.offset)
self.assertEqual(5, len(list(WebUser.by_domain(TEST_DOMAIN))))
def test_products_migration(self):
checkpoint = MigrationCheckpoint(
domain=TEST_DOMAIN, start_date=datetime.utcnow(), date=datetime.utcnow(), api="product", limit=100, offset=0
)
product_api = ApiSyncObject("product", self.endpoint.get_products, self.api_object.product_sync)
synchronization(product_api, checkpoint, None, 100, 0)
self.assertEqual("product", checkpoint.api)
self.assertEqual(100, checkpoint.limit)
self.assertEqual(0, checkpoint.offset)
self.assertEqual(6, len(list(Prod.by_domain(TEST_DOMAIN))))
def test_webusers_migration(self):
checkpoint = MigrationCheckpoint(domain=TEST_DOMAIN,
start_date=datetime.utcnow(),
date=datetime.utcnow(),
api='product',
limit=100,
offset=0)
location_api = ApiSyncObject('webuser', self.endpoint.get_webusers,
self.api_object.web_user_sync)
synchronization(location_api, checkpoint, None, 100, 0)
self.assertEqual('webuser', checkpoint.api)
self.assertEqual(100, checkpoint.limit)
self.assertEqual(0, checkpoint.offset)
self.assertEqual(5, len(list(WebUser.by_domain(TEST_DOMAIN))))
def test_locations_migration(self):
checkpoint = MigrationCheckpoint(
domain=TEST_DOMAIN,
start_date=datetime.now(),
date=datetime.now(),
api='product',
limit=100,
offset=0
)
synchronization('location_facility',
self.endpoint.get_locations,
self.api_object.location_sync, checkpoint, None, 100, 0, filters=dict(type='facility'))
self.assertEqual('location_facility', checkpoint.api)
self.assertEqual(100, checkpoint.limit)
self.assertEqual(0, checkpoint.offset)
self.assertEqual(4, len(list(Location.by_domain(TEST_DOMAIN))))
def test_smsusers_migration(self):
checkpoint = MigrationCheckpoint(
domain=TEST_DOMAIN,
start_date=datetime.now(),
date=datetime.now(),
api='product',
limit=100,
offset=0
)
synchronization('smsuser',
self.endpoint.get_smsusers,
self.api_object.sms_user_sync, checkpoint, None, 100, 0)
self.assertEqual('smsuser', checkpoint.api)
self.assertEqual(100, checkpoint.limit)
self.assertEqual(0, checkpoint.offset)
self.assertEqual(6, len(CommCareUser.by_domain(TEST_DOMAIN)))
def test_locations_migration(self):
checkpoint = MigrationCheckpoint(
domain=TEST_DOMAIN,
start_date=datetime.now(),
date=datetime.now(),
api='product',
limit=100,
offset=0
)
synchronization('product',
self.endpoint.get_products,
self.api_object.product_sync, checkpoint, None, 100, 0)
self.assertEqual('product', checkpoint.api)
self.assertEqual(100, checkpoint.limit)
self.assertEqual(0, checkpoint.offset)
self.assertEqual(6, len(list(Prod.by_domain(TEST_DOMAIN))))
def test_products_migration(self):
checkpoint = MigrationCheckpoint(
domain=TEST_DOMAIN,
start_date=datetime.utcnow(),
date=datetime.utcnow(),
api='product',
limit=100,
offset=0
)
product_api = ApiSyncObject(
'product',
self.endpoint.get_products,
self.api_object.product_sync
)
synchronization(product_api, checkpoint, None, 100, 0)
self.assertEqual('product', checkpoint.api)
self.assertEqual(100, checkpoint.limit)
self.assertEqual(0, checkpoint.offset)
self.assertEqual(6, len(list(Prod.by_domain(TEST_DOMAIN))))
def add_products_to_loc(api):
endpoint = api.endpoint
synchronization(None, endpoint.get_locations, api.location_sync, None, None, 100, 0)
def sms_users_fix(api):
endpoint = api.endpoint
synchronization(None, endpoint.get_smsusers, api.add_language_to_user, None, None, 100, 0)
def add_products_to_loc(api):
endpoint = api.endpoint
synchronization(None, endpoint.get_locations, api.location_sync, None, None, 100, 0,
filters={"is_active": True})
def resync_web_users(api_object):
web_users_sync = api_object.apis[4]
synchronization(web_users_sync, None, None, 100, 0)
def add_products_to_loc(api):
endpoint = api.endpoint
synchronization(None, endpoint.get_locations, api.location_sync, None, None, 100, 0,
filters={"is_active": True})
def sms_users_fix(api):
endpoint = api.endpoint
api.set_default_backend()
synchronization(None, endpoint.get_smsusers, partial(api.add_language_to_user),
None, None, 100, 0)
def resync_web_users(api_object):
web_users_sync = api_object.apis[5]
synchronization(web_users_sync, None, None, 100, 0)