def test_locations_migration(self): checkpoint = MigrationCheckpoint( domain=TEST_DOMAIN, start_date=datetime.utcnow(), date=datetime.utcnow(), api='product', limit=100, offset=0 ) location_api = ApiSyncObject( 'location_facility', self.endpoint.get_locations, self.api_object.location_sync, filters=dict(type='facility') ) synchronization(location_api, checkpoint, None, 100, 0) self.assertEqual('location_facility', checkpoint.api) self.assertEqual(100, checkpoint.limit) self.assertEqual(0, checkpoint.offset) self.assertEqual(5, len(list(Location.by_domain(TEST_DOMAIN)))) self.assertEqual(5, SQLLocation.objects.filter(domain=TEST_DOMAIN).count()) sql_location = SQLLocation.objects.get(domain=TEST_DOMAIN, site_code='DM520053') self.assertEqual('FACILITY', sql_location.location_type.name) self.assertIsNotNone(sql_location.supply_point_id) sql_location2 = SQLLocation.objects.get(domain=TEST_DOMAIN, site_code='region-dodoma') self.assertEqual('REGION', sql_location2.location_type.name) self.assertIsNone(sql_location2.supply_point_id)
def test_locations_migration(self): checkpoint = MigrationCheckpoint( domain=TEST_DOMAIN, start_date=datetime.utcnow(), date=datetime.utcnow(), api='product', limit=100, offset=0 ) location_api = ApiSyncObject( 'location_facility', self.endpoint.get_locations, self.api_object.location_sync, filters=dict(type='facility') ) synchronization(location_api, checkpoint, None, 100, 0) self.assertEqual('location_facility', checkpoint.api) self.assertEqual(100, checkpoint.limit) self.assertEqual(0, checkpoint.offset) self.assertEqual(6, len(list(Location.by_domain(TEST_DOMAIN)))) self.assertEqual(6, SQLLocation.objects.filter(domain=TEST_DOMAIN).count()) sql_location = SQLLocation.objects.get(domain=TEST_DOMAIN, site_code='DM520053') self.assertEqual('FACILITY', sql_location.location_type.name) self.assertIsNotNone(sql_location.supply_point_id) sql_location2 = SQLLocation.objects.get(domain=TEST_DOMAIN, site_code='region-dodoma') self.assertEqual('REGION', sql_location2.location_type.name) self.assertIsNone(sql_location2.supply_point_id)
def stock_data_task(api_object): # checkpoint logic start_date = datetime.today() default_api = api_object.apis[0][0] checkpoint, _ = StockDataCheckpoint.objects.get_or_create( domain=api_object.domain, defaults={ "api": default_api, "date": None, "limit": 1000, "offset": 0, "location": None, "start_date": start_date, }, ) if not checkpoint.api: checkpoint.api = default_api if not checkpoint.start_date: checkpoint.start_date = start_date checkpoint.save() if not api_object.all_stock_data: facilities = api_object.test_facilities else: facilities = api_object.get_ids() if checkpoint.location: external_id = api_object.get_last_processed_location(checkpoint) if external_id: facilities = list(itertools.dropwhile(lambda x: int(x) != int(external_id), facilities)) process_facility_task(api_object, facilities[0], start_from=checkpoint.api) facilities = facilities[1:] if not checkpoint.date or checkpoint.location: # use subtasks only during initial migration facilities_chunked_list = chunked(facilities, 5) for chunk in facilities_chunked_list: api_object.process_data(process_facility_task, chunk) else: offset = checkpoint.offset for stock_api in itertools.dropwhile(lambda x: x.name != checkpoint.api, api_object.get_stock_apis_objects()): stock_api.add_date_filter(checkpoint.date, checkpoint.start_date) synchronization( stock_api, checkpoint, checkpoint.date, 1000, offset, params={"domain": api_object.domain}, domain=api_object.domain, atomic=True, ) offset = 0 checkpoint = StockDataCheckpoint.objects.get(domain=api_object.domain) save_stock_data_checkpoint(checkpoint, default_api, 1000, 0, checkpoint.start_date, None, False) checkpoint.start_date = None checkpoint.save()
def stock_data_task(api_object): # checkpoint logic start_date = datetime.today() default_api = api_object.apis[0][0] checkpoint, _ = StockDataCheckpoint.objects.get_or_create(domain=api_object.domain, defaults={ 'api': default_api, 'date': None, 'limit': 1000, 'offset': 0, 'location': None, 'start_date': start_date }) if not checkpoint.api: checkpoint.api = default_api if not checkpoint.start_date: checkpoint.start_date = start_date checkpoint.save() if not api_object.all_stock_data: facilities = api_object.test_facilities else: facilities = api_object.get_ids() if checkpoint.location: external_id = api_object.get_last_processed_location(checkpoint) if external_id: facilities = list(itertools.dropwhile(lambda x: int(x) != int(external_id), facilities)) process_facility_task(api_object, facilities[0], start_from=checkpoint.api) facilities = facilities[1:] if not checkpoint.date or checkpoint.location: # use subtasks only during initial migration facilities_chunked_list = chunked(facilities, 5) for chunk in facilities_chunked_list: api_object.process_data(process_facility_task, chunk) else: offset = checkpoint.offset for stock_api in itertools.dropwhile( lambda x: x.name != checkpoint.api, api_object.get_stock_apis_objects() ): stock_api.add_date_filter(checkpoint.date, checkpoint.start_date) synchronization( stock_api, checkpoint, checkpoint.date, 1000, offset, params={'domain': api_object.domain}, domain=api_object.domain ) offset = 0 checkpoint = StockDataCheckpoint.objects.get(domain=api_object.domain) save_stock_data_checkpoint(checkpoint, default_api, 1000, 0, checkpoint.start_date, None, False) checkpoint.start_date = None checkpoint.save()
def test_webusers_migration(self): checkpoint = MigrationCheckpoint( domain=TEST_DOMAIN, start_date=datetime.utcnow(), date=datetime.utcnow(), api="product", limit=100, offset=0 ) location_api = ApiSyncObject("webuser", self.endpoint.get_webusers, self.api_object.web_user_sync) synchronization(location_api, checkpoint, None, 100, 0) self.assertEqual("webuser", checkpoint.api) self.assertEqual(100, checkpoint.limit) self.assertEqual(0, checkpoint.offset) self.assertEqual(5, len(list(WebUser.by_domain(TEST_DOMAIN))))
def test_products_migration(self): checkpoint = MigrationCheckpoint( domain=TEST_DOMAIN, start_date=datetime.utcnow(), date=datetime.utcnow(), api="product", limit=100, offset=0 ) product_api = ApiSyncObject("product", self.endpoint.get_products, self.api_object.product_sync) synchronization(product_api, checkpoint, None, 100, 0) self.assertEqual("product", checkpoint.api) self.assertEqual(100, checkpoint.limit) self.assertEqual(0, checkpoint.offset) self.assertEqual(6, len(list(Prod.by_domain(TEST_DOMAIN))))
def test_webusers_migration(self): checkpoint = MigrationCheckpoint(domain=TEST_DOMAIN, start_date=datetime.utcnow(), date=datetime.utcnow(), api='product', limit=100, offset=0) location_api = ApiSyncObject('webuser', self.endpoint.get_webusers, self.api_object.web_user_sync) synchronization(location_api, checkpoint, None, 100, 0) self.assertEqual('webuser', checkpoint.api) self.assertEqual(100, checkpoint.limit) self.assertEqual(0, checkpoint.offset) self.assertEqual(5, len(list(WebUser.by_domain(TEST_DOMAIN))))
def test_locations_migration(self): checkpoint = MigrationCheckpoint( domain=TEST_DOMAIN, start_date=datetime.now(), date=datetime.now(), api='product', limit=100, offset=0 ) synchronization('location_facility', self.endpoint.get_locations, self.api_object.location_sync, checkpoint, None, 100, 0, filters=dict(type='facility')) self.assertEqual('location_facility', checkpoint.api) self.assertEqual(100, checkpoint.limit) self.assertEqual(0, checkpoint.offset) self.assertEqual(4, len(list(Location.by_domain(TEST_DOMAIN))))
def test_smsusers_migration(self): checkpoint = MigrationCheckpoint( domain=TEST_DOMAIN, start_date=datetime.now(), date=datetime.now(), api='product', limit=100, offset=0 ) synchronization('smsuser', self.endpoint.get_smsusers, self.api_object.sms_user_sync, checkpoint, None, 100, 0) self.assertEqual('smsuser', checkpoint.api) self.assertEqual(100, checkpoint.limit) self.assertEqual(0, checkpoint.offset) self.assertEqual(6, len(CommCareUser.by_domain(TEST_DOMAIN)))
def test_locations_migration(self): checkpoint = MigrationCheckpoint( domain=TEST_DOMAIN, start_date=datetime.now(), date=datetime.now(), api='product', limit=100, offset=0 ) synchronization('product', self.endpoint.get_products, self.api_object.product_sync, checkpoint, None, 100, 0) self.assertEqual('product', checkpoint.api) self.assertEqual(100, checkpoint.limit) self.assertEqual(0, checkpoint.offset) self.assertEqual(6, len(list(Prod.by_domain(TEST_DOMAIN))))
def test_products_migration(self): checkpoint = MigrationCheckpoint( domain=TEST_DOMAIN, start_date=datetime.utcnow(), date=datetime.utcnow(), api='product', limit=100, offset=0 ) product_api = ApiSyncObject( 'product', self.endpoint.get_products, self.api_object.product_sync ) synchronization(product_api, checkpoint, None, 100, 0) self.assertEqual('product', checkpoint.api) self.assertEqual(100, checkpoint.limit) self.assertEqual(0, checkpoint.offset) self.assertEqual(6, len(list(Prod.by_domain(TEST_DOMAIN))))
def add_products_to_loc(api): endpoint = api.endpoint synchronization(None, endpoint.get_locations, api.location_sync, None, None, 100, 0)
def sms_users_fix(api): endpoint = api.endpoint synchronization(None, endpoint.get_smsusers, api.add_language_to_user, None, None, 100, 0)
def add_products_to_loc(api): endpoint = api.endpoint synchronization(None, endpoint.get_locations, api.location_sync, None, None, 100, 0, filters={"is_active": True})
def resync_web_users(api_object): web_users_sync = api_object.apis[4] synchronization(web_users_sync, None, None, 100, 0)
def sms_users_fix(api): endpoint = api.endpoint api.set_default_backend() synchronization(None, endpoint.get_smsusers, partial(api.add_language_to_user), None, None, 100, 0)
def resync_web_users(api_object): web_users_sync = api_object.apis[5] synchronization(web_users_sync, None, None, 100, 0)