def test_chunk_list(self):
"""Tests the chunk_list function"""
self.assertEqual(
[[10, 11, 12, 13], [14, 15, 16, 17], [18, 19]],
list(utils.chunk_list(list(range(10, 20)), 4))
)
self.assertEqual(
[[1]],
list(utils.chunk_list([1], 7))
)
self.assertEqual(
[['L', 'o', 'r'], ['e', 'm', ' '], ['I', 'p', 's'], ['u', 'm']],
list(utils.chunk_list(list('Lorem Ipsum'), 3))
)
def run(self):
"""
Fetches the products to update via api. Queues a SynchronizeProductsTask and calls a new instance of itself after all
tasks are done. If no products found for update, sleeps until the next update time is reached.
:return: the result is always true
:rtype: bool
"""
logger.info('FindProductsToSynchronizeTask was called')
# get all products that shall be updated
products = self.__get_products_to_sync()
if products:
# chunk the products into 10 products each
products_chunked = list(chunk_list(list(products), 10))
logger.info(
'Starting chord for synchronization of %d products in %d chunks',
len(products), len(products_chunked))
# after all single product synchronize tasks are done recall the FindProductsToSynchronizeTask. That is because we do not know how long it takes to
# synchronize the products and there can be new ones meanwhile. If the newly called task finds no products, it will handle the new callback to the
# correct time.
chord(SynchronizeProductsTask().s(
[product.asin for product in product_list])
for product_list in products_chunked)(
FindProductsToSynchronizeTask().si())
else:
logger.info('No products found to update now')
# One might think this may interfere with newly created products and their synchronization if they are added before the
# FindProductsToSynchronizeTask is called again, but it doesn't. The new product is updated on creation and the next synchronization is always
# after the next task call.
oldest_synchronization = Product.objects.filter(
subscription__isnull=False, status__in=[0, 1]).aggregate(
Min('date_last_synced')
)['date_last_synced__min'] or datetime.now()
next_synchronization = oldest_synchronization + timedelta(
minutes=app_settings.
PRICE_MONITOR_AMAZON_PRODUCT_REFRESH_THRESHOLD_MINUTES)
logger.info('Eta for next FindProductsToSynchronizeTask run is %s',
next_synchronization)
FindProductsToSynchronizeTask().apply_async(
eta=next_synchronization)
return True
def run(self):
"""
Fetches the products to update via api. Queues a SynchronizeProductsTask and calls a new instance of itself after all
tasks are done. If no products found for update, sleeps until the next update time is reached.
:return: the result is always true
:rtype: bool
"""
logger.info('FindProductsToSynchronizeTask was called')
# get all products that shall be updated
products = self.__get_products_to_sync()
if products:
# chunk the products into 10 products each
products_chunked = list(chunk_list(list(products), 10))
logger.info('Starting chord for synchronization of %d products in %d chunks', len(products), len(products_chunked))
# after all single product synchronize tasks are done recall the FindProductsToSynchronizeTask. That is because we do not know how long it takes to
# synchronize the products and there can be new ones meanwhile. If the newly called task finds no products, it will handle the new callback to the
# correct time.
chord(
SynchronizeProductsTask().s([product.asin for product in product_list]) for product_list in products_chunked
)(
FindProductsToSynchronizeTask().si()
)
else:
logger.info('No products found to update now')
# One might think this may interfere with newly created products and their synchronization if they are added before the
# FindProductsToSynchronizeTask is called again, but it doesn't. The new product is updated on creation and the next synchronization is always
# after the next task call.
oldest_synchronization = Product.objects.filter(subscription__isnull=False, status__in=[0, 1]).aggregate(
Min('date_last_synced')
)['date_last_synced__min'] or datetime.now()
next_synchronization = oldest_synchronization + timedelta(minutes=app_settings.PRICE_MONITOR_AMAZON_PRODUCT_REFRESH_THRESHOLD_MINUTES)
logger.info('Eta for next FindProductsToSynchronizeTask run is %s', next_synchronization)
FindProductsToSynchronizeTask().apply_async(eta=next_synchronization)
return True
