def start_test(self):
tile1 = Tile.objects.get(pk=10)
tile2 = Tile.objects.get(pk=11)
pro = Product.objects.get(pk=13)
content = Content.objects.get(pk=6)
queue = TileSerializationQueue()
with mock.patch('apps.assets.signals.tile_saved') as mock_tile_saved:
with mock.patch('apps.assets.signals.tile_m2m_changed'
) as mock_tile_m2m_changed:
with mock.patch('apps.assets.signals.content_m2m_changed'
) as mock_content_m2m_changed:
enable_tile_serialization_signals()
# verify signals work
tile1.save()
self.assertEquals(mock_tile_saved.call_count, 1)
# tile_m2m_changed & content_m2m_changed are both m2m_changed signal recievers
tile2.content.add(content)
self.assertEquals(mock_tile_m2m_changed.call_count,
2) # m2m_changed pre_add, post_add
content.tagged_products.add(pro)
self.assertEquals(mock_content_m2m_changed.call_count,
4) # 2x m2m_changed pre_add, post_add
queue.start()
# None of call counts should have incremented
tile1.save() # queue's tile1
self.assertEquals(mock_tile_saved.call_count, 1)
tile2.products.add(pro) # queue's tile2
self.assertEquals(mock_tile_m2m_changed.call_count, 4)
content.tagged_products.add(pro) # queue's tile2 again
self.assertEquals(mock_content_m2m_changed.call_count, 4)
# queue is tile1 & tile2
self.assertEquals(len(queue), 2)
def add_tiles_test(self):
tile1 = Tile.objects.get(pk=10)
tile2 = Tile.objects.get(pk=11)
tiles = [tile1, tile2]
queue = TileSerializationQueue()
queue.add(tiles)
self.assertEquals(len(queue), 2)
def serialize_test(self):
tile1 = Tile.objects.get(pk=10)
tile2 = Tile.objects.get(pk=11)
pro = Product.objects.get(pk=13)
content = Content.objects.get(pk=6)
queue = TileSerializationQueue()
with mock.patch('apps.assets.models.Tile.update_ir_cache',
mock.Mock(return_value=('', False))) as mocked_handler:
queue.start()
tile1.save()
tile2.save()
self.assertEquals(mocked_handler.call_count, 0)
queue.stop()
queue.serialize()
self.assertEquals(mocked_handler.call_count, 2)
self.assertEquals(len(queue), 0)
class TileSerializationPipeline(object):
""" Tiles that are updated throughout this scrape job are queued. When the job is
finished, tiles are serialized in bulk """
def __init__(self, *args, **kwargs):
self.queue = TileSerializationQueue()
super(TileSerializationPipeline, self).__init__(*args, **kwargs)
def open_spider(self, spider):
""" Captures all tile serialization signals, keeps queue of tiles """
spider.logger.info(u"Delaying tile serialization")
self.queue.start()
def close_spider(self, spider):
self.queue.stop()
spider.logger.info(u"Serializing {} delayed tiles".format(
len(self.queue)))
self.queue.serialize()
def stop_test(self):
tile1 = Tile.objects.get(pk=10)
tile2 = Tile.objects.get(pk=11)
pro = Product.objects.get(pk=13)
content = Content.objects.get(pk=6)
queue = TileSerializationQueue()
with mock.patch('apps.assets.signals.tile_saved') as mock_tile_saved:
with mock.patch('apps.assets.signals.tile_m2m_changed'
) as mock_tile_m2m_changed:
with mock.patch('apps.assets.signals.content_m2m_changed'
) as mock_content_m2m_changed:
enable_tile_serialization_signals()
queue.start()
# verify signals disabled
tile1.save()
self.assertEquals(mock_tile_saved.call_count, 0)
tile2.content.add(content)
self.assertEquals(mock_tile_m2m_changed.call_count, 0)
content.tagged_products.add(pro)
self.assertEquals(mock_content_m2m_changed.call_count, 0)
self.assertEquals(len(queue), 2) # queue is tile1 & tile2
# reset queue
queue.tiles_to_serialize = set()
queue.stop()
# Signals reconnected
tile1.save()
self.assertEquals(mock_tile_saved.call_count, 1)
# tile_m2m_changed & content_m2m_changed are both m2m_changed signal recievers
tile2.products.add(pro)
self.assertEquals(mock_tile_m2m_changed.call_count,
2) # m2m_changed pre_add, post_add
content.tagged_products.add(pro)
self.assertEquals(mock_content_m2m_changed.call_count,
4) # 2x m2m_changed pre_add, post_add
# No tiles queued
self.assertEquals(len(queue), 0)
def __init__(self, *args, **kwargs):
self.queue = TileSerializationQueue()
super(TileSerializationPipeline, self).__init__(*args, **kwargs)
def add_keyerror_test(self):
tile1 = Tile.objects.get(pk=10)
tiles = [tile1, 2]
queue = TileSerializationQueue()
with self.assertRaises(TypeError):
queue.add(tiles)
def add_tile_test(self):
tile = Tile.objects.get(pk=10)
queue = TileSerializationQueue()
queue.add(tile)
self.assertEquals(len(queue), 1)