def test_incremental_crawl_failure(self, bucket_mock, conn_mock, crawl_mock):
def failure_feed(url):
if '/feed' in url:
return {'notdata': [{'ooga': 'booga'}]}
self.facebook_patch = patch(
'targetshare.integration.facebook.client.urllib2.urlopen',
crawl_mock(1, 250, failure_feed)
)
self.facebook_patch.start()
the_past = epoch.from_date(timezone.now() - timedelta(days=365))
# Test runs in under a second typically, so we need to be slightly
# behind present time, so that we can see fbm.incremental_epoch
# get updated
present = epoch.from_date(timezone.now() - timedelta(seconds=30))
fbm = models.FBSyncMap.items.create(
fbid_primary=self.fbid, fbid_secondary=self.fbid, token=self.token.token,
back_filled=False, back_fill_epoch=the_past,
incremental_epoch=present,
status=models.FBSyncMap.COMPLETE, bucket='test_bucket_0'
)
existing_key = Mock()
existing_key.get_contents_as_string.return_value = '{"updated": 1, "data": [{"test": "testing"}]}'
bucket_mock.return_value = existing_key
conn_mock.return_value = s3_feed.BucketManager()
tasks.incremental_crawl(fbm.fbid_primary, fbm.fbid_secondary)
new_fbm = models.FBSyncMap.items.get_item(
fbid_primary=self.fbid, fbid_secondary=self.fbid)
self.assertEqual(fbm.status, fbm.COMPLETE)
self.assertEqual(int(new_fbm.incremental_epoch), present)
self.assertFalse(existing_key.set_contents_from_string.called)
def test_incremental_crawl(self, bucket_mock, conn_mock):
the_past = epoch.from_date(timezone.now() - timedelta(days=365))
# Test runs in under a second typically, so we need to be slightly
# behind present time, so that we can see fbm.incremental_epoch
# get updated
present = epoch.from_date(timezone.now() - timedelta(seconds=30))
fbm = models.FBSyncMap.items.create(
fbid_primary=self.fbid, fbid_secondary=self.fbid, token=self.token.token,
back_filled=False, back_fill_epoch=the_past,
incremental_epoch=present,
status=models.FBSyncMap.COMPLETE, bucket='test_bucket_0'
)
existing_key = Mock()
existing_key.data = {"updated": 1, "data": [{"test": "testing"}]}
bucket_mock.return_value = existing_key
conn_mock.return_value = s3_feed.BucketManager()
tasks.incremental_crawl(fbm.fbid_primary, fbm.fbid_secondary)
new_fbm = models.FBSyncMap.items.get_item(
fbid_primary=self.fbid, fbid_secondary=self.fbid)
self.assertEqual(fbm.status, fbm.COMPLETE)
self.assertGreater(int(new_fbm.incremental_epoch), present)
self.assertTrue(existing_key.extend_s3_data.called)
self.assertSequenceEqual(
existing_key.extend_s3_data.call_args_list[0][0],
(False,)
)