def test_derive_key_name_from_video(self):
self._set_responses_xrange(BATCH_SIZE)
_task_handler('UUID')
videos = Video.all().fetch(BATCH_SIZE)
for v in videos:
key = VideoSubtitles.get_key_name('en', v.youtube_id)
subs = VideoSubtitles.get_by_key_name(key)
self.assertIsNotNone(subs)
def test_derive_key_name_from_video(self):
self._set_responses_xrange(BATCH_SIZE)
_task_handler('UUID')
videos = Video.all().fetch(BATCH_SIZE)
for v in videos:
key = VideoSubtitles.get_key_name('en', v.youtube_id)
subs = VideoSubtitles.get_by_key_name(key)
self.assertIsNotNone(subs)
def test_delay_between_batches(self):
self._set_responses_xrange(BATCH_SIZE)
expected_eta = (gaetasktime.eta_utcnow() +
datetime.timedelta(seconds=DEFER_SECONDS))
_task_handler('UUID')
self.assertTasksInQueue(n=1)
task, = self.get_tasks()
self.assertTrue(task.eta >= expected_eta)
def test_delay_between_batches(self):
self._set_responses_xrange(BATCH_SIZE)
expected_eta = (gaetasktime.eta_utcnow() +
datetime.timedelta(seconds=DEFER_SECONDS))
_task_handler('UUID')
self.assertTasksInQueue(n=1)
task, = self.get_tasks()
self.assertTrue(task.eta >= expected_eta)
def test_report_number_of_fetches(self, defer):
self._set_responses_xrange(BATCH_SIZE)
def _defer(func, uid, id, cursor, report, **kwargs):
self.assertEqual(report['fetches'], BATCH_SIZE)
self.assertEqual(report['writes'], BATCH_SIZE)
defer.side_effect = _defer
_task_handler('UUID')
self.assertEqual(defer.call_count, 1, 'task should be enqueued')
def test_log_and_report_download_error(self, error, defer):
self._set_responses_xrange(1, content=DownloadError())
def _defer(func, uid, report, **kwargs):
self.assertEqual(report['errors'], 1)
defer.side_effect = _defer
_task_handler('UUID')
self.assertEqual(error.call_count, 1, 'download error should be logged')
self.assertEqual(defer.call_count, 1, 'task should be enqueued')
def test_report_number_of_fetches(self, defer):
self._set_responses_xrange(BATCH_SIZE)
def _defer(func, uid, id, cursor, report, **kwargs):
self.assertEqual(report['fetches'], BATCH_SIZE)
self.assertEqual(report['writes'], BATCH_SIZE)
defer.side_effect = _defer
_task_handler('UUID')
self.assertEqual(defer.call_count, 1, 'task should be enqueued')
def test_assume_utf8_encoded_content(self):
# Universal Subtitles API returns utf-8
# u'\xc4\xd0' is unicode for the utf-8 byte string '\xc3\x84\xc3\x90'
utf8_str = '\xc3\x84\xc3\x90'
unicode_str = u'\xc4\xd0'
self._set_responses_xrange(1, content=utf8_str)
_task_handler('UUID')
self.assertEqual(VideoSubtitles.all().count(), 1)
subs = VideoSubtitles.all().get()
self.assertEqual(subs.json, unicode_str)
def test_log_and_report_download_error(self, error, defer):
self._set_responses_xrange(1, content=DownloadError())
def _defer(func, uid, report, **kwargs):
self.assertEqual(report['errors'], 1)
defer.side_effect = _defer
_task_handler('UUID')
self.assertEqual(error.call_count, 1,
'download error should be logged')
self.assertEqual(defer.call_count, 1, 'task should be enqueued')
def test_assume_utf8_encoded_content(self):
# Universal Subtitles API returns utf-8
# u'\xc4\xd0' is unicode for the utf-8 byte string '\xc3\x84\xc3\x90'
utf8_str = '\xc3\x84\xc3\x90'
unicode_str = u'\xc4\xd0'
self._set_responses_xrange(1, content=utf8_str)
_task_handler('UUID')
self.assertEqual(VideoSubtitles.all().count(), 1)
subs = VideoSubtitles.all().get()
self.assertEqual(subs.json, unicode_str)
def test_process_next_batch_on_nonempty_cursor(self):
offset = 3
# these should be skipped, they'll DownloadError
for i in xrange(0, offset):
Video(youtube_id=str(i)).put()
# these should be downloaded
self._set_responses_xrange(offset, BATCH_SIZE + offset)
query = Video.all()
query.fetch(offset)
cursor = query.cursor()
_task_handler('UUID', cursor=cursor)
self.assertEqual(VideoSubtitles.all().count(), BATCH_SIZE)
def test_process_next_batch_on_nonempty_cursor(self):
offset = 3
# these should be skipped, they'll DownloadError
for i in xrange(0, offset):
Video(youtube_id=str(i)).put()
# these should be downloaded
self._set_responses_xrange(offset, BATCH_SIZE + offset)
query = Video.all()
query.fetch(offset)
cursor = query.cursor()
_task_handler('UUID', cursor=cursor)
self.assertEqual(VideoSubtitles.all().count(), BATCH_SIZE)
def test_should_not_put_duplicate_subtitles(self, info):
self._set_responses_xrange(BATCH_SIZE, content="some json")
# first fetch
_task_handler('UUID', 0)
self.assertEqual(VideoSubtitles.all().count(), BATCH_SIZE)
self.assertEqual(info.call_count, 0)
with patch('unisubs.VideoSubtitles') as MockVideoSubtitles:
MockVideoSubtitles.get_key_name = VideoSubtitles.get_key_name
MockVideoSubtitles.get_by_key_name = VideoSubtitles.get_by_key_name
# second fetch, same content
_task_handler('UUID', 1)
self.assertEqual(MockVideoSubtitles.return_value.put.call_count, 0,
'duplicate subtitles should not be put()')
self.assertEqual(info.call_count, BATCH_SIZE,
'skipped put should be logged')
def test_should_not_put_duplicate_subtitles(self, info):
self._set_responses_xrange(BATCH_SIZE, content="some json")
# first fetch
_task_handler('UUID', 0)
self.assertEqual(VideoSubtitles.all().count(), BATCH_SIZE)
self.assertEqual(info.call_count, 0)
with patch('unisubs.VideoSubtitles') as MockVideoSubtitles:
MockVideoSubtitles.get_key_name = VideoSubtitles.get_key_name
MockVideoSubtitles.get_by_key_name = VideoSubtitles.get_by_key_name
# second fetch, same content
_task_handler('UUID', 1)
self.assertEqual(MockVideoSubtitles.return_value.put.call_count, 0,
'duplicate subtitles should not be put()')
self.assertEqual(info.call_count, BATCH_SIZE,
'skipped put should be logged')
def test_log_and_report_subtitles_put_error(self, error, defer,
VideoSubtitles):
self._set_responses_xrange(1)
VideoSubtitles.return_value.put.side_effect = \
RuntimeError('failed mock put()')
def _defer(func, uid, report, **kwargs):
self.assertEqual(report['errors'], 1)
self.assertEqual(report['writes'], 0)
defer.side_effect = _defer
_task_handler('UUID')
self.assertEqual(error.call_count, 1, 'put error should be logged')
self.assertEqual(defer.call_count, 1, 'task should be enqueued')
self.assertEqual(VideoSubtitles.return_value.put.call_count, 1,
'put() should have been called')
def test_log_and_report_subtitles_put_error(self, error, defer,
VideoSubtitles):
self._set_responses_xrange(1)
VideoSubtitles.return_value.put.side_effect = \
RuntimeError('failed mock put()')
def _defer(func, uid, report, **kwargs):
self.assertEqual(report['errors'], 1)
self.assertEqual(report['writes'], 0)
defer.side_effect = _defer
_task_handler('UUID')
self.assertEqual(error.call_count, 1, 'put error should be logged')
self.assertEqual(defer.call_count, 1, 'task should be enqueued')
self.assertEqual(VideoSubtitles.return_value.put.call_count, 1,
'put() should have been called')
def test_log_error_for_non_200_status(self, error):
self._set_responses_xrange(1, status_code=404)
_task_handler('UUID')
self.assertEqual(error.call_count, 1, 'status error should be logged')
logged_message = error.call_args[0][0]
self.assertEqual(logged_message.count('404'), 1)
def test_report_started_immediately(self):
_task_handler('UUID')
self.assertEqual(VideoSubtitlesFetchReport.all().count(), 1)
def test_process_first_batch_on_empty_cursor(self):
self._set_responses_xrange(BATCH_SIZE)
_task_handler('UUID')
self.assertEqual(VideoSubtitles.all().count(), BATCH_SIZE)
def test_log_error_for_non_200_status(self, error):
self._set_responses_xrange(1, status_code=404)
_task_handler('UUID')
self.assertEqual(error.call_count, 1, 'status error should be logged')
logged_message = error.call_args[0][0]
self.assertEqual(logged_message.count('404'), 1)
def test_schedule_report_after_processing_partial_batch(self):
self._set_responses_xrange(1)
_task_handler('UUID')
self.assertTasksInQueue(n=1,
name='UUID_report',
queue_names=UNISUBS_FETCH_Q)
def test_schedule_task_after_processing_full_batch(self):
self._set_responses_xrange(BATCH_SIZE)
_task_handler('UUID')
self.assertTasksInQueue(n=1,
name='UUID_1',
queue_names=UNISUBS_FETCH_Q)
def test_process_first_batch_on_empty_cursor(self):
self._set_responses_xrange(BATCH_SIZE)
_task_handler('UUID')
self.assertEqual(VideoSubtitles.all().count(), BATCH_SIZE)
def test_schedule_report_after_processing_empty_batch(self):
_task_handler('UUID')
self.assertTasksInQueue(n=1,
name='UUID_report',
queue_names=UNISUBS_FETCH_Q)
def test_report_started_immediately(self):
_task_handler('UUID')
self.assertEqual(VideoSubtitlesFetchReport.all().count(), 1)
def test_schedule_task_after_processing_full_batch(self):
self._set_responses_xrange(BATCH_SIZE)
_task_handler('UUID')
self.assertTasksInQueue(n=1, name='UUID_1',
queue_names=UNISUBS_FETCH_Q)
def test_schedule_report_after_processing_empty_batch(self):
_task_handler('UUID')
self.assertTasksInQueue(n=1, name='UUID_report',
queue_names=UNISUBS_FETCH_Q)
def test_schedule_report_after_processing_partial_batch(self):
self._set_responses_xrange(1)
_task_handler('UUID')
self.assertTasksInQueue(n=1, name='UUID_report',
queue_names=UNISUBS_FETCH_Q)
