def test_derive_key_name_from_video(self):
self._set_responses_xrange(BATCH_SIZE)
_task_handler("UUID")
videos = Video.all().fetch(BATCH_SIZE)
for v in videos:
key = VideoSubtitles.get_key_name("en", v.youtube_id)
subs = VideoSubtitles.get_by_key_name(key)
self.assertIsNotNone(subs)
def test_delay_between_batches(self):
self._set_responses_xrange(BATCH_SIZE)
expected_eta = gaetasktime.eta_utcnow() + datetime.timedelta(seconds=DEFER_SECONDS)
_task_handler("UUID")
self.assertTasksInQueue(n=1)
task, = self.get_tasks()
self.assertTrue(task.eta >= expected_eta)
def test_log_and_report_download_error(self, error, defer):
self._set_responses_xrange(1, content=DownloadError())
def _defer(func, uid, report, **kwargs):
self.assertEqual(report["errors"], 1)
defer.side_effect = _defer
_task_handler("UUID")
self.assertEqual(error.call_count, 1, "download error should be logged")
self.assertEqual(defer.call_count, 1, "task should be enqueued")
def test_report_number_of_fetches(self, defer):
self._set_responses_xrange(BATCH_SIZE)
def _defer(func, uid, id, cursor, report, **kwargs):
self.assertEqual(report["fetches"], BATCH_SIZE)
self.assertEqual(report["writes"], BATCH_SIZE)
defer.side_effect = _defer
_task_handler("UUID")
self.assertEqual(defer.call_count, 1, "task should be enqueued")
def test_assume_utf8_encoded_content(self):
# Universal Subtitles API returns utf-8
# u'\xc4\xd0' is unicode for the utf-8 byte string '\xc3\x84\xc3\x90'
utf8_str = "\xc3\x84\xc3\x90"
unicode_str = u"\xc4\xd0"
self._set_responses_xrange(1, content=utf8_str)
_task_handler("UUID")
self.assertEqual(VideoSubtitles.all().count(), 1)
subs = VideoSubtitles.all().get()
self.assertEqual(subs.json, unicode_str)
def test_process_next_batch_on_nonempty_cursor(self):
offset = 3
# these should be skipped, they'll DownloadError
for i in xrange(0, offset):
Video(youtube_id=str(i)).put()
# these should be downloaded
self._set_responses_xrange(offset, BATCH_SIZE + offset)
query = Video.all()
query.fetch(offset)
cursor = query.cursor()
_task_handler("UUID", cursor=cursor)
self.assertEqual(VideoSubtitles.all().count(), BATCH_SIZE)
def test_log_and_report_subtitles_put_error(self, error, defer, VideoSubtitles):
self._set_responses_xrange(1)
VideoSubtitles.return_value.put.side_effect = RuntimeError("failed mock put()")
def _defer(func, uid, report, **kwargs):
self.assertEqual(report["errors"], 1)
self.assertEqual(report["writes"], 0)
defer.side_effect = _defer
_task_handler("UUID")
self.assertEqual(error.call_count, 1, "put error should be logged")
self.assertEqual(defer.call_count, 1, "task should be enqueued")
self.assertEqual(VideoSubtitles.return_value.put.call_count, 1, "put() should have been called")
def test_should_not_put_duplicate_subtitles(self, info):
self._set_responses_xrange(BATCH_SIZE, content="some json")
# first fetch
_task_handler("UUID", 0)
self.assertEqual(VideoSubtitles.all().count(), BATCH_SIZE)
self.assertEqual(info.call_count, 0)
with patch("unisubs.handlers.VideoSubtitles") as MockVideoSubtitles:
MockVideoSubtitles.get_key_name = VideoSubtitles.get_key_name
MockVideoSubtitles.get_by_key_name = VideoSubtitles.get_by_key_name
# second fetch, same content
_task_handler("UUID", 1)
self.assertEqual(
MockVideoSubtitles.return_value.put.call_count, 0, "duplicate subtitles should not be put()"
)
self.assertEqual(info.call_count, BATCH_SIZE, "skipped put should be logged")
def test_schedule_task_after_processing_full_batch(self):
self._set_responses_xrange(BATCH_SIZE)
_task_handler("UUID")
self.assertTasksInQueue(n=1, name="UUID_1", queue_names=UNISUBS_FETCH_Q)
def test_process_first_batch_on_empty_cursor(self):
self._set_responses_xrange(BATCH_SIZE)
_task_handler("UUID")
self.assertEqual(VideoSubtitles.all().count(), BATCH_SIZE)
def test_report_started_immediately(self):
_task_handler("UUID")
self.assertEqual(VideoSubtitlesFetchReport.all().count(), 1)
def test_log_error_for_non_200_status(self, error):
self._set_responses_xrange(1, status_code=404)
_task_handler("UUID")
self.assertEqual(error.call_count, 1, "status error should be logged")
logged_message = error.call_args[0][0]
self.assertEqual(logged_message.count("404"), 1)
def test_schedule_report_after_processing_empty_batch(self):
_task_handler("UUID")
self.assertTasksInQueue(n=1, name="UUID_report", queue_names=UNISUBS_FETCH_Q)
def test_schedule_report_after_processing_partial_batch(self):
self._set_responses_xrange(1)
_task_handler("UUID")
self.assertTasksInQueue(n=1, name="UUID_report", queue_names=UNISUBS_FETCH_Q)
