def test_derive_key_name_from_video(self): self._set_responses_xrange(BATCH_SIZE) _task_handler('UUID') videos = Video.all().fetch(BATCH_SIZE) for v in videos: key = VideoSubtitles.get_key_name('en', v.youtube_id) subs = VideoSubtitles.get_by_key_name(key) self.assertIsNotNone(subs)
def test_delay_between_batches(self): self._set_responses_xrange(BATCH_SIZE) expected_eta = (gaetasktime.eta_utcnow() + datetime.timedelta(seconds=DEFER_SECONDS)) _task_handler('UUID') self.assertTasksInQueue(n=1) task, = self.get_tasks() self.assertTrue(task.eta >= expected_eta)
def test_report_number_of_fetches(self, defer): self._set_responses_xrange(BATCH_SIZE) def _defer(func, uid, id, cursor, report, **kwargs): self.assertEqual(report['fetches'], BATCH_SIZE) self.assertEqual(report['writes'], BATCH_SIZE) defer.side_effect = _defer _task_handler('UUID') self.assertEqual(defer.call_count, 1, 'task should be enqueued')
def test_log_and_report_download_error(self, error, defer): self._set_responses_xrange(1, content=DownloadError()) def _defer(func, uid, report, **kwargs): self.assertEqual(report['errors'], 1) defer.side_effect = _defer _task_handler('UUID') self.assertEqual(error.call_count, 1, 'download error should be logged') self.assertEqual(defer.call_count, 1, 'task should be enqueued')
def test_assume_utf8_encoded_content(self): # Universal Subtitles API returns utf-8 # u'\xc4\xd0' is unicode for the utf-8 byte string '\xc3\x84\xc3\x90' utf8_str = '\xc3\x84\xc3\x90' unicode_str = u'\xc4\xd0' self._set_responses_xrange(1, content=utf8_str) _task_handler('UUID') self.assertEqual(VideoSubtitles.all().count(), 1) subs = VideoSubtitles.all().get() self.assertEqual(subs.json, unicode_str)
def test_process_next_batch_on_nonempty_cursor(self): offset = 3 # these should be skipped, they'll DownloadError for i in xrange(0, offset): Video(youtube_id=str(i)).put() # these should be downloaded self._set_responses_xrange(offset, BATCH_SIZE + offset) query = Video.all() query.fetch(offset) cursor = query.cursor() _task_handler('UUID', cursor=cursor) self.assertEqual(VideoSubtitles.all().count(), BATCH_SIZE)
def test_should_not_put_duplicate_subtitles(self, info): self._set_responses_xrange(BATCH_SIZE, content="some json") # first fetch _task_handler('UUID', 0) self.assertEqual(VideoSubtitles.all().count(), BATCH_SIZE) self.assertEqual(info.call_count, 0) with patch('unisubs.VideoSubtitles') as MockVideoSubtitles: MockVideoSubtitles.get_key_name = VideoSubtitles.get_key_name MockVideoSubtitles.get_by_key_name = VideoSubtitles.get_by_key_name # second fetch, same content _task_handler('UUID', 1) self.assertEqual(MockVideoSubtitles.return_value.put.call_count, 0, 'duplicate subtitles should not be put()') self.assertEqual(info.call_count, BATCH_SIZE, 'skipped put should be logged')
def test_log_and_report_subtitles_put_error(self, error, defer, VideoSubtitles): self._set_responses_xrange(1) VideoSubtitles.return_value.put.side_effect = \ RuntimeError('failed mock put()') def _defer(func, uid, report, **kwargs): self.assertEqual(report['errors'], 1) self.assertEqual(report['writes'], 0) defer.side_effect = _defer _task_handler('UUID') self.assertEqual(error.call_count, 1, 'put error should be logged') self.assertEqual(defer.call_count, 1, 'task should be enqueued') self.assertEqual(VideoSubtitles.return_value.put.call_count, 1, 'put() should have been called')
def test_log_error_for_non_200_status(self, error): self._set_responses_xrange(1, status_code=404) _task_handler('UUID') self.assertEqual(error.call_count, 1, 'status error should be logged') logged_message = error.call_args[0][0] self.assertEqual(logged_message.count('404'), 1)
def test_report_started_immediately(self): _task_handler('UUID') self.assertEqual(VideoSubtitlesFetchReport.all().count(), 1)
def test_process_first_batch_on_empty_cursor(self): self._set_responses_xrange(BATCH_SIZE) _task_handler('UUID') self.assertEqual(VideoSubtitles.all().count(), BATCH_SIZE)
def test_schedule_report_after_processing_partial_batch(self): self._set_responses_xrange(1) _task_handler('UUID') self.assertTasksInQueue(n=1, name='UUID_report', queue_names=UNISUBS_FETCH_Q)
def test_schedule_task_after_processing_full_batch(self): self._set_responses_xrange(BATCH_SIZE) _task_handler('UUID') self.assertTasksInQueue(n=1, name='UUID_1', queue_names=UNISUBS_FETCH_Q)
def test_schedule_report_after_processing_empty_batch(self): _task_handler('UUID') self.assertTasksInQueue(n=1, name='UUID_report', queue_names=UNISUBS_FETCH_Q)