def setUp(self): self.video = make_video() users = User.objects.all() (self.u1, self.u2) = users[:2] self.anon = User.get_anonymous() v = self.video self.en1 = pipeline.add_subtitles(v, 'en', title="title 1", description="desc 1", subtitles=[(100, 200, "sub 1")], author=self.u1) self.en2 = pipeline.add_subtitles(v, 'en', title="title 2", description="desc 2", subtitles=[(100, 200, "sub 2")], author=self.u2) self.en3 = pipeline.add_subtitles(v, 'en', title="title 3", description="desc 3", subtitles=[(100, 200, "sub 3")], author=self.u1)
def test_add_empty_versions(self): # Start with no SubtitleLanguages. self.assertEqual( SubtitleLanguage.objects.filter(video=self.video).count(), 0) # Put a version through the pipeline. pipeline.add_subtitles(self.video, 'en', None) # It should create the SubtitleLanguage automatically, with one version. self.assertEqual( SubtitleLanguage.objects.filter(video=self.video).count(), 1) self.assertEqual( SubtitleVersion.objects.filter(video=self.video).count(), 1) sl = SubtitleLanguage.objects.get(video=self.video, language_code='en') # Make sure the version seems sane. v = sl.get_tip(full=True) self.assertEqual(v.version_number, 1) self.assertEqual(v.language_code, 'en') # Put another version through the pipeline. pipeline.add_subtitles(self.video, 'en', None) # Now we should have two versions for a single language. self.assertEqual( SubtitleLanguage.objects.filter(video=self.video).count(), 1) self.assertEqual( SubtitleVersion.objects.filter(video=self.video).count(), 2) # Make sure it looks sane too. sl.clear_tip_cache() v = sl.get_tip(full=True) self.assertEqual(v.version_number, 2) self.assertEqual(v.language_code, 'en')
def quick_add_subs(language, subs_texts, escape=True): subtitles = babelsubs.storage.SubtitleSet( language_code=language.language_code) for i, text in enumerate(subs_texts): subtitles.append_subtitle(i * 1000, i * 1000 + 999, text, escape=escape) add_subtitles(language.video, language.language_code, subtitles)
def test_delete_translation_tasks(self): # We should delete translation tasks if there are no more languages # with public versions available. However, we should not delete # in-progress translation tasks, or review/approve tasks. Those can # continue alright with the forked language. # make a translation task Task(team=self.team, team_video=self.team_video, assignee=None, language='de', type=Task.TYPE_IDS['Translate']).save() # make an in-progress translation task v = pipeline.add_subtitles(self.video, 'ja', None, complete=False, visibility='private') Task(team=self.team, team_video=self.team_video, language='ja', type=Task.TYPE_IDS['Translate'], assignee=self.user, new_subtitle_version=v).save() # make review/approve tasks test_factories.make_review_task(self.team_video, 'es', self.user) test_factories.make_approve_task(self.team_video, 'sv', self.user) # check initial task counts translate_qs = Task.objects.incomplete_translate().filter( language='de') in_progress_qs = Task.objects.incomplete_translate().filter( language='ja') review_qs = Task.objects.incomplete_review().filter(language='es') approve_qs = Task.objects.incomplete_approve().filter(language='sv') self.assertEquals(translate_qs.count(), 1) self.assertEquals(in_progress_qs.count(), 1) self.assertEquals(review_qs.count(), 1) self.assertEquals(approve_qs.count(), 1) # make a second language. If we delete that language, we should still # keep translation tasks. other_lang_version = pipeline.add_subtitles(self.video, 'fr', None) other_lang_version.subtitle_language.nuke_language() self.assertEquals(translate_qs.count(), 1) self.assertEquals(in_progress_qs.count(), 1) self.assertEquals(review_qs.count(), 1) self.assertEquals(approve_qs.count(), 1) # but when we delete our original language, then there's no source # languages, so we should delete the translation task, but keep # in-progress translation tasks, as well as review tasks self.language.nuke_language() self.assertEquals(translate_qs.count(), 0) self.assertEquals(in_progress_qs.count(), 1) self.assertEquals(review_qs.count(), 1) self.assertEquals(approve_qs.count(), 1)
def test_two_languages(self): user = User.objects.all()[0] video = Video.objects.filter(teamvideo__isnull=False)[0] video.user = user video.save() sv_en = add_subtitles(video, 'en', make_subtitle_lines(4), author=user, complete=True) video_changed_tasks(video.pk, sv_en.pk) sv_cs = add_subtitles(video, 'cs', make_subtitle_lines(4), complete=True, author=user) video_changed_tasks(video.pk, sv_cs.pk) self.assertEquals(2, BillingRecord.objects.all().count())
def test_approved(self): self.assertEquals(0, Workflow.objects.count()) self.team.workflow_enabled = True self.team.save() Workflow.objects.create(team=self.team, approve_allowed=20) self.assertEquals(1, Workflow.objects.count()) self.assertTrue(self.team.get_workflow().approve_enabled) english = make_subtitle_language(self.video, 'en') spanish = make_subtitle_language(self.video, 'es') for i in range(1, 10): add_subtitles(self.video, english.language_code,[], created=datetime(2012, 1, i, 0, 0, 0), visibility='private', ) # make two versions public to be sure we're selecting the very first one v1_en = SubtitleVersion.objects.get(subtitle_language=english, version_number=3) v2_en = SubtitleVersion.objects.get(subtitle_language=english, version_number=6) v1_en.publish() v1_en.save() v2_en.publish() v2_en.save() b = BillingReport.objects.create( start_date=date(2012, 1, 1), end_date=date(2012, 1, 2)) b.teams.add(self.team) past_date = self.team.created - timedelta(days=5) make_subtitle_version(spanish, created=past_date, note=FROM_YOUTUBE_MARKER) langs = self.video.newsubtitlelanguage_set.all() self.assertEqual(len(langs) , 2) created, imported, _ = b._get_lang_data(langs, datetime(2012, 1, 1, 13, 30, 0), self.team ) self.assertEqual(len(created) , 1) v = created[0][1] self.assertEquals(v.version_number, 3) self.assertEqual(v.subtitle_language , english)
def new_version(v): subs = [(0, 1000, 'Hello', {}), (2000, 5000, 'world.', {})] add_subtitles(v, 'en', subs, author=self.author, committer=self.author) subs = [(0, 1000, 'Hello', {}), (3000, 5000, 'world.', {})] return add_subtitles(v, 'en', subs, author=self.author, committer=self.author)
def _stack_version(sv, nsl): """Stack the given version onto the given new SL.""" from apps.subtitles import pipeline visibility = get_visibility_from_old_version(sv) subtitles = _get_subtitles(sv) try: subtitles = list(subtitles) # set subtitle set as the pipeline will pass escaping # otherwise and it will break sset = SubtitleSet.from_list(nsl.language_code, subtitles) nsv = pipeline.add_subtitles(nsl.video, nsl.language_code, sset, title=sv.title, description=sv.description, parents=[], visibility=visibility, author=sv.user, created=sv.datetime_started) except: log_subtitle_error(sv, subtitles) raise sv.new_subtitle_version = nsv sv.needs_sync = False sv.save(tern_sync=True) log('SubtitleVersion', 'stacked', sv.pk, nsv.pk)
def bulk_subs(sub_data): """Create a bunch of videos/languages/versions sub_data is a dict of dicts containing the data to create the objects with: * sub_data maps video titles to language data * language data map language codes to a list of version data * version data is a dict containing kwargs to pass to pipeline.create_subtitles(). returns a tuple of dicts: * a dict that maps video titles to videos * a dict that maps (title, language_code) to languages * a dict that maps (title, language_code, version_number) to versions """ videos = {} langs = {} versions = {} for video_title, language_data in sub_data.items(): video = create_video(title=video_title) videos[video_title] = video for language_code, version_data in language_data.items(): lang = SubtitleLanguage.objects.create( video=video, language_code=language_code) langs[video_title, language_code] = lang for kwargs in version_data: v = pipeline.add_subtitles(video, language_code, None, **kwargs) versions[video_title, language_code, v.version_number] = v return videos, langs, versions
def make_approve_task(team_video, language_code, user): """Move a video through the tasks process to the approve stage, then return that task. assumptions: - there are no Tasks or SubtitleVersions for this video+language - approve is enabled for the team """ team = team_video.team assert team.get_workflow().approve_allowed != 0 task = Task(team=team, team_video=team_video, assignee=None, language=language_code, type=Task.TYPE_IDS['Translate']) task.save() v = pipeline.add_subtitles(team_video.video, language_code, None, complete=False, visibility='private') task.assignee = user task.new_subtitle_version = v task = task.complete() if task.type == Task.TYPE_IDS['Review']: task.assignee = user task.approved = Task.APPROVED_IDS['Approved'] return task.complete() else: # approve task return task
def save(self): # If the primary audio language code was given, we adjust it on the # video NOW, before saving the subtitles, so that the pipeline can take # it into account when determining task types. self._save_primary_audio_language_code() language_code = self.cleaned_data['language_code'] from_language_code = self.cleaned_data['from_language_code'] complete = self.cleaned_data['complete'] subtitles = self._parsed_subtitles if from_language_code: # If this is a translation, its subtitles should use the timing data # from the source. We know that the source has at least as many # subtitles as the new version, so we can just match them up # first-come, first-serve. source_subtitles = self.from_sv.get_subtitles() i = 0 # instead of translating to subtitle_items, we're updating the # dfxp elements in place. This guarantees no monkey business with # escaping / styling for old, new in izip(source_subtitles.subtitle_items(), subtitles.get_subtitles()): subtitles.update(i, from_ms=old.start_time, to_ms=old.end_time) i += 1 else: # Otherwise we can just use the subtitles the user uploaded as-is. # No matter what, text files that aren't translations cannot be # complete because they don't have timing data. if self.extension == 'txt': complete = False title, description = self._find_title_description(language_code) parents = self._find_parents(from_language_code) version = pipeline.add_subtitles(self.video, language_code, subtitles, title=title, description=description, author=self.user, parents=parents, committer=self.user, complete=complete, origin=ORIGIN_UPLOAD) # Handle forking SubtitleLanguages that were translations when # a standalone version is uploaded. # # For example: assume there is a French translation of English. # Uploading a "straight from video" version of French should fork it. sl = version.subtitle_language if not from_language_code and is_dependent(sl): sl.fork() # TODO: Pipeline this. video_changed_tasks.delay(sl.video_id, version.id) return version
def test_video_language_title_translation(self): # for translated languages, we display the title in the same way. In # the past we displayed it differently, this test is still useful video = test_factories.create_video(primary_audio_language_code='en', title='Video Title') en_version = pipeline.add_subtitles(video, 'en', None, title="English Title") fr_version = pipeline.add_subtitles(video, 'fr', None, title="French Title", parents=[en_version]) fr = fr_version.subtitle_language self.assertEquals(views.language_page_title(fr), 'French Title with subtitles | Amara')
def test_video_language_title_fallback(self): # if a language doesn't have a title, then we fall back to the video # title (which is the english title, since that's the primary audoio video = test_factories.create_video(primary_audio_language_code='en', title='Video Title') en_version = pipeline.add_subtitles(video, 'en', None) en = en_version.subtitle_language self.assertEquals(views.language_page_title(en), 'Video Title with subtitles | Amara')
def test_record_insertion(self): user = User.objects.all()[0] video = Video.objects.filter(teamvideo__isnull=False)[0] video.primary_audio_language_code = 'en' video.user = user video.save() now = datetime.now() sv = add_subtitles(video, 'en', make_subtitle_lines(4), complete=True, author=user, created=now) sl = sv.subtitle_language video_changed_tasks(video.pk, sv.pk) self.assertEquals(1, BillingRecord.objects.all().count()) br = BillingRecord.objects.all()[0] self.assertEquals(br.video.pk, video.pk) self.assertEquals(br.team.pk, video.get_team_video().team.pk) self.assertEquals(br.created, now) self.assertEquals(br.is_original, sl.is_primary_audio_language()) self.assertEquals(br.user.pk, user.pk) self.assertEquals(br.new_subtitle_language.pk, sl.pk) team = video.get_team_video().team start = datetime(2013, 1, 1, 0, 0) end = datetime.now() + timedelta(days=1) csv_data = BillingRecord.objects.csv_report_for_team(team, start, end) self.assertEquals(2, len(csv_data)) self.assertEquals(10, len(csv_data[1])) # 2 sv = add_subtitles(video, 'en', make_subtitle_lines(4), author=user, created=now) sl = sv.subtitle_language video_changed_tasks(video.pk, sv.pk) # A new one shouldn't be created for the same language self.assertEquals(1, BillingRecord.objects.all().count())
def test_update_source_language(self): """ https://unisubs.sifterapp.com/issues/2225 Create a version not synced. Create a translation. Then later finish the original one """ user = User.objects.all()[0] video = Video.objects.filter(teamvideo__isnull=False)[0] video.user = user video.save() original_version = add_subtitles( video, 'en', make_subtitle_lines(4, is_synced=False), complete=False, author=user ) original_lang = original_version.subtitle_language video_changed_tasks(video.pk, original_version.pk) self.assertEquals(0, BillingRecord.objects.all().count()) translation_version = add_subtitles( video, 'pt', make_subtitle_lines(4, is_synced=False), author=user, parents=[original_version]) translation_language = translation_version.subtitle_language # no billing for this one, because it isn't synced! self.assertEquals(0, BillingRecord.objects.all().count()) # now sync them original_version = add_subtitles( video, 'en', make_subtitle_lines(4, is_synced=True), complete=True, author=user) original_lang = original_version.subtitle_language video_changed_tasks(video.pk, original_version.pk) bl_original = BillingRecord.objects.filter(new_subtitle_language=original_lang) self.assertEquals(1, bl_original.count()) translation_version = add_subtitles( video, 'pt', make_subtitle_lines(5), author=user, parents=[original_version], complete=True) video_changed_tasks(video.pk, translation_version.pk) bl_translation = BillingRecord.objects.filter(new_subtitle_language=translation_language) self.assertEquals(1, bl_translation.count())
def test_multiple_languages(self): # english is the original, completed language en = self.add_completed_subtitles('en', [ (0, 1000, "Hello, ", { 'new_paragraph': True }), (1500, 2500, "World"), ]) # Kurdish is completed ar = self.add_completed_subtitles('ar', [ (0, 1000, "Hello, ", { 'new_paragraph': True }), (1500, 2500, "World"), ]) # french is incomplete fr = self.add_not_completed_subtitles('fr', [ (0, 1000, "Hello, ", { 'new_paragraph': True }), (1500, 2500, "World"), ]) # japanese is incomplete, and timing is missing ja = self.add_not_completed_subtitles('ja', [ (0, 1000, "Hello, ", { 'new_paragraph': True }), (None, None, "World"), ]) # portuguese shouldn't be listed because there are no lines pipeline.add_subtitles(self.video, 'pt', None) # LanguageList should return lines for all the languages, with # the original first, then the rest in alphabetical order. self.assertEquals( views.LanguageList(self.video).items, [ ('English', 'complete', ['original'], en.get_absolute_url()), ('Arabic', 'complete', [], ar.get_absolute_url()), ('French', 'incomplete', ['incomplete' ], fr.get_absolute_url()), ('Japanese', 'needs-timing', ['incomplete' ], ja.get_absolute_url()), ])
def test_video_language_title(self): video = test_factories.create_video(primary_audio_language_code='en', title='Video Title') en_version = pipeline.add_subtitles(video, 'en', None, title="English Title") en = en_version.subtitle_language self.assertEquals(views.language_page_title(en), 'English Title with subtitles | Amara')
def test_rollback_parents(self): v = self.video de1 = pipeline.add_subtitles(v, 'de', []) is1 = pipeline.add_subtitles(v, 'is', []) is2 = pipeline.add_subtitles(v, 'is', [], parents=[de1]) is3 = pipeline.add_subtitles(v, 'is', []) def _ids(s): return set(i.id for i in s) self.assertEqual(_ids(is2.parents.full()), _ids([is1, de1])) # Rollbacks do not inherit the parents of their sources. is4 = pipeline.rollback_to(v, 'is', 1) self.assertEqual(_ids(is4.parents.full()), _ids([is3])) is5 = pipeline.rollback_to(v, 'is', 2) self.assertEqual(_ids(is5.parents.full()), _ids([is4]))
def test_get_imported(self): team = Team.objects.all()[0] video = Video.objects.all()[0] team_created = team.created b = BillingReport.objects.create( start_date=date(2012, 1, 1), end_date=date(2012, 1, 2)) b.teams.add(team) SubtitleLanguage.objects.all().delete() sl_en = SubtitleLanguage.objects.create(video=video, language_code='en') sl_cs = SubtitleLanguage.objects.create(video=video, language_code='cs') sl_fr = SubtitleLanguage.objects.create(video=video, language_code='fr') sl_es = SubtitleLanguage.objects.create(video=video, language_code='es') SubtitleLanguage.objects.create(video=video, language_code='ru') before_team_created = team_created - timedelta(days=10) after_team_created = team_created + timedelta(days=10) # Imported add_subtitles(video, 'fr', [], created=before_team_created, note=FROM_YOUTUBE_MARKER) # Created add_subtitles(video, 'fr', [], created=after_team_created) add_subtitles(video, 'en', [], created=before_team_created, note=FROM_YOUTUBE_MARKER) # Imported add_subtitles(video, 'es', [], created=before_team_created) # Imported add_subtitles(video, 'cs', [], created=after_team_created, note=FROM_YOUTUBE_MARKER) # Done with setup, let's test things languages = SubtitleLanguage.objects.all() imported, crowd_created = b._separate_languages(languages) self.assertEquals(len(imported), 3) imported_pks = [i.pk for i in imported] self.assertTrue(sl_fr.pk in imported_pks) self.assertTrue(sl_es.pk in imported_pks) self.assertTrue(sl_cs.pk in imported_pks)
def test_update_language_complete(self): """ https://unisubs.sifterapp.com/issues/2225 Create a version not synced. Then later """ user = User.objects.all()[0] video = Video.objects.filter(teamvideo__isnull=False)[0] video.user = user video.save() first_version = add_subtitles(video, 'en', make_subtitle_lines(4, is_synced=False), complete=False, author=user) # create a transla video_changed_tasks(video.pk, first_version.pk) self.assertEquals(0, BillingRecord.objects.all().count()) second_version = add_subtitles(video, 'en', make_subtitle_lines(4), complete=True, author=user) video_changed_tasks(video.pk, second_version.pk) self.assertEquals(1, BillingRecord.objects.all().count())
def _create_subtitle_version(sv, last_version): """Sync the old SubtitleVersion by creating a new SubtitleVersion. If this language is a translation, and we're creating the final version in the chain, the parents of the new version will set to the tip of the source: """ from apps.subtitles import pipeline from django.core.exceptions import MultipleObjectsReturned sl = sv.language nsl = sl.new_subtitle_language visibility = get_visibility_from_old_version(sv) subtitles = _get_subtitles(sv) parents = [] if last_version and sl.is_dependent(): if sl.standard_language: tip = sl.standard_language.new_subtitle_language.get_tip() if tip: parents = [tip] else: log('SubtitleVersion', 'ORPHAN', sl.pk, None) if not dry: try: subtitles = list(subtitles) nsv = pipeline.add_subtitles(nsl.video, nsl.language_code, subtitles, title=sv.title, description=sv.description, parents=parents, visibility=visibility, author=sv.user, created=sv.datetime_started) except MultipleObjectsReturned: log('SubtitleVersion', 'DUPLICATE_TASKS', sv.pk, None) except: log_subtitle_error(sv, subtitles) raise sv.new_subtitle_version = nsv sv.needs_sync = False sv.save(tern_sync=True) log('SubtitleVersion', 'create', sv.pk, nsv.pk)
def test_data_prep(self): video = Video.objects.all()[0] subs = [ (0, 1000, 'Hi'), (2000, 3000, 'How are you?'), ] new_sv = pipeline.add_subtitles(video, 'en', subs) content, t, code = _prepare_subtitle_data_for_version(new_sv) srt = "1\r\n00:00:00,000 --> 00:00:01,000\r\nHi\r\n\r\n2\r\n00:00:02,000 --> 00:00:03,000\r\nHow are you?\r\n\r\n3\r\n00:01:52,000 --> 00:01:55,000\r\nSubtitles by the Amara.org community\r\n" self.assertEquals(srt, content) self.assertEquals('', t) self.assertEquals('en', code)
def test_original_language(self): user = User.objects.all()[0] video = Video.objects.filter(teamvideo__isnull=False)[0] video.user = user video.primary_audio_language_code = '' video.save() sv_en = add_subtitles(video, 'en', make_subtitle_lines(4), complete=True) video_changed_tasks(video.pk, sv_en.pk) self.assertEquals(1, BillingRecord.objects.all().count()) br = BillingRecord.objects.all()[0] self.assertFalse(br.is_original)
def setUp(self): original_video, created = Video.get_or_create_for_url( "http://www.example.com/original.mp4") original_video.duration = 10 original_video.save() self.original_video = original_video self.language = SubtitleLanguage.objects.create(video=original_video, language_code='en', is_forked=True) self.version = pipeline.add_subtitles( self.original_video, self.language.language_code, [], created=datetime.datetime.now(), )
def add_subs(self, **kwargs): defaults = { 'language_code': 'en', 'complete': True, 'visibility': 'public' } defaults.update(kwargs) default_subs = ('apps/webdriver_testing/subtitle_data' '/basic_subs.dfxp') s = defaults.get('subtitles', default_subs) subs = load_from_file(s, language=defaults['language_code']) sub_items = subs.to_internal() defaults['subtitles'] = sub_items v = pipeline.add_subtitles(**defaults) return v
def setUp(self): self.team = test_factories.create_team(workflow_enabled=True) self.workflow = test_factories.create_workflow(self.team) self.user = test_factories.create_user(is_staff=True) self.member = test_factories.create_team_member(self.team, self.user, role=ROLE_ADMIN) self.video = test_factories.create_video() self.team_video = test_factories.create_team_video( self.team, self.user, self.video) self.non_team_video = test_factories.create_video() # create a bunch of versions self.versions = [ pipeline.add_subtitles(self.video, 'en', None) for i in xrange(5) ] self.language = self.video.get_primary_audio_subtitle_language()
def make_review_task(team_video, language_code, user): """Move a video through the tasks process to the review stage, then return that task. assumptions: - there are no Tasks or SubtitleVersions for this video+language - review is enabled for the team """ team = team_video.team task = Task(team=team, team_video=team_video, assignee=None, language=language_code, type=Task.TYPE_IDS['Translate']) task.save() v = pipeline.add_subtitles(team_video.video, language_code, None, complete=False, visibility='private') task.assignee = user task.new_subtitle_version = v return task.complete()
def _sub_list_to_sv(self, subs): sublines = [] for sub in subs: sublines.append( SubtitleLine( sub['start'], sub['end'], sub['text'], {}, )) user = User.objects.all()[0] new_sv = pipeline.add_subtitles( self.original_video, self.language.language_code, sublines, author=user, ) return new_sv
def make_subtitle_version(subtitle_language, subtitles=[], author=None, parents=None, committer=None, complete=None, title=None, description=None, created=None, note=None): committer = committer or author return pipeline.add_subtitles(subtitle_language.video, subtitle_language.language_code, subtitles, author=author, parents=parents, committer=committer, complete=complete, title=title, created=created, note=note, description=description)
def test_non_ascii_text(self): non_ascii_text = u'abcd\xe9' user = test_factories.create_user(username=non_ascii_text) test_factories.create_team_member(self.team, user) self.video.title = non_ascii_text self.video.save() sv = add_subtitles(self.video, 'en', make_subtitle_lines(4), title=non_ascii_text, author=user, description=non_ascii_text, complete=True) video_changed_tasks(self.video.pk, sv.pk) report = BillingReport.objects.create( start_date=sv.created - timedelta(days=1), end_date=sv.created + timedelta(days=1), type=BillingReport.TYPE_NEW, ) report.teams.add(self.team) self.process_report(report)