def rollback(request, version):
is_writelocked = version.subtitle_language.is_writelocked
if not user_can_edit_subtitles(request.user, version.video,
version.subtitle_language.language_code):
messages.error(
request,
_(u"You don't have permission to rollback "
"this language"))
elif is_writelocked:
messages.error(
request,
u'Can not rollback now, because someone is editing subtitles.')
elif not version.next_version():
messages.error(request,
message=u'Can not rollback to the last version')
else:
messages.success(request, message=u'Rollback successful')
version = rollback_to(version.video,
version.subtitle_language.language_code,
version_number=version.version_number,
rollback_author=request.user)
video_changed_tasks.delay(version.video.id, version.id)
return redirect(version.subtitle_language.get_absolute_url() +
'#revisions')
return redirect(version)
def _save_review(self, request, save_for_later, task_id=None, body=None,
approved=None, new_version=None):
"""
If the task performer has edited this version, then we need to
set the task's version to the new one that he has edited.
"""
data = {'task': task_id, 'body': body, 'approved': approved}
form = FinishReviewForm(request, data)
if form.is_valid():
task = form.cleaned_data['task']
task.body = form.cleaned_data['body']
task.approved = form.cleaned_data['approved']
# If there is a new version, update the task's version.
if new_version:
task.new_subtitle_version = new_version
task.save()
if not save_for_later:
if task.approved in Task.APPROVED_FINISHED_IDS:
task.complete()
task.new_subtitle_version.subtitle_language.release_writelock()
task.new_subtitle_version.subtitle_language.followers.add(request.user)
video_changed_tasks.delay(task.team_video.video_id)
else:
return {'error_msg': _(u'\n'.join(flatten_errorlists(form.errors)))}
def _add_lang_to_video(video, props, translated_from=None):
if props.get("is_original", False):
video.subtitle_language().delete()
sub_lang = SubtitleLanguage(
video=video,
is_original=props.get("is_original", False),
is_complete=props.get("is_complete", False),
language=props.get("code"),
has_version=True,
had_version=True,
is_forked=True,
)
sub_lang.save()
num_subs = props.get("num_subs", 0)
if not translated_from:
_add_subtitles(sub_lang, num_subs)
if translated_from:
sub_lang.is_original = False
sub_lang.is_forked = False
sub_lang.standard_language = translated_from
sub_lang.save()
_copy_subtitles(translated_from, sub_lang, num_subs)
for translation_prop in props.get("translations", []):
_add_lang_to_video(video, translation_prop, translated_from=sub_lang)
sub_lang.is_complete = props.get("is_complete", False)
sub_lang.save()
from videos.tasks import video_changed_tasks
video_changed_tasks.delay(sub_lang.video.id)
return sub_lang
def save(self):
subtitles = self.cleaned_data['subtitles']
is_complete = self.cleaned_data.get('is_complete')
text = subtitles.read()
parser = self._get_parser(subtitles.name)(force_unicode(
text,
chardet.detect(text)['encoding']))
sl = self.save_subtitles(parser,
update_video=False,
is_complete=is_complete)
sl.is_complete = is_complete
latest_version = sl.latest_version()
if latest_version and len(latest_version.subtitles()) > 0:
# this will eventually get updated on the async test
# but if it takes too long on html file uplods
# then users will not see the language added which is very
# confusing from a UI point of view
sl.had_version = sl.has_version = True
sl.save()
self.verify_tasks(is_complete)
if latest_version and sl.latest_version():
video_changed_tasks.delay(sl.video_id, sl.latest_version().id)
else:
video_changed_tasks.delay(sl.video_id)
return sl
def rollback(request, version):
# Normally, we only accept POST methods, but the old template code uses
# GET, so we allow that too.
if should_use_old_view(request) or request.method == 'POST':
is_writelocked = version.subtitle_language.is_writelocked
if not user_can_edit_subtitles(
request.user, version.video,
version.subtitle_language.language_code):
messages.error(
request,
_(u"You don't have permission to rollback "
"this language"))
elif is_writelocked:
messages.error(
request,
u'Can not rollback now, because someone is editing subtitles.')
elif not version.next_version():
messages.error(request,
message=u'Can not rollback to the last version')
else:
messages.success(request, message=u'Rollback successful')
version = rollback_to(version.video,
version.subtitle_language.language_code,
version_number=version.version_number,
rollback_author=request.user)
video_changed_tasks.delay(version.video.id, version.id)
return redirect(version.subtitle_language.get_absolute_url() +
'#revisions')
return redirect(version)
def test_filtering(self):
self.assertTrue(Video.objects.count())
for video in Video.objects.all():
video_changed_tasks.delay(video.pk)
reset_solr()
rpc = SearchApiClass()
rdata = RpcMultiValueDict(dict(q=u' ', video_lang='en'))
result = rpc.search(rdata, self.user, testing=True)['sqs']
self.assertTrue(len(result))
for video in VideoIndex.public():
if video.video_language == 'en':
self.assertTrue(
video.object in [item.object for item in result])
rdata = RpcMultiValueDict(dict(q=u' ', langs='en'))
result = rpc.search(rdata, self.user, testing=True)['sqs']
self.assertTrue(len(result))
for video in VideoIndex.public():
if video.languages and 'en' in video.languages:
self.assertTrue(
video.object in [item.object for item in result])
def test_video_languages_count(self):
# TODO: Merge this into the metadata tests file?
video = get_video()
# Start with no languages.
self.assertEqual(video.languages_count, 0)
self.assertEqual(
video.newsubtitlelanguage_set.having_nonempty_tip().count(), 0)
# Create one.
sl_en = make_subtitle_language(video, 'en')
make_subtitle_version(sl_en, [(100, 200, "foo")])
# The query should immediately show it.
self.assertEqual(
video.newsubtitlelanguage_set.having_nonempty_tip().count(), 1)
# But the model object will not.
self.assertEqual(video.languages_count, 0)
# Even if we refresh it, the model still doesn't show it.
video = Video.objects.get(pk=video.pk)
self.assertEqual(video.languages_count, 0)
# Until we run the proper tasks.
video_changed_tasks.delay(video.pk)
test_utils.video_changed_tasks.run_original()
# But we still need to refresh it to see the change.
self.assertEqual(video.languages_count, 0)
video = Video.objects.get(pk=video.pk)
self.assertEqual(video.languages_count, 1)
def _add_lang_to_video(video, props, translated_from=None):
if props.get('is_original', False):
video.subtitle_language().delete()
sub_lang = SubtitleLanguage(
video=video,
is_original=props.get('is_original', False),
is_complete=props.get('is_complete', False),
language=props.get('code'),
has_version=True,
had_version=True,
is_forked=True,
)
sub_lang.save()
num_subs = props.get("num_subs", 0)
if not translated_from:
_add_subtitles(sub_lang, num_subs)
if translated_from:
sub_lang.is_original = False
sub_lang.is_forked = False
sub_lang.standard_language = translated_from
sub_lang.save()
_copy_subtitles(translated_from, sub_lang, num_subs)
for translation_prop in props.get("translations", []):
_add_lang_to_video(video, translation_prop, translated_from=sub_lang)
sub_lang.is_complete = props.get("is_complete", False)
sub_lang.save()
from videos.tasks import video_changed_tasks
video_changed_tasks.delay(sub_lang.video.id)
return sub_lang
def submit(self, request):
"""Handle form submission."""
with transaction.atomic():
if self.subtitle_language.is_writelocked:
messages.error(request,
_(u'Subtitles are currently being edited'))
return
self.do_submit(request)
video_changed_tasks.delay(self.video.pk)
def complete_approve_tasks(tasks):
lang_ct = ContentType.objects.get_for_model(SubtitleLanguage)
video_ids = set()
for task in tasks:
task.do_complete_approve(lang_ct=lang_ct)
api_subtitles_approved.send(task.get_subtitle_version())
video_ids.add(task.team_video.video_id)
for video_id in video_ids:
video_changed_tasks.delay(video_id)
def save(self):
# If the primary audio language code was given, we adjust it on the
# video NOW, before saving the subtitles, so that the pipeline can take
# it into account when determining task types.
self._save_primary_audio_language_code()
language_code = self.cleaned_data['language_code']
from_language_code = self.cleaned_data['from_language_code']
complete = self.cleaned_data['complete']
subtitles = self._parsed_subtitles
subtitles.set_language(language_code)
if from_language_code:
# If this is a translation, its subtitles should use the timing data
# from the source. We know that the source has at least as many
# subtitles as the new version, so we can just match them up
# first-come, first-serve.
source_subtitles = self.from_sv.get_subtitles()
i = 0
# instead of translating to subtitle_items, we're updating the
# dfxp elements in place. This guarantees no monkey business with
# escaping / styling
for old, new in izip(source_subtitles.subtitle_items(), subtitles.get_subtitles()):
subtitles.update(i, from_ms=old.start_time, to_ms=old.end_time)
i += 1
else:
# Otherwise we can just use the subtitles the user uploaded as-is.
# No matter what, text files that aren't translations cannot be
# complete because they don't have timing data.
if self.extension == 'txt':
complete = False
title, description, metadata = self._find_title_description_metadata(language_code)
parents = self._find_parents(from_language_code)
version = pipeline.add_subtitles(
self.video, language_code, subtitles,
title=title, description=description, author=self.user,
parents=parents, committer=self.user, complete=complete,
metadata=metadata, origin=ORIGIN_UPLOAD)
# Handle forking SubtitleLanguages that were translations when
# a standalone version is uploaded.
#
# For example: assume there is a French translation of English.
# Uploading a "straight from video" version of French should fork it.
sl = version.subtitle_language
if not from_language_code and is_dependent(sl):
sl.fork()
# TODO: Pipeline this.
video_changed_tasks.delay(sl.video_id, version.id)
return version
def finished_subtitles(self, request, session_pk, subtitles=None,
new_title=None, completed=None,
forked=False,
throw_exception=False):
session = SubtitlingSession.objects.get(pk=session_pk)
if not request.user.is_authenticated():
return { 'response': 'not_logged_in' }
if not session.language.can_writelock(request):
return { "response" : "unlockable" }
if not session.matches_request(request):
return { "response" : "does not match request" }
if throw_exception:
raise Exception('purposeful exception for testing')
from apps.teams.moderation import is_moderated, user_can_moderate
language = session.language
new_version = None
if subtitles is not None and \
(len(subtitles) > 0 or language.latest_version(public_only=False) is not None):
new_version = self._create_version_from_session(session, request.user, forked)
new_version.save()
self._save_subtitles(
new_version.subtitle_set, subtitles, new_version.is_forked)
language.release_writelock()
if completed is not None:
language.is_complete = completed
if new_title is not None:
language.title = new_title
language.save()
if new_version is not None:
video_changed_tasks.delay(language.video.id, new_version.id)
else:
video_changed_tasks.delay(language.video.id)
# we have a default user message, since the UI lets users save non
# changed subs, but the backend will realize and will not save that
# version. In those cases, we want to show the defatul user message.
user_message = "Thank you for uploading. It will take a minute or so for your subtitles to appear."
if new_version is not None and new_version.version_no == 0:
user_message = "Thank you for uploading. It will take a minute or so for your subtitles to appear."
elif new_version and is_moderated(new_version):
if user_can_moderate(language.video, request.user) is False:
user_message = """This video is moderated by %s.
You will not see your subtitles in our widget when you leave this page, they will only appear on our site. We have saved your work for the team moderator to review. After they approve your subtitles, they will show up on our site and in the widget.
""" % (new_version.video.moderated_by.name)
return {
'user_message': user_message,
'response': 'ok' }
def setUp(self):
from teams.tests import reset_solr
self.video = Video.objects.all()[0]
self.user = User.objects.get(username=u'admin')
self.DEFAULTS = {
'description': 'A request description just for test',
'track': True,
}
video_changed_tasks.delay(self.video)
reset_solr()
def save_finished(self,
request,
user,
session,
subtitles,
new_title=None,
completed=None,
forked=False,
new_description=None,
new_metadata=None,
task_id=None,
task_notes=None,
task_approved=None,
task_type=None,
save_for_later=None):
# TODO: lock all this in a transaction please!
language = session.language
new_version = self._get_new_version_for_save(subtitles, language,
session, user, new_title,
new_description,
new_metadata,
save_for_later)
language.release_writelock()
# do this here, before _update_language_a... changes it ;)
complete_changed = bool(completed) != language.subtitles_complete
self._update_language_attributes_for_save(language, completed, session,
forked)
if new_version:
video_changed_tasks.delay(language.video.id, new_version.id)
else:
video_changed_tasks.delay(language.video.id)
api_video_edited.send(language.video)
if completed and complete_changed:
# not a new version, but if this just got marked as complete
# we want to push this to the third parties:
subtitles_complete_changed(language.pk)
user_message = self._get_user_message_for_save(
user, language, language.subtitles_complete)
error = self._save_tasks_for_save(request, save_for_later, language,
new_version,
language.subtitles_complete, task_id,
task_type, task_notes, task_approved)
if error:
return error
return {'response': 'ok', 'user_message': user_message}
def setUp(self):
from teams.tests import reset_solr
self.video = Video.objects.all()[0]
self.user = User.objects.get(username=u'admin')
self.DEFAULTS = {
'description': 'A request description just for test',
'track': True,
}
video_changed_tasks.delay(self.video)
reset_solr()
def rollback(request, pk):
version = get_object_or_404(SubtitleVersion, pk=pk)
is_writelocked = version.language.is_writelocked
if is_writelocked:
messages.error(request, u'Can not rollback now, because someone is editing subtitles.')
elif not version.next_version():
messages.error(request, message=u'Can not rollback to the last version')
else:
messages.success(request, message=u'Rollback successful')
version = version.rollback(request.user)
video_changed_tasks.delay(version.video.id, version.id)
return redirect(version.language.get_absolute_url()+'#revisions')
return redirect(version)
def save_subtitle(video, language, parser, user=None, update_video=True):
from videos.models import SubtitleVersion, Subtitle
from videos.tasks import video_changed_tasks
key = str(uuid4()).replace('-', '')
video._make_writelock(user, key)
video.save()
try:
old_version = language.subtitleversion_set.all()[:1].get()
version_no = old_version.version_no + 1
except ObjectDoesNotExist:
old_version = None
version_no = 0
version = None
if not is_version_same(old_version, parser):
version = SubtitleVersion(language=language,
version_no=version_no,
datetime_started=datetime.now(),
user=user,
note=u'Uploaded',
is_forked=True,
time_change=1,
text_change=1)
version.save()
ids = []
for i, item in enumerate(parser):
id = int(random.random() * 10e12)
while id in ids:
id = int(random.random() * 10e12)
ids.append(id)
caption = Subtitle(**item)
caption.version = version
caption.subtitle_id = str(id)
caption.subtitle_order = i + 1
caption.save()
language.video.release_writelock()
language.video.save()
translations = video.subtitlelanguage_set.filter(
standard_language=language)
[t.fork(from_version=old_version, user=user) for t in translations]
if update_video:
video_changed_tasks.delay(video.id,
None if version is None else version.id)
return language
def save_subtitle(video, language, parser, user=None, update_video=True):
from videos.models import SubtitleVersion, Subtitle
from videos.tasks import video_changed_tasks
key = str(uuid4()).replace("-", "")
video._make_writelock(user, key)
video.save()
try:
old_version = language.subtitleversion_set.all()[:1].get()
version_no = old_version.version_no + 1
except ObjectDoesNotExist:
old_version = None
version_no = 0
version = None
if not is_version_same(old_version, parser):
version = SubtitleVersion(
language=language,
version_no=version_no,
datetime_started=datetime.now(),
user=user,
note=u"Uploaded",
is_forked=True,
time_change=1,
text_change=1,
)
version.save()
ids = []
for i, item in enumerate(parser):
id = int(random.random() * 10e12)
while id in ids:
id = int(random.random() * 10e12)
ids.append(id)
caption = Subtitle(**item)
caption.version = version
caption.subtitle_id = str(id)
caption.subtitle_order = i + 1
caption.save()
language.video.release_writelock()
language.video.save()
translations = video.subtitlelanguage_set.filter(standard_language=language)
[t.fork(from_version=old_version, user=user) for t in translations]
if update_video:
video_changed_tasks.delay(video.id, None if version is None else version.id)
return language
def save(self):
subtitles = self.cleaned_data['subtitles']
text = subtitles.read()
parser = self._get_parser(subtitles.name)(force_unicode(text, chardet.detect(text)['encoding']))
sl = self.save_subtitles(parser, update_video=False)
is_complete = self.cleaned_data.get('is_complete')
sl.is_complete = is_complete
if len(sl.latest_version().subtitles()) > 0:
# this will eventually get updated on the async test
# but if it takes too long on html file uplods
# then users will not see the language added which is very
# confusing from a UI point of view
sl.had_version = sl.has_version = True
sl.save()
video_changed_tasks.delay(sl.video_id, sl.latest_version().id)
return sl
def rollback(request, pk):
version = get_object_or_404(SubtitleVersion, pk=pk)
is_writelocked = version.language.is_writelocked
if is_writelocked:
messages.error(
request,
u'Can not rollback now, because someone is editing subtitles.')
elif not version.next_version():
messages.error(request,
message=u'Can not rollback to the last version')
else:
messages.success(request, message=u'Rollback successful')
version = version.rollback(request.user)
video_changed_tasks.delay(version.video.id, version.id)
return redirect(version.language.get_absolute_url() + '#revisions')
return redirect(version)
def save_finished(self,
request,
user,
session,
subtitles,
new_title=None,
completed=None,
forked=False,
new_description=None,
task_id=None,
task_notes=None,
task_approved=None,
task_type=None,
save_for_later=None):
# TODO: lock all this in a transaction please!
language = session.language
new_version = self._get_new_version_for_save(subtitles, language,
session, user, forked,
new_title,
new_description,
save_for_later)
language.release_writelock()
self._update_language_attributes_for_save(language, completed)
if new_version:
video_changed_tasks.delay(language.video.id, new_version.id)
api_subtitles_edited.send(new_version)
else:
video_changed_tasks.delay(language.video.id)
api_video_edited.send(language.video)
is_complete = language.is_complete or language.calculate_percent_done(
) == 100
user_message = self._get_user_message_for_save(user, language,
is_complete)
error = self._save_tasks_for_save(request, save_for_later, language,
new_version, is_complete, task_id,
task_type, task_notes, task_approved)
if error:
return error
return {'response': 'ok', 'user_message': user_message}
def _complete_task(user, video, subtitle_language, saved_version, approved):
team_video = video.get_team_video()
task = (team_video.task_set.incomplete_review_or_approve().get(
language=subtitle_language.language_code))
if task.assignee is None and can_assign_task(task, user):
task.assignee = user
elif task.assignee != user:
raise ValueError("Task not assigned to user")
task.new_subtitle_version = subtitle_language.get_tip()
task.approved = approved
task.complete()
if saved_version is None:
if saved_version is None:
version_id = None
else:
version_id = saved_version.id
video_changed_tasks.delay(team_video.video_id, version_id)
def save_finished(self,
user,
session,
subtitles,
new_title=None,
completed=None,
forked=False):
from apps.teams.moderation import is_moderated, user_can_moderate
language = session.language
new_version = None
if subtitles is not None and \
(len(subtitles) > 0 or language.latest_version(public_only=False) is not None):
new_version = self._create_version_from_session(
session, user, forked)
new_version.save()
self._save_subtitles(new_version.subtitle_set, subtitles,
new_version.is_forked)
language.release_writelock()
if completed is not None:
language.is_complete = completed
if new_title is not None:
language.title = new_title
language.save()
if new_version is not None:
video_changed_tasks.delay(language.video.id, new_version.id)
else:
video_changed_tasks.delay(language.video.id)
# we have a default user message, since the UI lets users save non
# changed subs, but the backend will realize and will not save that
# version. In those cases, we want to show the defatul user message.
user_message = "Your changes have been saved. It will take a minute or so for your subtitles to appear."
if new_version is not None and new_version.version_no == 0:
user_message = "Your changes have been saved. It will take a minute or so for your subtitles to appear."
elif new_version and is_moderated(new_version):
if user_can_moderate(language.video, user) is False:
user_message = """This video is moderated by %s.
You will not see your subtitles in our widget when you leave this page, they will only appear on our site. We have saved your work for the team moderator to review. After they approve your subtitles, they will show up on our site and in the widget.
""" % (new_version.video.moderated_by.name)
return {'user_message': user_message, 'response': 'ok'}
def finished_subtitles(self, request, session_pk, subtitles=None,
new_title=None, completed=None,
forked=False,
throw_exception=False):
session = SubtitlingSession.objects.get(pk=session_pk)
if not request.user.is_authenticated():
return { 'response': 'not_logged_in' }
if not session.language.can_writelock(request):
return { "response" : "unlockable" }
if not session.matches_request(request):
return { "response" : "does not match request" }
if throw_exception:
raise Exception('purposeful exception for testing')
language = session.language
new_version = None
if subtitles is not None and \
(len(subtitles) > 0 or language.latest_version() is not None):
new_version = self._create_version_from_session(session, request.user, forked)
new_version.save()
self._save_subtitles(
new_version.subtitle_set, subtitles, new_version.is_forked)
language.release_writelock()
if completed is not None:
language.is_complete = completed
if new_title is not None:
language.title = new_title
language.save()
if new_version is not None:
video_changed_tasks.delay(language.video.id, new_version.id)
else:
video_changed_tasks.delay(language.video.id)
user_message = None
if new_version is not None and new_version.version_no == 0:
user_message = {
"body": ("Thank you for uploading. It will take a minute "
"or so for your subtitles to appear.") }
return {
'_user_message': user_message,
'response': 'ok' }
def rollback(request, version):
if version.video.is_moderated:
return HttpResponseForbidden(
"Moderated videos cannot be rollbacked, they need to be unpublished"
)
is_writelocked = version.language.is_writelocked
if is_writelocked:
messages.error(
request,
u'Can not rollback now, because someone is editing subtitles.')
elif not version.next_version():
messages.error(request,
message=u'Can not rollback to the last version')
else:
messages.success(request, message=u'Rollback successful')
version = version.rollback(request.user)
video_changed_tasks.delay(version.video.id, version.id)
return redirect(version.language.get_absolute_url() + '#revisions')
return redirect(version)
def save_finished(self, user, session, subtitles, new_title=None, completed=None, forked=False):
from apps.teams.moderation import is_moderated, user_can_moderate
language = session.language
new_version = None
if subtitles is not None and (len(subtitles) > 0 or language.latest_version(public_only=False) is not None):
new_version = self._create_version_from_session(session, user, forked)
new_version.save()
self._save_subtitles(new_version.subtitle_set, subtitles, new_version.is_forked)
language.release_writelock()
if completed is not None:
language.is_complete = completed
if new_title is not None:
language.title = new_title
language.save()
if new_version is not None:
video_changed_tasks.delay(language.video.id, new_version.id)
else:
video_changed_tasks.delay(language.video.id)
# we have a default user message, since the UI lets users save non
# changed subs, but the backend will realize and will not save that
# version. In those cases, we want to show the defatul user message.
user_message = "Your changes have been saved. It will take a minute or so for your subtitles to appear."
if new_version is not None and new_version.version_no == 0:
user_message = "Your changes have been saved. It will take a minute or so for your subtitles to appear."
elif new_version and is_moderated(new_version):
if user_can_moderate(language.video, user) is False:
user_message = """This video is moderated by %s.
You will not see your subtitles in our widget when you leave this page, they will only appear on our site. We have saved your work for the team moderator to review. After they approve your subtitles, they will show up on our site and in the widget.
""" % (
new_version.video.moderated_by.name
)
return {"user_message": user_message, "response": "ok"}
def test_filtering(self):
self.assertTrue(Video.objects.count())
for video in Video.objects.all():
video_changed_tasks.delay(video.pk)
reset_solr()
rpc = SearchApiClass()
rdata = RpcMultiValueDict(dict(q=u' ', video_lang='en'))
result = rpc.search(rdata, self.user, testing=True)['sqs']
self.assertTrue(len(result))
for video in SearchQuerySet().models(Video):
if video.video_language == 'en':
self.assertTrue(video.object in [item.object for item in result])
rdata = RpcMultiValueDict(dict(q=u' ', langs='en'))
result = rpc.search(rdata, self.user, testing=True)['sqs']
self.assertTrue(len(result))
for video in SearchQuerySet().models(Video):
if video.languages and 'en' in video.languages:
self.assertTrue(video.object in [item.object for item in result])
def _refresh(video):
video_changed_tasks.delay(video.pk)
test_utils.video_changed_tasks.run_original()
return Video.objects.get(pk=video.pk)
def _refresh(video):
video_changed_tasks.delay(video.pk)
return Video.objects.get(pk=video.pk)
def save_subtitles_for_lang(lang, video_pk, youtube_id):
from django.utils.encoding import force_unicode
from videos.models import Video
from videos.tasks import video_changed_tasks
from subtitles.pipeline import add_subtitles
from subtitles.models import ORIGIN_IMPORTED
yt_lc = lang.get('lang_code')
# TODO: Make sure we can store all language data given to us by Youtube.
# Right now, the bcp47 codec will refuse data it can't reliably parse.
try:
lc = LanguageCode(yt_lc, "bcp47").encode("unisubs")
except KeyError:
logger.warn("Youtube import did not find language code",
extra={
"data": {
"language_code": yt_lc,
"youtube_id": youtube_id,
}
})
return
if not lc in SUPPORTED_LANGUAGE_CODES:
logger.warn(
"Youtube import did not find language code",
extra={"data": {
"language_code": lc,
"youtube_id": youtube_id,
}})
return
try:
video = Video.objects.get(pk=video_pk)
except Video.DoesNotExist:
return
url = u'http://www.youtube.com/api/timedtext?v=%s&lang=%s&name=%s&fmt=srt'
url = url % (youtube_id, yt_lc, urlquote(lang.get('name', u'')))
xml = YoutubeVideoType._get_response_from_youtube(url, return_string=True)
if not bool(xml):
return
xml = force_unicode(xml, 'utf-8')
subs = babelsubs.parsers.discover('srt').parse(xml).to_internal()
version = add_subtitles(video,
lc,
subs,
note="From youtube",
complete=True,
origin=ORIGIN_IMPORTED)
# do not pass a version_id else, we'll trigger emails for those edits
video_changed_tasks.delay(video.pk)
Meter('youtube.lang_imported').inc()
from apps.teams.models import BillingRecord
# there is a caveat here, if running with CELERY_ALWAYS_EAGER,
# this is called before there's a team video, and the billing records won't
# be created. On the real world, it should be safe to assume that between
# calling the youtube api and the db insertion, we'll get this called
# when the video is already part of a team
BillingRecord.objects.insert_record(version)
def save_subtitle(video,
language,
parser,
user=None,
update_video=True,
forks=True,
as_forked=True,
translated_from=None):
from videos.models import SubtitleVersion, Subtitle, SubtitleMetadata
from videos.tasks import video_changed_tasks
key = str(uuid4()).replace('-', '')
if language.is_original:
as_forked = True
video._make_writelock(user, key)
video.save()
try:
old_version = language.subtitleversion_set.all()[:1].get()
version_no = old_version.version_no + 1
except ObjectDoesNotExist:
old_version = None
version_no = 0
version = None
if not is_version_same(old_version, parser):
forked_from = as_forked and translated_from and translated_from.version(
)
title = old_version.title if old_version and old_version.title else video.get_title_display(
)
description = old_version.description if old_version and old_version.description else video.get_description_display(
)
version = SubtitleVersion(language=language,
version_no=version_no,
datetime_started=datetime.now(),
user=user,
note=u'Uploaded',
is_forked=as_forked,
time_change=1,
text_change=1,
forked_from=forked_from,
title=title,
description=description)
if len(parser) > 0:
version.has_version = True
version.save()
ids = []
for i, item in enumerate(parser):
id = int(random.random() * 10e12)
while id in ids:
id = int(random.random() * 10e12)
ids.append(id)
metadata = item.pop('metadata', None)
caption = Subtitle(**item)
caption.version = version
caption.datetime_started = datetime.now()
caption.subtitle_id = str(id)
caption.subtitle_order = i + 1
caption.save()
if metadata:
for name, value in metadata.items():
SubtitleMetadata(subtitle=caption, key=name,
data=value).save()
version = version or old_version
if version.is_forked != as_forked:
version.is_forked = as_forked
version.save()
if version.user != user:
# we might be only uptading the user , as in per bulk imports
version.user = user
version.save()
language.video.release_writelock()
language.video.save()
if forks:
translations = video.subtitlelanguage_set.filter(
standard_language=language)
[t.fork(from_version=old_version, user=user) for t in translations]
if update_video:
video_changed_tasks.delay(video.id,
None if version is None else version.id)
return language
def save_subtitles_for_lang(lang, video_pk, youtube_id):
from videos.models import Video
lc = lang.get('lang_code')
if not lc in SUPPORTED_LANGUAGES_DICT:
return
try:
video = Video.objects.get(pk=video_pk)
except Video.DoesNotExist:
return
from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle
url = u'http://www.youtube.com/api/timedtext?v=%s&lang=%s&name=%s'
url = url % (youtube_id, lc, urlquote(lang.get('name', u'')))
xml = YoutubeVideoType._get_response_from_youtube(url)
if not xml:
return
parser = YoutubeXMLParser(xml)
if not parser:
return
language, create = SubtitleLanguage.objects.get_or_create(video=video,
language=lc)
language.is_original = False
language.is_forked = True
language.save()
try:
version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
.get().version_no + 1
except SubtitleVersion.DoesNotExist:
version_no = 0
version = SubtitleVersion(language=language)
version.version_no = version_no
version.datetime_started = datetime.now()
version.user = User.get_youtube_anonymous()
version.note = u'From youtube'
version.is_forked = True
version.save()
for i, item in enumerate(parser):
subtitle = Subtitle()
subtitle.subtitle_text = item['subtitle_text']
subtitle.start_time = item['start_time']
subtitle.end_time = item['end_time']
subtitle.version = version
subtitle.subtitle_id = int(random.random() * 10e12)
subtitle.subtitle_order = i + 1
subtitle.save()
assert subtitle.start_time or subtitle.end_time, item['subtitle_text']
version.finished = True
version.save()
language.has_version = True
language.had_version = True
language.is_complete = True
language.save()
from videos.tasks import video_changed_tasks
video_changed_tasks.delay(video.pk)
def save_subtitles_for_lang(lang, video_pk, youtube_id):
from videos.models import Video
yt_lc = lang.get('lang_code')
lc = LanguageCode(yt_lc, "youtube").encode("unisubs")
if not lc in SUPPORTED_LANGUAGE_CODES:
logger.warn(
"Youtube import did not find language code",
extra={"data": {
"language_code": lc,
"youtube_id": youtube_id,
}})
return
try:
video = Video.objects.get(pk=video_pk)
except Video.DoesNotExist:
return
from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle
url = u'http://www.youtube.com/api/timedtext?v=%s&lang=%s&name=%s'
url = url % (youtube_id, yt_lc, urlquote(lang.get('name', u'')))
xml = YoutubeVideoType._get_response_from_youtube(url)
if xml is None:
return
parser = YoutubeXMLParser(xml)
if not parser:
return
language, create = SubtitleLanguage.objects.get_or_create(
video=video, language=lc, defaults={
'created': datetime.now(),
})
language.is_original = False
language.is_forked = True
language.save()
try:
version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
.get().version_no + 1
except SubtitleVersion.DoesNotExist:
version_no = 0
version = SubtitleVersion(language=language)
version.title = video.title
version.description = video.description
version.version_no = version_no
version.datetime_started = datetime.now()
version.user = User.get_anonymous()
version.note = u'From youtube'
version.is_forked = True
version.save()
for i, item in enumerate(parser):
subtitle = Subtitle()
subtitle.subtitle_text = item['subtitle_text']
subtitle.start_time = item['start_time']
subtitle.end_time = item['end_time']
subtitle.version = version
subtitle.subtitle_id = int(random.random() * 10e12)
subtitle.subtitle_order = i + 1
subtitle.save()
assert subtitle.start_time or subtitle.end_time, item['subtitle_text']
version.finished = True
version.save()
language.has_version = True
language.had_version = True
language.is_complete = True
language.save()
from videos.tasks import video_changed_tasks
video_changed_tasks.delay(video.pk)
def save_subtitles_for_lang(lang, video_pk, youtube_id):
from videos.models import Video
lc = lang.get('lang_code')
if not lc in SUPPORTED_LANGUAGES_DICT:
return
try:
video = Video.objects.get(pk=video_pk)
except Video.DoesNotExist:
return
from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle
url = u'http://www.youtube.com/api/timedtext?v=%s&lang=%s&name=%s'
url = url % (youtube_id, lc, urlquote(lang.get('name', u'')))
xml = YoutubeVideoType._get_response_from_youtube(url)
if xml is None:
return
parser = YoutubeXMLParser(xml)
if not parser:
return
language, create = SubtitleLanguage.objects.get_or_create(video=video, language=lc)
language.is_original = False
language.is_forked = True
language.save()
try:
version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
.get().version_no + 1
except SubtitleVersion.DoesNotExist:
version_no = 0
version = SubtitleVersion(language=language)
version.version_no = version_no
version.datetime_started = datetime.now()
version.user = User.get_anonymous()
version.note = u'From youtube'
version.is_forked = True
version.save()
for i, item in enumerate(parser):
subtitle = Subtitle()
subtitle.subtitle_text = item['subtitle_text']
subtitle.start_time = item['start_time']
subtitle.end_time = item['end_time']
subtitle.version = version
subtitle.subtitle_id = int(random.random()*10e12)
subtitle.subtitle_order = i+1
subtitle.save()
assert subtitle.start_time or subtitle.end_time, item['subtitle_text']
version.finished = True
version.save()
language.has_version = True
language.had_version = True
language.is_complete = True
language.save()
from videos.tasks import video_changed_tasks
video_changed_tasks.delay(video.pk)
def reset_metadata(request, video_id):
video = get_object_or_404(Video, video_id=video_id)
video_changed_tasks.delay(video.id)
return HttpResponse('ok')
def reset_metadata(request, video_id):
video = get_object_or_404(Video, video_id=video_id)
video_changed_tasks.delay(video.id)
return HttpResponse('ok')
def delete_model(self, request, obj):
video = obj.video
super(SubtitleLanguageAdmin, self).delete_model(request, obj)
video_changed_tasks.delay(video.pk)
