def save_subtitles(self, parser):
video = self.cleaned_data['video']
key = str(uuid4()).replace('-', '')
video._make_writelock(self.user, key)
video.save()
if not video.has_original_language():
self._save_original_language(
video, self.cleaned_data['video_language'])
language = video.subtitle_language(self.cleaned_data['language'])
if not language:
language = SubtitleLanguage(video=video, is_original=False, is_forked=True)
language.language = self.cleaned_data['language']
language.save()
try:
old_version = language.subtitleversion_set.all()[:1].get()
version_no = old_version.version_no + 1
except ObjectDoesNotExist:
old_version = None
version_no = 0
if not self.is_version_same(old_version, parser):
version = SubtitleVersion(
language=language, version_no=version_no,
datetime_started=datetime.now(), user=self.user,
note=u'Uploaded', is_forked=True, time_change=1, text_change=1)
version.save()
ids = []
for i, item in enumerate(parser):
id = int(random.random()*10e12)
while id in ids:
id = int(random.random()*10e12)
ids.append(id)
caption = Subtitle(**item)
caption.version = version
caption.subtitle_id = str(id)
caption.subtitle_order = i+1
caption.save()
version.save()
version.update_percent_done()
language.update_complete_state()
language.save()
if language.is_original:
video.update_complete_state()
video.release_writelock()
video.save()
return language
def save_subtitles(self, parser):
video = self.cleaned_data['video']
key = str(uuid4()).replace('-', '')
video._make_writelock(self.user, key)
video.save()
if not video.has_original_language():
self._save_original_language(video,
self.cleaned_data['video_language'])
language = self._find_appropriate_language(
video, self.cleaned_data['language'])
try:
old_version = language.subtitleversion_set.all()[:1].get()
version_no = old_version.version_no + 1
except ObjectDoesNotExist:
old_version = None
version_no = 0
if not self.is_version_same(old_version, parser):
version = SubtitleVersion(language=language,
version_no=version_no,
datetime_started=datetime.now(),
user=self.user,
note=u'Uploaded',
is_forked=True,
time_change=1,
text_change=1)
version.save()
ids = []
for i, item in enumerate(parser):
id = int(random.random() * 10e12)
while id in ids:
id = int(random.random() * 10e12)
ids.append(id)
caption = Subtitle(**item)
caption.version = version
caption.subtitle_id = str(id)
caption.subtitle_order = i + 1
caption.save()
version.save()
version.update_percent_done()
language.update_complete_state()
language.save()
if language.is_original:
video.update_complete_state()
video.release_writelock()
video.save()
return language
def _get_subtitles_from_youtube(self, video_obj):
from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle
url = 'http://www.youtube.com/watch_ajax?action_get_caption_track_all&v=%s' % video_obj.youtube_videoid
h = httplib2.Http()
resp, content = h.request(url, "GET")
if resp.status < 200 or resp.status >= 400:
logger.warning("Youtube subtitles error",
extra={
'data': {
"url": url,
"video_id": video_obj.youtube_videoid,
"status_code": resp.status,
"response": content
}
})
return
parser = YoutubeSubtitleParser(content)
if not parser:
return
language, create = SubtitleLanguage.objects.get_or_create(
video=video_obj, language=parser.language)
language.is_original = False
language.is_forked = True
language.save()
try:
version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
.get().version_no + 1
except SubtitleVersion.DoesNotExist:
version_no = 0
version = SubtitleVersion(language=language)
version.version_no = version_no
version.datetime_started = datetime.now()
version.user = User.get_youtube_anonymous()
version.note = u'From youtube'
version.is_forked = True
version.save()
for i, item in enumerate(parser):
subtitle = Subtitle(**item)
subtitle.version = version
subtitle.subtitle_id = int(random.random() * 10e12)
subtitle.subtitle_order = i + 1
subtitle.save()
version.finished = True
version.save()
language.had_version = True
language.is_complete = True
language.save()
def _get_subtitles_from_youtube(self, video_obj):
from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle
url = 'http://www.youtube.com/watch_ajax?action_get_caption_track_all&v=%s' % video_obj.youtube_videoid
h = httplib2.Http()
resp, content = h.request(url, "GET")
if resp.status < 200 or resp.status >= 400:
logger.warning("Youtube subtitles error", extra={
'data': {
"url": url,
"video_id": video_obj.youtube_videoid,
"status_code": resp.status,
"response": content
}
})
return
parser = YoutubeSubtitleParser(content)
if not parser:
return
language, create = SubtitleLanguage.objects.get_or_create(video=video_obj, language = parser.language)
language.is_original = False
language.is_forked = True
language.save()
try:
version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
.get().version_no + 1
except SubtitleVersion.DoesNotExist:
version_no = 0
version = SubtitleVersion(language=language)
version.version_no = version_no
version.datetime_started = datetime.now()
version.user = User.get_youtube_anonymous()
version.note = u'From youtube'
version.is_forked = True
version.save()
for i, item in enumerate(parser):
subtitle = Subtitle(**item)
subtitle.version = version
subtitle.subtitle_id = int(random.random()*10e12)
subtitle.subtitle_order = i+1
subtitle.save()
version.finished = True
version.save()
language.had_version = True
language.is_complete = True
language.save()
def test_adding_to_original(self):
latest_version = self.original_language.latest_version()
s = Subtitle()
s.version = latest_version
s.subtitle_id = 'asdasdsadasdasdasd'
s.subtitle_order = 5
s.subtitle_text = 'new subtitle'
s.start_time = 50
s.end_time = 51
s.save()
self.translation.update_percent_done()
self.assertEqual(self.translation.percent_done, 4/5.*100)
def test_adding_to_original(self):
latest_version = self.original_language.latest_version()
s = Subtitle()
s.version = latest_version
s.subtitle_id = 'asdasdsadasdasdasd'
s.subtitle_order = 5
s.subtitle_text = 'new subtitle'
s.start_time = 50
s.end_time = 51
s.save()
self.translation.update_percent_done()
self.assertEqual(self.translation.percent_done, 4/5.*100)
def test_many_subtitles(self):
latest_version = self.original_language.latest_version()
for i in range(5, 450):
s = Subtitle()
s.version = latest_version
s.subtitle_id = 'sadfdasf%s' % i
s.subtitle_order = i
s.start_time = 50 + i
s.end_time = 51 + i
s.save()
self.translation.update_percent_done()
self.assertEqual(self.translation.percent_done, 0)
def test_many_subtitles(self):
latest_version = self.original_language.latest_version()
for i in range(5, 450):
s = Subtitle()
s.version = latest_version
s.subtitle_id = 'sadfdasf%s' % i
s.subtitle_order = i
s.start_time = 50 + i
s.end_time = 51 + i
s.save()
self.translation.update_percent_done()
self.assertEqual(self.translation.percent_done, 0)
def save_subtitle(video, language, parser, user=None, update_video=True):
from videos.models import SubtitleVersion, Subtitle
from videos.tasks import video_changed_tasks
key = str(uuid4()).replace('-', '')
video._make_writelock(user, key)
video.save()
try:
old_version = language.subtitleversion_set.all()[:1].get()
version_no = old_version.version_no + 1
except ObjectDoesNotExist:
old_version = None
version_no = 0
version = None
if not is_version_same(old_version, parser):
version = SubtitleVersion(language=language,
version_no=version_no,
datetime_started=datetime.now(),
user=user,
note=u'Uploaded',
is_forked=True,
time_change=1,
text_change=1)
version.save()
ids = []
for i, item in enumerate(parser):
id = int(random.random() * 10e12)
while id in ids:
id = int(random.random() * 10e12)
ids.append(id)
caption = Subtitle(**item)
caption.version = version
caption.subtitle_id = str(id)
caption.subtitle_order = i + 1
caption.save()
language.video.release_writelock()
language.video.save()
translations = video.subtitlelanguage_set.filter(
standard_language=language)
[t.fork(from_version=old_version, user=user) for t in translations]
if update_video:
video_changed_tasks.delay(video.id,
None if version is None else version.id)
return language
def save_subtitle(video, language, parser, user=None, update_video=True):
from videos.models import SubtitleVersion, Subtitle
from videos.tasks import video_changed_tasks
key = str(uuid4()).replace("-", "")
video._make_writelock(user, key)
video.save()
try:
old_version = language.subtitleversion_set.all()[:1].get()
version_no = old_version.version_no + 1
except ObjectDoesNotExist:
old_version = None
version_no = 0
version = None
if not is_version_same(old_version, parser):
version = SubtitleVersion(
language=language,
version_no=version_no,
datetime_started=datetime.now(),
user=user,
note=u"Uploaded",
is_forked=True,
time_change=1,
text_change=1,
)
version.save()
ids = []
for i, item in enumerate(parser):
id = int(random.random() * 10e12)
while id in ids:
id = int(random.random() * 10e12)
ids.append(id)
caption = Subtitle(**item)
caption.version = version
caption.subtitle_id = str(id)
caption.subtitle_order = i + 1
caption.save()
language.video.release_writelock()
language.video.save()
translations = video.subtitlelanguage_set.filter(standard_language=language)
[t.fork(from_version=old_version, user=user) for t in translations]
if update_video:
video_changed_tasks.delay(video.id, None if version is None else version.id)
return language
def save(self):
subtitles = self.cleaned_data["subtitles"]
video = self.cleaned_data["video"]
key = str(uuid4()).replace("-", "")
video._make_writelock(self.user, key)
video.save()
language = video.subtitle_language()
if not language:
language = SubtitleLanguage(video=video, is_original=True, is_complete=True)
language.save()
latest_captions = language.latest_version()
if latest_captions is None:
version_no = 0
else:
version_no = latest_captions.version_no + 1
version = SubtitleVersion(
language=language, version_no=version_no, datetime_started=datetime.now(), user=self.user, note=u"Uploaded"
)
version.save()
text = subtitles.read()
parser = self._get_parser(subtitles.name)(force_unicode(text, chardet.detect(text)["encoding"]))
ids = []
for i, item in enumerate(parser):
id = int(random.random() * 10e12)
while id in ids:
id = int(random.random() * 10e12)
ids.append(id)
caption = Subtitle(**item)
caption.version = version
caption.subtitle_id = str(id)
caption.sub_order = i + 1
caption.save()
version.finished = True
version.save()
video.release_writelock()
video.save()
def _get_subtitles_from_youtube(self, video_obj):
from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle
url = 'http://www.youtube.com/watch_ajax?action_get_caption_track_all&v=%s' % video_obj.youtube_videoid
d = urllib.urlopen(url)
parser = YoutubeSubtitleParser(d.read())
if not parser:
return
language, create = SubtitleLanguage.objects.get_or_create(
video=video_obj, language=parser.language)
language.is_original = False
language.is_forked = True
language.save()
try:
version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
.get().version_no + 1
except SubtitleVersion.DoesNotExist:
version_no = 0
version = SubtitleVersion(language=language)
version.version_no = version_no
version.datetime_started = datetime.now()
version.user = User.get_youtube_anonymous()
version.note = u'From youtube'
version.is_forked = True
version.save()
for i, item in enumerate(parser):
subtitle = Subtitle(**item)
subtitle.version = version
subtitle.subtitle_id = int(random.random() * 10e12)
subtitle.subtitle_order = i + 1
subtitle.save()
version.finished = True
version.save()
language.had_version = True
language.is_complete = True
language.save()
def _get_subtitles_from_youtube(self, video_obj):
from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle
url = 'http://www.youtube.com/watch_ajax?action_get_caption_track_all&v=%s' % video_obj.youtube_videoid
d = urllib.urlopen(url)
parser = YoutubeSubtitleParser(d.read())
if not parser:
return
language, create = SubtitleLanguage.objects.get_or_create(video=video_obj, language = parser.language)
language.is_original = False
language.is_forked = True
language.save()
try:
version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
.get().version_no + 1
except SubtitleVersion.DoesNotExist:
version_no = 0
version = SubtitleVersion(language=language)
version.version_no = version_no
version.datetime_started = datetime.now()
version.user = User.get_youtube_anonymous()
version.note = u'From youtube'
version.is_forked = True
version.save()
for i, item in enumerate(parser):
subtitle = Subtitle(**item)
subtitle.version = version
subtitle.subtitle_id = int(random.random()*10e12)
subtitle.subtitle_order = i+1
subtitle.save()
version.finished = True
version.save()
language.was_complete = True
language.is_complete = True
language.save()
def test_notification_sending(self):
from videos.tasks import send_notification, check_alarm, detect_language
latest_version = self.language.latest_version()
v = SubtitleVersion()
v.language = self.language
v.datetime_started = datetime.now()
v.version_no = latest_version.version_no+1
v.save()
for s in latest_version.subtitle_set.all():
s.duplicate_for(v).save()
s = Subtitle()
s.version = v
s.subtitle_id = 'asdasdsadasdasdasd'
s.subtitle_order = 5
s.subtitle_text = 'new subtitle'
s.start_time = 50
s.end_time = 51
s.save()
v.update_percent_done()
self.assertEqual(len(mail.outbox), 1)
result = send_notification.delay(v.id)
if result.failed():
self.fail(result.traceback)
result = check_alarm.delay(v.id)
if result.failed():
self.fail(result.traceback)
result = detect_language.delay(v.id)
if result.failed():
self.fail(result.traceback)
def save_subtitles_for_lang(lang, video_pk, youtube_id):
from videos.models import Video
lc = lang.get('lang_code')
if not lc in SUPPORTED_LANGUAGES_DICT:
return
try:
video = Video.objects.get(pk=video_pk)
except Video.DoesNotExist:
return
from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle
url = u'http://www.youtube.com/api/timedtext?v=%s&lang=%s&name=%s'
url = url % (youtube_id, lc, urlquote(lang.get('name', u'')))
xml = YoutubeVideoType._get_response_from_youtube(url)
if xml is None:
return
parser = YoutubeXMLParser(xml)
if not parser:
return
language, create = SubtitleLanguage.objects.get_or_create(video=video, language=lc)
language.is_original = False
language.is_forked = True
language.save()
try:
version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
.get().version_no + 1
except SubtitleVersion.DoesNotExist:
version_no = 0
version = SubtitleVersion(language=language)
version.version_no = version_no
version.datetime_started = datetime.now()
version.user = User.get_anonymous()
version.note = u'From youtube'
version.is_forked = True
version.save()
for i, item in enumerate(parser):
subtitle = Subtitle()
subtitle.subtitle_text = item['subtitle_text']
subtitle.start_time = item['start_time']
subtitle.end_time = item['end_time']
subtitle.version = version
subtitle.subtitle_id = int(random.random()*10e12)
subtitle.subtitle_order = i+1
subtitle.save()
assert subtitle.start_time or subtitle.end_time, item['subtitle_text']
version.finished = True
version.save()
language.has_version = True
language.had_version = True
language.is_complete = True
language.save()
from videos.tasks import video_changed_tasks
video_changed_tasks.delay(video.pk)
def save_subtitles_for_lang(lang, video_pk, youtube_id):
from videos.models import Video
lc = lang.get('lang_code')
if not lc in SUPPORTED_LANGUAGES_DICT:
return
try:
video = Video.objects.get(pk=video_pk)
except Video.DoesNotExist:
return
from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle
url = u'http://www.youtube.com/api/timedtext?v=%s&lang=%s&name=%s'
url = url % (youtube_id, lc, urlquote(lang.get('name', u'')))
xml = YoutubeVideoType._get_response_from_youtube(url)
if not xml:
return
parser = YoutubeXMLParser(xml)
if not parser:
return
language, create = SubtitleLanguage.objects.get_or_create(video=video,
language=lc)
language.is_original = False
language.is_forked = True
language.save()
try:
version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
.get().version_no + 1
except SubtitleVersion.DoesNotExist:
version_no = 0
version = SubtitleVersion(language=language)
version.version_no = version_no
version.datetime_started = datetime.now()
version.user = User.get_youtube_anonymous()
version.note = u'From youtube'
version.is_forked = True
version.save()
for i, item in enumerate(parser):
subtitle = Subtitle()
subtitle.subtitle_text = item['subtitle_text']
subtitle.start_time = item['start_time']
subtitle.end_time = item['end_time']
subtitle.version = version
subtitle.subtitle_id = int(random.random() * 10e12)
subtitle.subtitle_order = i + 1
subtitle.save()
assert subtitle.start_time or subtitle.end_time, item['subtitle_text']
version.finished = True
version.save()
language.has_version = True
language.had_version = True
language.is_complete = True
language.save()
from videos.tasks import video_changed_tasks
video_changed_tasks.delay(video.pk)
def save_subtitles_for_lang(lang, video_pk, youtube_id):
from videos.models import Video
yt_lc = lang.get('lang_code')
lc = LanguageCode(yt_lc, "youtube").encode("unisubs")
if not lc in SUPPORTED_LANGUAGE_CODES:
logger.warn(
"Youtube import did not find language code",
extra={"data": {
"language_code": lc,
"youtube_id": youtube_id,
}})
return
try:
video = Video.objects.get(pk=video_pk)
except Video.DoesNotExist:
return
from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle
url = u'http://www.youtube.com/api/timedtext?v=%s&lang=%s&name=%s'
url = url % (youtube_id, yt_lc, urlquote(lang.get('name', u'')))
xml = YoutubeVideoType._get_response_from_youtube(url)
if xml is None:
return
parser = YoutubeXMLParser(xml)
if not parser:
return
language, create = SubtitleLanguage.objects.get_or_create(
video=video, language=lc, defaults={
'created': datetime.now(),
})
language.is_original = False
language.is_forked = True
language.save()
try:
version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
.get().version_no + 1
except SubtitleVersion.DoesNotExist:
version_no = 0
version = SubtitleVersion(language=language)
version.title = video.title
version.description = video.description
version.version_no = version_no
version.datetime_started = datetime.now()
version.user = User.get_anonymous()
version.note = u'From youtube'
version.is_forked = True
version.save()
for i, item in enumerate(parser):
subtitle = Subtitle()
subtitle.subtitle_text = item['subtitle_text']
subtitle.start_time = item['start_time']
subtitle.end_time = item['end_time']
subtitle.version = version
subtitle.subtitle_id = int(random.random() * 10e12)
subtitle.subtitle_order = i + 1
subtitle.save()
assert subtitle.start_time or subtitle.end_time, item['subtitle_text']
version.finished = True
version.save()
language.has_version = True
language.had_version = True
language.is_complete = True
language.save()
from videos.tasks import video_changed_tasks
video_changed_tasks.delay(video.pk)
def save_subtitle(video,
language,
parser,
user=None,
update_video=True,
forks=True,
as_forked=True,
translated_from=None):
from videos.models import SubtitleVersion, Subtitle, SubtitleMetadata
from videos.tasks import video_changed_tasks
key = str(uuid4()).replace('-', '')
if language.is_original:
as_forked = True
video._make_writelock(user, key)
video.save()
try:
old_version = language.subtitleversion_set.all()[:1].get()
version_no = old_version.version_no + 1
except ObjectDoesNotExist:
old_version = None
version_no = 0
version = None
if not is_version_same(old_version, parser):
forked_from = as_forked and translated_from and translated_from.version(
)
title = old_version.title if old_version and old_version.title else video.get_title_display(
)
description = old_version.description if old_version and old_version.description else video.get_description_display(
)
version = SubtitleVersion(language=language,
version_no=version_no,
datetime_started=datetime.now(),
user=user,
note=u'Uploaded',
is_forked=as_forked,
time_change=1,
text_change=1,
forked_from=forked_from,
title=title,
description=description)
if len(parser) > 0:
version.has_version = True
version.save()
ids = []
for i, item in enumerate(parser):
id = int(random.random() * 10e12)
while id in ids:
id = int(random.random() * 10e12)
ids.append(id)
metadata = item.pop('metadata', None)
caption = Subtitle(**item)
caption.version = version
caption.datetime_started = datetime.now()
caption.subtitle_id = str(id)
caption.subtitle_order = i + 1
caption.save()
if metadata:
for name, value in metadata.items():
SubtitleMetadata(subtitle=caption, key=name,
data=value).save()
version = version or old_version
if version.is_forked != as_forked:
version.is_forked = as_forked
version.save()
if version.user != user:
# we might be only uptading the user , as in per bulk imports
version.user = user
version.save()
language.video.release_writelock()
language.video.save()
if forks:
translations = video.subtitlelanguage_set.filter(
standard_language=language)
[t.fork(from_version=old_version, user=user) for t in translations]
if update_video:
video_changed_tasks.delay(video.id,
None if version is None else version.id)
return language