Esempio n. 1
0
    def save_subtitles(self, parser):
        video = self.cleaned_data['video']
        
        key = str(uuid4()).replace('-', '')

        video._make_writelock(self.user, key)
        video.save()

        if not video.has_original_language():
            self._save_original_language(
                video, self.cleaned_data['video_language'])
  
        language = video.subtitle_language(self.cleaned_data['language'])
        
        if not language:
            language = SubtitleLanguage(video=video, is_original=False, is_forked=True)
        
        language.language = self.cleaned_data['language']
        language.save()
        
        try:
            old_version = language.subtitleversion_set.all()[:1].get()    
            version_no = old_version.version_no + 1
        except ObjectDoesNotExist:
            old_version = None
            version_no = 0
        if not self.is_version_same(old_version, parser):
            version = SubtitleVersion(
                language=language, version_no=version_no,
                datetime_started=datetime.now(), user=self.user,
                note=u'Uploaded', is_forked=True, time_change=1, text_change=1)
            version.save()
    
            ids = []
    
            for i, item in enumerate(parser):
                id = int(random.random()*10e12)
                while id in ids:
                    id = int(random.random()*10e12)
                ids.append(id)
                caption = Subtitle(**item)
                caption.version = version
                caption.subtitle_id = str(id)
                caption.subtitle_order = i+1
                caption.save()

            version.save()
            version.update_percent_done()

            language.update_complete_state()
            language.save()
            if language.is_original:
                video.update_complete_state()

        video.release_writelock()
        video.save()
        
        return language
Esempio n. 2
0
    def save_subtitles(self, parser):
        video = self.cleaned_data['video']

        key = str(uuid4()).replace('-', '')

        video._make_writelock(self.user, key)
        video.save()

        if not video.has_original_language():
            self._save_original_language(video,
                                         self.cleaned_data['video_language'])

        language = self._find_appropriate_language(
            video, self.cleaned_data['language'])

        try:
            old_version = language.subtitleversion_set.all()[:1].get()
            version_no = old_version.version_no + 1
        except ObjectDoesNotExist:
            old_version = None
            version_no = 0
        if not self.is_version_same(old_version, parser):
            version = SubtitleVersion(language=language,
                                      version_no=version_no,
                                      datetime_started=datetime.now(),
                                      user=self.user,
                                      note=u'Uploaded',
                                      is_forked=True,
                                      time_change=1,
                                      text_change=1)
            version.save()

            ids = []

            for i, item in enumerate(parser):
                id = int(random.random() * 10e12)
                while id in ids:
                    id = int(random.random() * 10e12)
                ids.append(id)
                caption = Subtitle(**item)
                caption.version = version
                caption.subtitle_id = str(id)
                caption.subtitle_order = i + 1
                caption.save()

            version.save()
            version.update_percent_done()

            language.update_complete_state()
            language.save()
            if language.is_original:
                video.update_complete_state()

        video.release_writelock()
        video.save()

        return language
Esempio n. 3
0
    def _get_subtitles_from_youtube(self, video_obj):
        from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle

        url = 'http://www.youtube.com/watch_ajax?action_get_caption_track_all&v=%s' % video_obj.youtube_videoid

        h = httplib2.Http()
        resp, content = h.request(url, "GET")

        if resp.status < 200 or resp.status >= 400:
            logger.warning("Youtube subtitles error",
                           extra={
                               'data': {
                                   "url": url,
                                   "video_id": video_obj.youtube_videoid,
                                   "status_code": resp.status,
                                   "response": content
                               }
                           })
            return
        parser = YoutubeSubtitleParser(content)

        if not parser:
            return

        language, create = SubtitleLanguage.objects.get_or_create(
            video=video_obj, language=parser.language)
        language.is_original = False
        language.is_forked = True
        language.save()

        try:
            version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
                .get().version_no + 1
        except SubtitleVersion.DoesNotExist:
            version_no = 0

        version = SubtitleVersion(language=language)
        version.version_no = version_no
        version.datetime_started = datetime.now()
        version.user = User.get_youtube_anonymous()
        version.note = u'From youtube'
        version.is_forked = True
        version.save()

        for i, item in enumerate(parser):
            subtitle = Subtitle(**item)
            subtitle.version = version
            subtitle.subtitle_id = int(random.random() * 10e12)
            subtitle.subtitle_order = i + 1
            subtitle.save()
        version.finished = True
        version.save()

        language.had_version = True
        language.is_complete = True
        language.save()
Esempio n. 4
0
    def _get_subtitles_from_youtube(self, video_obj):
        from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle
        
        url = 'http://www.youtube.com/watch_ajax?action_get_caption_track_all&v=%s' % video_obj.youtube_videoid

        h = httplib2.Http()
        resp, content = h.request(url, "GET")

        if resp.status < 200 or resp.status >= 400:
            logger.warning("Youtube subtitles error", extra={
                    'data': {
                        "url": url,
                        "video_id": video_obj.youtube_videoid,
                        "status_code": resp.status,
                        "response": content
                        }
                    })
            return
        parser = YoutubeSubtitleParser(content)

        if not parser:
            return
        
        language, create = SubtitleLanguage.objects.get_or_create(video=video_obj, language = parser.language)
        language.is_original = False
        language.is_forked = True
        language.save()
        
        try:
            version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
                .get().version_no + 1
        except SubtitleVersion.DoesNotExist:
            version_no = 0
            
        version = SubtitleVersion(language=language)
        version.version_no = version_no
        version.datetime_started = datetime.now()
        version.user = User.get_youtube_anonymous()
        version.note = u'From youtube'
        version.is_forked = True
        version.save()

        for i, item in enumerate(parser):
            subtitle = Subtitle(**item)
            subtitle.version = version
            subtitle.subtitle_id = int(random.random()*10e12)
            subtitle.subtitle_order = i+1
            subtitle.save()
        version.finished = True
        version.save()

        language.had_version = True
        language.is_complete = True
        language.save()
Esempio n. 5
0
 def test_adding_to_original(self):
     latest_version = self.original_language.latest_version()
     s = Subtitle()
     s.version = latest_version
     s.subtitle_id = 'asdasdsadasdasdasd'
     s.subtitle_order = 5
     s.subtitle_text = 'new subtitle'
     s.start_time = 50
     s.end_time = 51
     s.save()
     self.translation.update_percent_done()
     self.assertEqual(self.translation.percent_done, 4/5.*100)
Esempio n. 6
0
 def test_adding_to_original(self):
     latest_version = self.original_language.latest_version()
     s = Subtitle()
     s.version = latest_version
     s.subtitle_id = 'asdasdsadasdasdasd'
     s.subtitle_order = 5
     s.subtitle_text = 'new subtitle'
     s.start_time = 50
     s.end_time = 51
     s.save()
     self.translation.update_percent_done()
     self.assertEqual(self.translation.percent_done, 4/5.*100)
Esempio n. 7
0
 def test_many_subtitles(self):
     latest_version = self.original_language.latest_version()
     for i in range(5, 450):
         s = Subtitle()
         s.version = latest_version
         s.subtitle_id = 'sadfdasf%s' % i
         s.subtitle_order = i
         s.start_time = 50 + i
         s.end_time = 51 + i
         s.save()
         
     self.translation.update_percent_done()
     self.assertEqual(self.translation.percent_done, 0)
Esempio n. 8
0
 def test_many_subtitles(self):
     latest_version = self.original_language.latest_version()
     for i in range(5, 450):
         s = Subtitle()
         s.version = latest_version
         s.subtitle_id = 'sadfdasf%s' % i
         s.subtitle_order = i
         s.start_time = 50 + i
         s.end_time = 51 + i
         s.save()
         
     self.translation.update_percent_done()
     self.assertEqual(self.translation.percent_done, 0)
Esempio n. 9
0
def save_subtitle(video, language, parser, user=None, update_video=True):
    from videos.models import SubtitleVersion, Subtitle
    from videos.tasks import video_changed_tasks

    key = str(uuid4()).replace('-', '')

    video._make_writelock(user, key)
    video.save()

    try:
        old_version = language.subtitleversion_set.all()[:1].get()
        version_no = old_version.version_no + 1
    except ObjectDoesNotExist:
        old_version = None
        version_no = 0
    version = None
    if not is_version_same(old_version, parser):
        version = SubtitleVersion(language=language,
                                  version_no=version_no,
                                  datetime_started=datetime.now(),
                                  user=user,
                                  note=u'Uploaded',
                                  is_forked=True,
                                  time_change=1,
                                  text_change=1)
        version.save()

        ids = []

        for i, item in enumerate(parser):
            id = int(random.random() * 10e12)
            while id in ids:
                id = int(random.random() * 10e12)
            ids.append(id)
            caption = Subtitle(**item)
            caption.version = version
            caption.subtitle_id = str(id)
            caption.subtitle_order = i + 1
            caption.save()

    language.video.release_writelock()
    language.video.save()
    translations = video.subtitlelanguage_set.filter(
        standard_language=language)
    [t.fork(from_version=old_version, user=user) for t in translations]
    if update_video:
        video_changed_tasks.delay(video.id,
                                  None if version is None else version.id)

    return language
Esempio n. 10
0
def save_subtitle(video, language, parser, user=None, update_video=True):
    from videos.models import SubtitleVersion, Subtitle
    from videos.tasks import video_changed_tasks

    key = str(uuid4()).replace("-", "")

    video._make_writelock(user, key)
    video.save()

    try:
        old_version = language.subtitleversion_set.all()[:1].get()
        version_no = old_version.version_no + 1
    except ObjectDoesNotExist:
        old_version = None
        version_no = 0
    version = None
    if not is_version_same(old_version, parser):
        version = SubtitleVersion(
            language=language,
            version_no=version_no,
            datetime_started=datetime.now(),
            user=user,
            note=u"Uploaded",
            is_forked=True,
            time_change=1,
            text_change=1,
        )
        version.save()

        ids = []

        for i, item in enumerate(parser):
            id = int(random.random() * 10e12)
            while id in ids:
                id = int(random.random() * 10e12)
            ids.append(id)
            caption = Subtitle(**item)
            caption.version = version
            caption.subtitle_id = str(id)
            caption.subtitle_order = i + 1
            caption.save()

    language.video.release_writelock()
    language.video.save()
    translations = video.subtitlelanguage_set.filter(standard_language=language)
    [t.fork(from_version=old_version, user=user) for t in translations]
    if update_video:
        video_changed_tasks.delay(video.id, None if version is None else version.id)

    return language
Esempio n. 11
0
    def save(self):
        subtitles = self.cleaned_data["subtitles"]
        video = self.cleaned_data["video"]

        key = str(uuid4()).replace("-", "")
        video._make_writelock(self.user, key)
        video.save()

        language = video.subtitle_language()

        if not language:
            language = SubtitleLanguage(video=video, is_original=True, is_complete=True)
            language.save()

        latest_captions = language.latest_version()
        if latest_captions is None:
            version_no = 0
        else:
            version_no = latest_captions.version_no + 1

        version = SubtitleVersion(
            language=language, version_no=version_no, datetime_started=datetime.now(), user=self.user, note=u"Uploaded"
        )
        version.save()

        text = subtitles.read()
        parser = self._get_parser(subtitles.name)(force_unicode(text, chardet.detect(text)["encoding"]))
        ids = []

        for i, item in enumerate(parser):
            id = int(random.random() * 10e12)
            while id in ids:
                id = int(random.random() * 10e12)
            ids.append(id)
            caption = Subtitle(**item)
            caption.version = version
            caption.subtitle_id = str(id)
            caption.sub_order = i + 1
            caption.save()

        version.finished = True
        version.save()

        video.release_writelock()
        video.save()
Esempio n. 12
0
    def _get_subtitles_from_youtube(self, video_obj):
        from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle

        url = 'http://www.youtube.com/watch_ajax?action_get_caption_track_all&v=%s' % video_obj.youtube_videoid
        d = urllib.urlopen(url)
        parser = YoutubeSubtitleParser(d.read())

        if not parser:
            return

        language, create = SubtitleLanguage.objects.get_or_create(
            video=video_obj, language=parser.language)
        language.is_original = False
        language.is_forked = True
        language.save()

        try:
            version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
                .get().version_no + 1
        except SubtitleVersion.DoesNotExist:
            version_no = 0

        version = SubtitleVersion(language=language)
        version.version_no = version_no
        version.datetime_started = datetime.now()
        version.user = User.get_youtube_anonymous()
        version.note = u'From youtube'
        version.is_forked = True
        version.save()

        for i, item in enumerate(parser):
            subtitle = Subtitle(**item)
            subtitle.version = version
            subtitle.subtitle_id = int(random.random() * 10e12)
            subtitle.subtitle_order = i + 1
            subtitle.save()
        version.finished = True
        version.save()

        language.had_version = True
        language.is_complete = True
        language.save()
Esempio n. 13
0
    def _get_subtitles_from_youtube(self, video_obj):
        from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle
        
        url = 'http://www.youtube.com/watch_ajax?action_get_caption_track_all&v=%s' % video_obj.youtube_videoid
        d = urllib.urlopen(url)
        parser = YoutubeSubtitleParser(d.read())

        if not parser:
            return
        
        language, create = SubtitleLanguage.objects.get_or_create(video=video_obj, language = parser.language)
        language.is_original = False
        language.is_forked = True
        language.save()
        
        try:
            version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
                .get().version_no + 1
        except SubtitleVersion.DoesNotExist:
            version_no = 0
            
        version = SubtitleVersion(language=language)
        version.version_no = version_no
        version.datetime_started = datetime.now()
        version.user = User.get_youtube_anonymous()
        version.note = u'From youtube'
        version.is_forked = True
        version.save()

        for i, item in enumerate(parser):
            subtitle = Subtitle(**item)
            subtitle.version = version
            subtitle.subtitle_id = int(random.random()*10e12)
            subtitle.subtitle_order = i+1
            subtitle.save()
        version.finished = True
        version.save()

        language.was_complete = True
        language.is_complete = True
        language.save()
Esempio n. 14
0
    def test_notification_sending(self):
        from videos.tasks import send_notification, check_alarm, detect_language
        
        latest_version = self.language.latest_version()
        
        v = SubtitleVersion()
        v.language = self.language
        v.datetime_started = datetime.now()
        v.version_no = latest_version.version_no+1
        v.save()

        for s in latest_version.subtitle_set.all():
            s.duplicate_for(v).save()
        
        s = Subtitle()
        s.version = v
        s.subtitle_id = 'asdasdsadasdasdasd'
        s.subtitle_order = 5
        s.subtitle_text = 'new subtitle'
        s.start_time = 50
        s.end_time = 51
        s.save()        

        v.update_percent_done()
        self.assertEqual(len(mail.outbox), 1)
        
        result = send_notification.delay(v.id)
        if result.failed():
            self.fail(result.traceback)
        
        result = check_alarm.delay(v.id)
        if result.failed():
            self.fail(result.traceback)
        
        result = detect_language.delay(v.id)
        if result.failed():
            self.fail(result.traceback)
Esempio n. 15
0
def save_subtitles_for_lang(lang, video_pk, youtube_id):
    from videos.models import Video
    
    lc = lang.get('lang_code')

    if not lc in SUPPORTED_LANGUAGES_DICT:
        return
    
    try:
        video = Video.objects.get(pk=video_pk)
    except Video.DoesNotExist:
        return
    
    from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle

    url = u'http://www.youtube.com/api/timedtext?v=%s&lang=%s&name=%s'
    url = url % (youtube_id, lc, urlquote(lang.get('name', u'')))

    xml = YoutubeVideoType._get_response_from_youtube(url)

    if xml is None:
        return

    parser = YoutubeXMLParser(xml)

    if not parser:
        return
    
    language, create = SubtitleLanguage.objects.get_or_create(video=video, language=lc)
    language.is_original = False
    language.is_forked = True
    language.save()
    
    try:
        version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
            .get().version_no + 1
    except SubtitleVersion.DoesNotExist:
        version_no = 0
        
    version = SubtitleVersion(language=language)
    version.version_no = version_no
    version.datetime_started = datetime.now()
    version.user = User.get_anonymous()
    version.note = u'From youtube'
    version.is_forked = True
    version.save()

    for i, item in enumerate(parser):
        subtitle = Subtitle()
        subtitle.subtitle_text = item['subtitle_text']
        subtitle.start_time = item['start_time']
        subtitle.end_time = item['end_time']
        subtitle.version = version
        subtitle.subtitle_id = int(random.random()*10e12)
        subtitle.subtitle_order = i+1
        subtitle.save()
        assert subtitle.start_time or subtitle.end_time, item['subtitle_text']
    version.finished = True
    version.save()
    
    language.has_version = True
    language.had_version = True
    language.is_complete = True
    language.save()
    
    from videos.tasks import video_changed_tasks
    video_changed_tasks.delay(video.pk)
Esempio n. 16
0
def save_subtitles_for_lang(lang, video_pk, youtube_id):
    from videos.models import Video

    lc = lang.get('lang_code')

    if not lc in SUPPORTED_LANGUAGES_DICT:
        return

    try:
        video = Video.objects.get(pk=video_pk)
    except Video.DoesNotExist:
        return

    from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle

    url = u'http://www.youtube.com/api/timedtext?v=%s&lang=%s&name=%s'
    url = url % (youtube_id, lc, urlquote(lang.get('name', u'')))

    xml = YoutubeVideoType._get_response_from_youtube(url)

    if not xml:
        return

    parser = YoutubeXMLParser(xml)

    if not parser:
        return

    language, create = SubtitleLanguage.objects.get_or_create(video=video,
                                                              language=lc)
    language.is_original = False
    language.is_forked = True
    language.save()

    try:
        version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
            .get().version_no + 1
    except SubtitleVersion.DoesNotExist:
        version_no = 0

    version = SubtitleVersion(language=language)
    version.version_no = version_no
    version.datetime_started = datetime.now()
    version.user = User.get_youtube_anonymous()
    version.note = u'From youtube'
    version.is_forked = True
    version.save()

    for i, item in enumerate(parser):
        subtitle = Subtitle()
        subtitle.subtitle_text = item['subtitle_text']
        subtitle.start_time = item['start_time']
        subtitle.end_time = item['end_time']
        subtitle.version = version
        subtitle.subtitle_id = int(random.random() * 10e12)
        subtitle.subtitle_order = i + 1
        subtitle.save()
        assert subtitle.start_time or subtitle.end_time, item['subtitle_text']
    version.finished = True
    version.save()

    language.has_version = True
    language.had_version = True
    language.is_complete = True
    language.save()

    from videos.tasks import video_changed_tasks
    video_changed_tasks.delay(video.pk)
Esempio n. 17
0
def save_subtitles_for_lang(lang, video_pk, youtube_id):
    from videos.models import Video

    yt_lc = lang.get('lang_code')
    lc = LanguageCode(yt_lc, "youtube").encode("unisubs")

    if not lc in SUPPORTED_LANGUAGE_CODES:
        logger.warn(
            "Youtube import did not find language code",
            extra={"data": {
                "language_code": lc,
                "youtube_id": youtube_id,
            }})
        return

    try:
        video = Video.objects.get(pk=video_pk)
    except Video.DoesNotExist:
        return

    from videos.models import SubtitleLanguage, SubtitleVersion, Subtitle

    url = u'http://www.youtube.com/api/timedtext?v=%s&lang=%s&name=%s'
    url = url % (youtube_id, yt_lc, urlquote(lang.get('name', u'')))

    xml = YoutubeVideoType._get_response_from_youtube(url)

    if xml is None:
        return

    parser = YoutubeXMLParser(xml)

    if not parser:
        return

    language, create = SubtitleLanguage.objects.get_or_create(
        video=video, language=lc, defaults={
            'created': datetime.now(),
        })
    language.is_original = False
    language.is_forked = True
    language.save()

    try:
        version_no = language.subtitleversion_set.order_by('-version_no')[:1] \
            .get().version_no + 1
    except SubtitleVersion.DoesNotExist:
        version_no = 0

    version = SubtitleVersion(language=language)
    version.title = video.title
    version.description = video.description
    version.version_no = version_no
    version.datetime_started = datetime.now()
    version.user = User.get_anonymous()
    version.note = u'From youtube'
    version.is_forked = True
    version.save()

    for i, item in enumerate(parser):
        subtitle = Subtitle()
        subtitle.subtitle_text = item['subtitle_text']
        subtitle.start_time = item['start_time']
        subtitle.end_time = item['end_time']
        subtitle.version = version
        subtitle.subtitle_id = int(random.random() * 10e12)
        subtitle.subtitle_order = i + 1
        subtitle.save()
        assert subtitle.start_time or subtitle.end_time, item['subtitle_text']
    version.finished = True
    version.save()

    language.has_version = True
    language.had_version = True
    language.is_complete = True
    language.save()

    from videos.tasks import video_changed_tasks
    video_changed_tasks.delay(video.pk)
Esempio n. 18
0
def save_subtitle(video,
                  language,
                  parser,
                  user=None,
                  update_video=True,
                  forks=True,
                  as_forked=True,
                  translated_from=None):
    from videos.models import SubtitleVersion, Subtitle, SubtitleMetadata
    from videos.tasks import video_changed_tasks

    key = str(uuid4()).replace('-', '')

    if language.is_original:
        as_forked = True

    video._make_writelock(user, key)
    video.save()

    try:
        old_version = language.subtitleversion_set.all()[:1].get()
        version_no = old_version.version_no + 1
    except ObjectDoesNotExist:
        old_version = None
        version_no = 0

    version = None

    if not is_version_same(old_version, parser):
        forked_from = as_forked and translated_from and translated_from.version(
        )

        title = old_version.title if old_version and old_version.title else video.get_title_display(
        )
        description = old_version.description if old_version and old_version.description else video.get_description_display(
        )

        version = SubtitleVersion(language=language,
                                  version_no=version_no,
                                  datetime_started=datetime.now(),
                                  user=user,
                                  note=u'Uploaded',
                                  is_forked=as_forked,
                                  time_change=1,
                                  text_change=1,
                                  forked_from=forked_from,
                                  title=title,
                                  description=description)

        if len(parser) > 0:
            version.has_version = True

        version.save()

        ids = []

        for i, item in enumerate(parser):
            id = int(random.random() * 10e12)

            while id in ids:
                id = int(random.random() * 10e12)

            ids.append(id)

            metadata = item.pop('metadata', None)

            caption = Subtitle(**item)
            caption.version = version
            caption.datetime_started = datetime.now()
            caption.subtitle_id = str(id)
            caption.subtitle_order = i + 1
            caption.save()

            if metadata:
                for name, value in metadata.items():
                    SubtitleMetadata(subtitle=caption, key=name,
                                     data=value).save()

    version = version or old_version

    if version.is_forked != as_forked:
        version.is_forked = as_forked
        version.save()

    if version.user != user:
        # we might be only uptading the user , as in per bulk imports
        version.user = user
        version.save()

    language.video.release_writelock()
    language.video.save()

    if forks:
        translations = video.subtitlelanguage_set.filter(
            standard_language=language)
        [t.fork(from_version=old_version, user=user) for t in translations]

    if update_video:
        video_changed_tasks.delay(video.id,
                                  None if version is None else version.id)

    return language