def main():
parsed = feedparser.parse(file("mostravideolivre.atom").read())
for i in parsed.entries:
mp4_url = i.content[0].src
ogg_url = mp4_url.replace(".mp4", ".ogg")
ogg_url = ogg_url.replace(".MP4", ".ogg")
thumb = ogg_url + ".png"
video = Video()
video.title = i.title
video.creation_date = datetime.now()
video.summary = i.get("summary", "")
video.author = i.author
video.license_name = "Creative Commons - Atribuição - Partilha nos Mesmos " + "Termos 3.0 Não Adaptada"
video.license_link = "http://creativecommons.org/licenses/by-sa/3.0/"
video.thumb_url = thumb
video.save()
tag = Tag.objects.get_or_create(name="mvl2010")[0]
tag.save()
video.tags.add(tag.id)
url = Url()
url.url = mp4_url
url.content_type = 'video/mp4; codecs="avc1.42E01E, mp4a.40.2"'
url.video = video
url.save()
url = Url()
url.url = ogg_url
url.content_type = 'video/ogg; codecs="theora, vorbis"'
url.video = video
url.save()
def init_cocreate(cocreate, generate_slug):
if len(cocreate.videos) <= 0:
return
if not cocreate.output_video:
slug = generate_slug(SLUG_LENGTH)
video = Video(title=cocreate.title, slug=slug, description=cocreate.description, uploader=cocreate.owner)
video.save()
cocreate.output_video = video
cocreate.save()
else:
# reset video status to encoding
video_status = cocreate.output_video.get_video_status()
video_status.set_to_encoding()
#create outline from the sections
VideoOutline.objects.filter(video=cocreate.output_video).delete()
outline = VideoOutline.objects.create(video=cocreate.output_video)
asections = cocreate.available_sections
for i in xrange(len(asections)):
outline.videooutlinepin_set.create(text=asections[i],
current_time=Decimal(str(i)))
# enqueue on cocreate task queue
enqueue_cocreate(cocreate)
def _create_video(self, video_url):
video, created = Video.get_or_create_for_url(video_url, self.user)
self.failUnless(video)
self.failUnless(created)
more_video, created = Video.get_or_create_for_url(video_url, self.user)
self.failIf(created)
self.failUnlessEqual(video, more_video)
def forwards(self, orm):
try:
from videos.models import Video
except ImportError:
return
try:
from statistic import video_view_counter, sub_fetch_total_counter, \
widget_views_total_counter
except ImportError:
if settings.DEBUG:
return
raise Exception('Some redis utilits is unavailable, maybe this migration was not updated after refactoring. You can ignore this migration with: python manage.py migrate statistic 0074 --fake, but all statistic data will be lost.')
try:
video_view_counter.r.ping()
except ConnectionError:
if settings.DEBUG:
return
raise Exception('Redis server is unavailable. You can ignore this migration with: python manage.py migrate statistic 00074 --fake, but all statistic data will be lost.')
video_view_counter.delete()
sub_fetch_total_counter.delete()
widget_views_total_counter.delete()
for obj in orm.Video.objects.all():
video_counter = Video.view_counter(obj.video_id)
video_counter.delete()
video_counter.set(obj.view_count)
video_view_counter.incr(obj.view_count)
sub_fetch_total_counter.incr(obj.subtitles_fetched_count)
widget_views_total_counter.incr(Video.widget_views_counter(obj.video_id).get())
def test_delete_index(self):
app_config = apps.get_app_config('videos')
create_index(app_config, 'Video')
self.assertTrue(index_exists(Video.get_index_name()))
delete_index(Video.get_index_name())
with self.assertRaises(IndexNotFound):
get_index(Video.get_index_name())
def save(self):
if self.cleaned_data.get('save_feed'):
for feed_url in self.feed_urls:
self.save_feed_url(feed_url)
for vt in self.video_types:
Video.get_or_create_for_url(vt=vt, user=self.user)
return len(self.video_types)
def handle(self, *args, **kwargs):
try:
sub_fetch_keys_set.r.ping()
except:
if settings.DEBUG:
raise
self.handle_error('Statistic update error', '', sys.exc_info())
return
count = sub_fetch_keys_set.scard()
while count:
count -= 1
key = sub_fetch_keys_set.spop()
if not key:
break
print 'Handle key: %s' % key
parts = key.split(':')
d = date(int(parts[-1]), int(parts[-2]), int(parts[-3]))
if len(parts) == 6:
lang = parts[2]
else:
lang = ''
try:
video = Video.objects.get(video_id=parts[1])
except Video.DoesNotExist:
print 'Video does not exist'
default_connection.delete(key)
continue
counter_obj, created = SubtitleFetchCounters.objects.get_or_create(date=d, video=video, language=lang)
counter_obj.count += int(default_connection.getset(key, 0))
counter_obj.save()
count = changed_video_set.scard()
while count:
count -= 1
video_id = changed_video_set.spop()
if not video_id:
break
print 'Update statistic for video: %s' % video_id
subtitles_fetched_counter = Video.subtitles_fetched_counter(video_id, True)
widget_views_counter = Video.widget_views_counter(video_id, True)
view_counter = Video.view_counter(video_id, True)
Video.objects.filter(video_id=video_id).update(view_count=F('view_count')+view_counter.getset(0))
Video.objects.update(widget_views_count=F('widget_views_count')+widget_views_counter.getset(0))
Video.objects.update(subtitles_fetched_count=F('subtitles_fetched_count')+subtitles_fetched_counter.getset(0))
def test_bliptv_twice(self):
VIDEO_FILE = 'http://blip.tv/file/get/Kipkay-AirDusterOfficeWeaponry223.m4v'
from vidscraper.sites import blip
old_video_file_url = blip.video_file_url
blip.video_file_url = lambda x: VIDEO_FILE
Video.get_or_create_for_url('http://blip.tv/file/4395490')
blip.video_file_url = old_video_file_url
# this test passes if the following line executes without throwing an error.
Video.get_or_create_for_url(VIDEO_FILE)
def create(self, validated_data):
# logged in required
user = ContextUtils(self.context).logged_in_user()
# create video
video = Video(**validated_data)
video.user = user
video.save()
# add video tags
tags = TagBuilder.get_or_create_tags(validated_data['title'])
for tag in tags:
video.tags.add(tag)
return video
def test_youtube_subs_response(self):
import os
from videos.types.youtube import YoutubeVideoType
import urllib
def urlopen_mockup(url, *args, **kwargs):
path = os.path.join(os.path.dirname(__file__), 'fixtures/youtube_subs_response.json')
return open(path)
_urlopen = urllib.urlopen
urllib.urlopen = urlopen_mockup
vt = YoutubeVideoType('http://www.youtube.com/watch?v=GcjgWov7mTM')
video, create = Video.get_or_create_for_url('http://www.youtube.com/watch?v=GcjgWov7mTM', vt)
vt._get_subtitles_from_youtube(video)
video = Video.objects.get(pk=video.pk)
version = video.version(language_code='en')
self.assertFalse(version is None)
self.assertTrue(len(version.subtitles()))
self.assertEqual(version.subtitles()[0].text, 'I think what is probably the most misunderstood\nconcept in all of science and as we all know')
subs = version.subtitles()
subs.sort(key=lambda s: s.start_time)
for i in range(1, len(subs)):
self.assertTrue(subs[i].sub_order > subs[i - 1].sub_order)
urllib.urlopen = _urlopen
def create(request):
if request.method == 'POST':
video_form = VideoForm(request.POST, label_suffix="")
if video_form.is_valid():
owner = request.user if request.user.is_authenticated() else None
video_url = video_form.cleaned_data['video_url']
try:
video, created = Video.get_or_create_for_url(video_url, owner)
except VidscraperError:
vidscraper_error = True
return render_to_response('videos/create.html', locals(),
context_instance=RequestContext(request))
messages.info(request, message=u'''Here is the subtitle workspace for your video. You can
share the video with friends, or get an embed code for your site. To add or
improve subtitles, click the button below the video''')
return redirect(video)
#if not video.owner or video.owner == request.user or video.allow_community_edits:
# return HttpResponseRedirect('{0}?autosub=true'.format(reverse(
# 'videos:video', kwargs={'video_id':video.video_id})))
#else:
# # TODO: better error page?
# return HttpResponse('You are not allowed to add transcriptions to this video.')
else:
video_form = VideoForm(label_suffix="")
return render_to_response('videos/create.html', locals(),
context_instance=RequestContext(request))
def subtitles(request):
callback = request.GET.get('callback')
video_url = request.GET.get('video_url')
language = request.GET.get('language')
revision = request.GET.get('revision')
format = request.GET.get('format', 'json')
if video_url is None:
return {'is_error': True, 'message': 'video_url not specified' }
video, created = Video.get_or_create_for_url(video_url)
if not video:
return {'is_error': True, 'message': 'unsuported video url' }
if format == 'json':
output = [s.for_json() for s in video.subtitles(version_no=revision, language_code = language)]
if callback:
result = json.dumps(output)
return HttpResponse('%s(%s);' % (callback, result), 'text/javascript')
else:
return output
else:
handler = GenerateSubtitlesHandler.get(format)
if not handler:
return {'is_error': True, 'message': 'undefined format' }
subtitles = [s.for_generator() for s in video.subtitles(version_no=revision, language_code = language)]
h = handler(subtitles, video)
return HttpResponse(unicode(h))
def _api_subtitles_json(request):
video_url = request.GET.get('video_url', None)
language = request.GET.get('language', None)
revision = request.GET.get('revision', None)
if video_url is None:
return {'is_error': True, 'message': 'video_url not specified' }
video, created = Video.get_or_create_for_url(video_url)
return [s.__dict__ for s in video.subtitles(
version_no=revision, language_code = language)]
def test_video_cache_busted_on_delete(self):
start_url = 'http://videos.mozilla.org/firefox/3.5/switch/switch.ogv'
video, created = Video.get_or_create_for_url(start_url)
video_url = video.get_video_url()
video_pk = video.pk
#
cache_id_1 = video_cache.get_video_id(video_url)
self.assertTrue(cache_id_1)
video.delete()
self.assertEqual(Video.objects.filter(pk=video_pk).count() , 0)
# when cache is not cleared this will return arn
cache_id_2 = video_cache.get_video_id(video_url)
self.assertNotEqual(cache_id_1, cache_id_2)
# create a new video with the same url, has to have same# key
video2, created= Video.get_or_create_for_url(start_url)
video_url = video2.get_video_url()
cache_id_3 = video_cache.get_video_id(video_url)
self.assertEqual(cache_id_3, cache_id_2)
def test_cache_has_right_value(self):
video_url = VideoUrl.objects.all()[0]
tv = TeamVideo(video=video_url.video, team=self.team,added_by=self.user)
tv.save()
res = self._get_widget_moderation_status(video_url.url)
self.assertFalse(res["is_moderated"])
video, created = Video.get_or_create_for_url(video_url.url)
add_moderation(video, self.team, self.user)
res = self._get_widget_moderation_status(video_url.url)
self.assertTrue(res["is_moderated"])
def save(self):
if self.cleaned_data.get('save_feed'):
for feed_url, last_entry_url in self.feed_urls:
self.save_feed_url(feed_url, last_entry_url)
videos = []
for vt, info in self.video_types:
videos.append(Video.get_or_create_for_url(vt=vt, user=self.user))
return videos
def test_get_document(self):
app_config = apps.get_app_config('videos')
create_index(app_config, 'Video')
self.assertTrue(index_exists(Video.get_index_name()))
time.sleep(1)
v = Video.objects.create(
title="Test Video",
video_id="Leg1tVid1D",
thumbnail_url="http://example.com/legitthumbnail.jpeg")
doc = get_document(v)
self.assertTrue(isinstance(doc, dict))
def test_set_values(self):
youtbe_url = 'http://www.youtube.com/watch?v=_ShmidkrcY0'
vt = self.vt(youtbe_url)
video, created = Video.get_or_create_for_url(youtbe_url)
vu = video.videourl_set.all()[:1].get()
self.assertEqual(vu.videoid, '_ShmidkrcY0')
self.assertTrue(video.title)
self.assertEqual(video.duration, 79)
self.assertTrue(video.thumbnail)
def read(self, request):
"""
Return subtitles for video.
Send in request:
<b>video url:</b> video_url
<b>video id:</b> video_id
<b>language:</b> language of video
<b>revision:</b> revision of subtitles
<b>sformat</b>: format of subtitles(srt, ass, ssa, ttml, sbv)
By default format of response is 'plain', so you get raw subtitles content in response.
If 'sformat' exists in request - format will be 'plain'.
If 'callback' exists in request - format will be 'json'.
curl http://127.0.0.1:8000/api/1.0/subtitles/ -d 'video_url=http://www.youtube.com/watch?v=YMBdMtbth0o' -G
curl http://127.0.0.1:8000/api/1.0/subtitles/ -d 'video_url=http://www.youtube.com/watch?v=YMBdMtbth0o' -d 'callback=callback' -G
curl http://127.0.0.1:8000/api/1.0/subtitles/ -d 'video_url=http://www.youtube.com/watch?v=YMBdMtbth0o' -d 'sformat=srt' -G
curl http://127.0.0.1:8000/api/1.0/subtitles/ -d 'video_url=http://www.youtube.com/watch?v=YMBdMtbth0o' -d 'sformat=srt' -G
curl http://127.0.0.1:8000/api/1.0/subtitles/ -d 'video_id=7Myc2QAeBco9' -G
"""
video_url = request.GET.get('video_url')
language = request.GET.get('language')
revision = request.GET.get('revision')
sformat = request.GET.get('sformat')
video_id = request.GET.get('video_id')
if not video_url and not video_id:
return rc.BAD_REQUEST
if video_id:
try:
video = Video.objects.get(video_id=video_id)
except Video.DoesNotExist:
return rc.NOT_FOUND
else:
video, created = Video.get_or_create_for_url(video_url)
if not video:
return rc.NOT_FOUND
if not sformat:
output = [s.for_json() for s in video.subtitles(version_no=revision, language_code = language)]
return output
else:
handler = GenerateSubtitlesHandler.get(sformat)
if not handler:
return rc.BAD_REQUEST
subtitles = [s.for_generator() for s in video.subtitles(version_no=revision, language_code = language)]
h = handler(subtitles, video)
return unicode(h)
def test_create_document(self):
app_config = apps.get_app_config('videos')
create_index(app_config, 'Video')
self.assertTrue(index_exists(Video.get_index_name()))
time.sleep(1)
v = Video.objects.create(
title="Test Video",
video_id="Leg1tVid1D",
thumbnail_url="http://example.com/legitthumbnail.jpeg")
# Created Via signal assert the document exists
doc = get_document(v)
self.assertEqual(doc['_source'], v.get_document_body())
def clean(self, value):
self.vt = None
self.video = None
super(UniSubBoundVideoField, self).clean(value)
video_url = value
if not video_url:
return video_url
host = Site.objects.get_current().domain
url_start = 'http://'+host
if video_url.startswith(url_start):
# UniSub URL
locale, path = strip_path(video_url[len(url_start):])
video_url = url_start+path
try:
video_url = self.format_url(video_url)
func, args, kwargs = resolve(video_url.replace(url_start, ''))
if not 'video_id' in kwargs:
raise forms.ValidationError(_('This URL does not contain video id.'))
try:
self.video = Video.objects.get(video_id=kwargs['video_id'])
except Video.DoesNotExist:
raise forms.ValidationError(_('Videos does not exist.'))
except Http404:
raise forms.ValidationError(_('Incorrect URL.'))
else:
# URL from other site
try:
self.vt = video_type_registrar.video_type_for_url(video_url)
if hasattr(self, 'user'):
user = self.user
else:
user = None
if self.vt:
self.video, created = Video.get_or_create_for_url(vt=self.vt, user=user)
except VideoTypeError, e:
self.video = None
raise forms.ValidationError(e)
if not self.video:
raise forms.ValidationError(mark_safe(_(u"""Universal Subtitles does not support that website or video format.
If you'd like to us to add support for a new site or format, or if you
think there's been some mistake, <a
href="mailto:%s">contact us</a>!""") % settings.FEEDBACK_EMAIL))
def forwards(self, orm):
if db.dry_run:
return
try:
from videos.models import Video, SubtitleLanguage
except ImportError:
return
try:
Video.widget_views_counter.r.ping()
except ConnectionError:
if settings.DEBUG:
return
raise Exception('Redis server is unavailable. You can ignore this migration with: python manage.py migrate videos 0068 --fake, but all statistic data will be lost.')
for obj in orm.Video.objects.all():
Video.subtitles_fetched_counter(obj.video_id).val = obj.subtitles_fetched_count
Video.widget_views_counter(obj.video_id).val = obj.widget_views_count
for obj in orm.SubtitleLanguage.objects.all():
SubtitleLanguage.subtitles_fetched_counter(obj.pk).val = obj.subtitles_fetched_count
def test_create_document_conflict(self):
app_config = apps.get_app_config('videos')
create_index(app_config, 'Video')
self.assertTrue(index_exists(Video.get_index_name()))
time.sleep(1)
v = Video.objects.create(
title="Test Video",
video_id="Leg1tVid1D",
thumbnail_url="http://example.com/legitthumbnail.jpeg")
err = create_document(v)
err_msg = "Conflict: document already exists for {0} with id {1}."
self.assertEqual(
err, err_msg.format(v.__class__.__name__, v.pk))
def test_type(self):
url = 'http://vimeo.com/15786066?some_param=111'
video, created = Video.get_or_create_for_url(url)
vu = video.videourl_set.all()[:1].get()
self.assertEqual(vu.videoid, '15786066')
self.assertTrue(self.vt.video_url(vu))
self.assertTrue(self.vt.matches_video_url(url))
self.assertFalse(self.vt.matches_video_url('http://vimeo.com'))
self.assertFalse(self.vt.matches_video_url(''))
def test_type(self):
url = 'http://blip.tv/file/4297824?utm_source=featured_ep&utm_medium=featured_ep'
video, created = Video.get_or_create_for_url(url)
vu = video.videourl_set.all()[:1].get()
self.assertEqual(vu.videoid, '4297824')
self.assertTrue(video.title)
self.assertTrue(video.thumbnail)
self.assertTrue(vu.url)
self.assertTrue(self.vt.matches_video_url(url))
self.assertTrue(self.vt.matches_video_url('http://blip.tv/file/4297824'))
self.assertFalse(self.vt.matches_video_url('http://blip.tv'))
self.assertFalse(self.vt.matches_video_url(''))
def _execute_crawler(self, term, max_depth, max_breadth):
token = os.environ.get('TOKEN_YOUTUBE')
crawler = Crawler(site='youtube', site_token=token,
max_breadth=max_breadth, max_depth=max_depth)
for video in crawler.run(term.split()):
if Video.objects.filter(id_source=video.id_source).exists():
continue
base_url = 'http://www.youtube.com/watch?v='
db_video = Video(id_source=video.id_source,
source='youtube',
user=self.get_user(),
title=video.title,
description=video.description,
filename=base_url+video.id_source)
db_video.save(force_insert=True) # Could be problematic on sqlite
for tag in video.tags:
db_video.tags.add(self.get_tag(tag))
# Erase CrawlerBot thread controller
self.mutex.acquire()
del(self.threads[threading.currentThread().ident])
self.mutex.release()
def test1(self):
#For this video Vimeo API returns response with strance error
#But we can get data from this response. See vidscraper.sites.vimeo.get_shortmem
#So if this test is failed - maybe API was just fixed and other response is returned
url = u'http://vimeo.com/22070806'
video, created = Video.get_or_create_for_url(url)
self.assertNotEqual(video.title, '')
self.assertNotEqual(video.description, '')
vu = video.videourl_set.all()[:1].get()
self.assertEqual(vu.videoid, '22070806')
self.assertTrue(self.vt.video_url(vu))
def main():
for index, line in enumerate(csv.reader(file('mostravideolivre.csv'))):
# I don't need the column names :)
if index == 0:
continue
# No title
if not line[0].strip():
continue
video = Video()
video.title = line[0]
video.creation_date = datetime.now()
video.summary = line[5]
video.author = line[1]
video.license_name = \
'Creative Commons - Atribuição - Partilha nos Mesmos ' + \
'Termos 3.0 Não Adaptada'
video.license_link = 'http://creativecommons.org/licenses/by-sa/3.0/'
video.thumb_url = findvideo(line[7]).replace('ogv', 'png')
video.save()
tag = Tag.objects.get_or_create(name='mvl2011')[0]
tag.save()
video.tags.add(tag.id)
for tag_name in line[4].split(','):
tag = Tag.objects.get_or_create(name=tag_name.strip())[0]
tag.save()
video.tags.add(tag.id)
url = Url()
url.url = findvideo(line[7])
url.content_type = 'video/ogg; codecs="theora, vorbis"'
url.video = video
url.save()
def test_set_values(self):
youtbe_url = 'http://www.youtube.com/watch?v=_ShmidkrcY0'
vt = self.vt(youtbe_url)
video, created = Video.get_or_create_for_url(youtbe_url)
vu = video.videourl_set.all()[:1].get()
self.assertEqual(vu.videoid, '_ShmidkrcY0')
self.assertTrue(video.title)
self.assertEqual(video.duration, 79)
self.assertTrue(video.thumbnail)
language = video.subtitlelanguage_set.all()[0]
version = language.latest_version()
self.assertEqual(len(version.subtitles()), 26)
self.assertEqual(self.vt.video_url(vu), youtbe_url)
def test_type(self):
url = 'http://someurl.com/video.flv?val=should&val1=be#removed'
clean_url = 'http://someurl.com/video.flv'
video, created = Video.get_or_create_for_url(url)
vu = video.videourl_set.all()[:1].get()
self.assertEqual(vu.url, clean_url)
self.assertEqual(self.vt.video_url(vu), vu.url)
self.assertTrue(self.vt.matches_video_url(url))
self.assertFalse(self.vt.matches_video_url('http://someurl.flv'))
self.assertFalse(self.vt.matches_video_url(''))
self.assertFalse(self.vt.matches_video_url('http://someurl.com/flv.video'))
def get_video_id(video_url):
cache_key = _video_id_key(video_url)
value = cache.get(cache_key)
if bool(value):
return value
else:
from videos.models import Video
try:
video, create = Video.get_or_create_for_url(video_url)
except VideoTypeError:
return None
if not video:
return None
video_id = video.video_id
cache.set(cache_key, video_id, TIMEOUT)
return video_id
def test_get_video_info_exception(self, mock_get_video_info):
video_info = google.VideoInfo('test-channel-id', 'title',
'description', 100,
'http://example.com/thumb.png')
mock_get_video_info.side_effect = google.APIError()
video, created = Video.get_or_create_for_url(
'http://www.youtube.com/watch?v=_ShmidkrcY0')
vu = video.videourl_set.all()[:1].get()
self.assertEqual(vu.videoid, '_ShmidkrcY0')
self.assertEqual(video.description, '')
self.assertEqual(video.duration, None)
self.assertEqual(video.thumbnail, '')
# since get_video_info failed, we don't know the channel id of our
# video URL. We should use a dummy value to make it easier to fix the
# issue in the future
self.assertEqual(vu.owner_username, None)
def setUp(self):
self.user_1 = User.objects.create(username='******')
self.user_2 = User.objects.create(username='******')
self.video = video = Video.get_or_create_for_url(
"http://www.example.com/video.mp4")[0]
video.primary_audio_language_code = 'en'
video.user = self.user_1
video.save()
mail.outbox = []
self.original_language = SubtitleLanguage.objects.create(
video=video, language_code='en')
subs = SubtitleSet.from_list('en', [
(1000, 2000, "1"),
(2000, 3000, "2"),
(3000, 4000, "3"),
])
self.original_language.add_version(subtitles=subs)
def test_type(self):
url = 'http://blip.tv/day9tv/day-9-daily-438-p3-build-orders-made-easy-newbie-tuesday-6066868'
video, created = Video.get_or_create_for_url(url)
vu = video.videourl_set.all()[:1].get()
# this is the id used to embed videos
self.assertEqual(vu.videoid, 'hdljgvKmGAI')
self.assertTrue(video.title)
self.assertTrue(video.thumbnail)
self.assertTrue(vu.url)
self.assertTrue(self.vt.matches_video_url(url))
self.assertTrue(
self.vt.matches_video_url(
'http://blip.tv/day9tv/day-9-daily-438-p3-build-orders-made-easy-newbie-tuesday-6066868'
))
self.assertFalse(self.vt.matches_video_url('http://blip.tv'))
self.assertFalse(self.vt.matches_video_url(''))
def apiFetch(f_stop):
search_url = 'https://www.googleapis.com/youtube/v3/search'
video_url = 'https://www.googleapis.com/youtube/v3/videos'
search_params = {
'part': 'snippet',
'q': 'nobody',
'key': settings.YOUTUBE_DATA_API_KEY,
'order': 'date',
'type': 'video'
}
r = requests.get(search_url, params=search_params)
results = r.json()['items']
video_ids = []
for result in results:
video_ids.append(result['id']['videoId'])
query_params = {
'key': settings.YOUTUBE_DATA_API_KEY,
'part': 'snippet,contentDetails',
'id': ','.join(video_ids),
'order': 'date'
}
r = requests.get(video_url, params=query_params)
results = r.json()['items']
videos = []
for result in results:
videos.append(
Video(title=result['snippet']['title'],
video_id=result['id'],
url=f'https://www.youtube.com/watch?v={ result["id"] }',
duration=timedelta(seconds=parse_duration(
result['contentDetails']['duration']).total_seconds()),
thumbnail=result['snippet']['thumbnails']['high']['url']))
Video.objects.bulk_create(videos, ignore_conflicts=True)
if not f_stop.is_set():
threading.Timer(10, apiFetch, [f_stop]).start()
def _create_video(self, video_type, info, entry):
from videos.models import Video
from teams.models import TeamVideo
def setup_video(video, video_url):
for name, value in info.items():
setattr(video, name, value)
if self.team:
tv = TeamVideo.objects.create(
video=video, team=self.team, added_by=self.user,
description=video.description)
try:
video, video_url = Video.add(video_type, self.user, setup_video,
self.team)
self._created_videos.append(video)
except Video.DuplicateUrlError, e:
# This shouldn't happen since we filtered out existing items
# already, but catch the exception anyways
pass
def incoming_message(request):
"""
View to receive the webhook request from slack and check for the youtube link.
If relevant link is incoming, save the data to DB
:param request:
:return: Response to slack events API, 200 status and the challenge code from the request in body
"""
request_body = json.loads(request.body.decode('utf8'))
validate_service = SlackService()
if validate_service.check_for_message(request_body):
message = request_body.get('event').get('message')
links = validate_service.retrieve_data(message)
relevant_data = validate_service.validate_links(links)
relevant_link_objs = [
Video(**link) for link in relevant_data
if Video.objects.filter(**link).count() < 1
]
Video.objects.bulk_create(relevant_link_objs)
return HttpResponse(status=200, content_type='text/plain')
def test_post_watch_video_missing_progress(self):
with NeoGraph() as graph:
obj = Video.select(graph).where(
'_.name = "{}"'.format('A')).first()
response = self.client.post(
reverse('api:watch_videos'),
json.dumps({
'data': {
'type': 'watches',
'attributes': {
'video_id': obj.id,
'date': datetime.now().isoformat(),
'rating': 1,
}
}
}),
content_type='application/vnd.api+json',
)
self.assertEqual(response.status_code, 400)
def test_signals(self, on_video_url_added, on_video_added):
def setup_callback(video, video_url):
assert_equal(on_video_added.call_count, 0)
assert_equal(on_video_url_added.call_count, 0)
video, video_url = Video.add(MockVideoType(self.url), self.user,
setup_callback)
assert_equal(on_video_added.call_count, 1)
assert_equal(
on_video_added.call_args,
mock.call(signal=signals.video_added,
sender=video,
video_url=video_url))
assert_equal(on_video_url_added.call_count, 1)
assert_equal(
on_video_url_added.call_args,
mock.call(signal=signals.video_url_added,
sender=video_url,
video=video,
new_video=True))
def setUp(self):
self.user_1 = User.objects.create(username='******',
notify_by_email=False)
self.user_2 = User.objects.create(username='******',
notify_by_email=False)
def setup_video(video, video_url):
video.primary_audio_language_code = 'en'
self.video = video = Video.add("http://www.example.com/video.mp4",
self.user_1)[0]
mail.outbox = []
self.original_language = SubtitleLanguage.objects.create(
video=video, language_code='en')
subs = SubtitleSet.from_list('en', [
(1000, 2000, "1"),
(2000, 3000, "2"),
(3000, 4000, "3"),
])
self.original_language.add_version(subtitles=subs)
def get_video_id(video_url, public_only=False, referer=None):
"""
Returns the cache video_id for this video
If public only is
"""
cache_key = _video_id_key(video_url)
value = cache.get(cache_key)
if bool(value):
return value
else:
from videos.models import Video
try:
video, _ = Video.add(video_url, None)
except Video.DuplicateUrlError, e:
video = e.video
video_id = video.video_id
cache.set(cache_key, video_id, TIMEOUT)
return video_id
def subtitle_existence(request):
video_url = request.GET.get('video_url')
if video_url is None:
return {'is_error': True, 'message': 'video_url not specified'}
video, created = Video.get_or_create_for_url(video_url)
if not video:
return {'is_error': True, 'message': 'unsuported video url'}
output = []
for item in video.subtitlelanguage_set.all():
output.append({
'code': item.language,
'name': item.get_language_display(),
'is_original': item.is_original
})
return output
def create(self, request):
data = request.data.get("data")
copy_from = request.data.get("copy_from")
data.pop("videos", None)
data.pop("id", None)
album = self.queryset.create(**data, user=request.user)
if copy_from:
copy_album = self.get_object(copy_from)
if copy_album:
videos_to_create = [
Video(name=v.name, youtube_id=v.youtube_id, album=album)
for v in copy_album.videos.all()
]
videos = Video.objects.bulk_create(videos_to_create)
serializer = self.serializer_class(album)
return JsonResponse(serializer.data)
def test_type(self):
url = 'http://someurl.com/video.ogv?val=should&val1=be#removed'
clean_url = 'http://someurl.com/video.ogv'
video, created = Video.get_or_create_for_url(url)
vu = video.videourl_set.all()[:1].get()
self.assertEqual(vu.url, clean_url)
self.assertEqual(self.vt.video_url(vu), vu.url)
self.assertTrue(self.vt.matches_video_url(url))
self.assertTrue(self.vt.matches_video_url('http://someurl.com/video.ogg'))
self.assertTrue(self.vt.matches_video_url('http://someurl.com/video.mp4'))
self.assertTrue(self.vt.matches_video_url('http://someurl.com/video.m4v'))
self.assertTrue(self.vt.matches_video_url('http://someurl.com/video.webm'))
self.assertFalse(self.vt.matches_video_url('http://someurl.ogv'))
self.assertFalse(self.vt.matches_video_url(''))
#for this is other type
self.assertFalse(self.vt.matches_video_url('http://someurl.com/video.flv'))
self.assertFalse(self.vt.matches_video_url('http://someurl.com/ogv.video'))
def subtitles(request):
callback = request.GET.get('callback')
video_url = request.GET.get('video_url')
language = request.GET.get('language')
revision = request.GET.get('revision')
format = request.GET.get('format', 'json')
if video_url is None:
return {'is_error': True, 'message': 'video_url not specified'}
video, created = Video.get_or_create_for_url(video_url)
if not video:
return {'is_error': True, 'message': 'unsuported video url'}
if format == 'json':
output = [
s.for_json() for s in video.subtitles(version_no=revision,
language_code=language)
]
if callback:
result = json.dumps(output)
return HttpResponse('%s(%s);' % (callback, result),
'text/javascript')
else:
return output
else:
handler = GenerateSubtitlesHandler.get(format)
if not handler:
return {'is_error': True, 'message': 'undefined format'}
subtitles = [
s.for_generator() for s in video.subtitles(version_no=revision,
language_code=language)
]
h = handler(subtitles, video)
return HttpResponse(unicode(h))
def clean_video_url(self):
video_url = self.cleaned_data['video_url']
if not video_url:
self.video = None
return video_url
host = Site.objects.get_current().domain
url_start = 'http://'+host
if video_url.startswith(url_start):
#UniSub URL
locale, path = strip_path(video_url[len(url_start):])
video_url = url_start+path
try:
video_url = self.format_url(video_url)
func, args, kwargs = resolve(video_url.replace(url_start, ''))
if not 'video_id' in kwargs:
raise forms.ValidationError(_('This URL does not contain video id.'))
try:
self.video = Video.objects.get(video_id=kwargs['video_id'])
except Video.DoesNotExist:
raise forms.ValidationError(_('Videos does not exist.'))
except Http404:
raise forms.ValidationError(_('Incorrect URL.'))
else:
#URL from other site
try:
self.video, created = Video.get_or_create_for_url(video_url)
except VideoTypeError, e:
raise forms.ValidationError(e)
if not self.video:
raise forms.ValidationError(mark_safe(_(u"""Universal Subtitles does not support that website or video format.
If you'd like to us to add support for a new site or format, or if you
think there's been some mistake, <a
href="mailto:%s">contact us</a>!""") % settings.FEEDBACK_EMAIL))
def post(self, request):
data = request.data.get("data")
playlist_id = request.data.get("playlist_id")
data.pop("videos", None)
data.pop("id", None)
user_id = request.user.pk
video_info = import_videos(playlist_id)
album = Album.objects.create(**data, user=request.user)
videos = Video.objects.bulk_create(
list(
map(
lambda vinfo: Video(
name=vinfo["name"],
youtube_id=vinfo["id"],
album=album,
), video_info)))
serializer = self.serializer_class(album)
return JsonResponse(serializer.data, safe=False)
def dismiss_and_next_videos(self, video_name, next_videos):
with NeoGraph() as graph:
obj = Video.select(graph).where(
'_.name = "{}"'.format(video_name)).first()
response = self.client.post(
reverse('api:watch_videos'),
json.dumps({
'data': {
'type': 'watches',
'attributes': {
'video_id': obj.id,
'date': datetime.now().isoformat(),
'rating': -1,
'progress': 0,
}
}
}),
content_type='application/vnd.api+json',
)
self.assertEqual(response.status_code, 204)
response = self.client.get(reverse('api:next_videos'))
self.assertEqual(response.status_code, 200)
self.assertJSONDataVideoNames(response, next_videos)
def create(self, validated_data):
def setup_video(video, video_url):
for key in ('title', 'description', 'duration', 'thumbnail',
'primary_audio_language_code'):
if validated_data.get(key):
setattr(video, key, validated_data[key])
if validated_data.get('metadata'):
video.update_metadata(validated_data['metadata'], commit=False)
try:
team = validated_data.get('team')
if team:
video, video_url = team.add_video(
validated_data['video_url'], self.context['user'],
self.calc_project(validated_data), setup_video)
else:
video, video_url = Video.add(validated_data['video_url'],
self.context['user'], setup_video)
return video
except VideoTypeError:
self.fail('invalid-url', url=validated_data['video_url'])
except Video.DuplicateUrlError:
self.fail('video-exists', url=validated_data['video_url'])
def get_video_id(video_url, public_only=False, referer=None):
"""
Returns the cache video_id for this video
If public only is
"""
cache_key = _video_id_key(video_url)
value = cache.get(cache_key)
if bool(value):
return value
else:
from videos.models import Video
try:
video, create = Video.get_or_create_for_url(video_url)
except VideoTypeError:
raise
if not video:
return None
video_id = video.video_id
cache.set(cache_key, video_id, TIMEOUT)
return video_id
def test_url_cache(self):
test_utils.invalidate_widget_video_cache.run_original_for_test()
video = get_video(1)
video_url = video.get_video_url()
# After adding the video, we should be able to look up its ID in the
# cache, given just the URL.
cache_id_1 = video_cache.get_video_id(video_url)
self.assertIsNotNone(cache_id_1)
self.assertTrue(Video.objects.filter(video_id=cache_id_1).exists())
# Remove the video (and make sure it's gone).
video.delete()
self.assertFalse(Video.objects.exists())
# Trying to get the video ID out of the cache now actually *creates* the
# video!
cache_id_2 = video_cache.get_video_id(video_url)
self.assertTrue(Video.objects.exists())
# The video_id will be different than before (since this is a new Video
# record) and the cache should have been updated properly.
self.assertNotEqual(cache_id_1, cache_id_2)
self.assertTrue(Video.objects.filter(video_id=cache_id_2).exists())
# Now try to create a new video with the same URL. This should return
# the existing video.
video2, created = Video.get_or_create_for_url(video_url)
self.assertFalse(created)
video2_url = video2.get_video_url()
# The cache should still have the correct ID, of course.
cache_id_3 = video_cache.get_video_id(video2_url)
self.assertEqual(cache_id_2, cache_id_3)
self.assertEqual(Video.objects.count(), 1)
def get_video(n=1, user=None, url=None):
if not url:
url = VIDEO_URLS[n]
video, _ = Video.get_or_create_for_url(url, user=user)
return video
def test_add_credit_on_new_video(self):
video, video_url = Video.add('http://youtube.com/watch?v=abcdef',
self.user)
self.mock_add_amara_credit.delay.assert_called_with(video_url.id)
def test_get_index(self):
app_config = apps.get_app_config('videos')
create_index(app_config, 'Video')
self.assertTrue(index_exists(Video.get_index_name()))
index = get_index(Video.get_index_name())
self.assertTrue(isinstance(index, dict))
def test_notify_by_message_false(self):
self.user.notify_by_message = False
video, video_url = Video.add(MockVideoType(self.url), self.user)
assert_false(video.followers.filter(id=self.user.id).exists())
def test_convert_video_url(self):
# We should allow the VideoType to alter the URL using the
# convert_to_video_url() method.
new_url = 'http://example.com/new_url.mp4'
video, video_url = Video.add(MockVideoType(new_url), self.user)
assert_equal(video_url.url, new_url)
def test_null_videoid(self):
# Test VideoType.video_id being None
mock_video_type = MockVideoType(self.url)
mock_video_type.video_id = None
video, video_url = Video.add(mock_video_type, self.user)
assert_equal(video_url.videoid, '')
def test_string_url(self, mock_registrar):
video, video_url = Video.add(self.url, self.user)
assert_equal(mock_registrar.video_type_for_url.call_args,
mock.call(self.url))
assert_equal(video_url.type, MockVideoType.abbreviation)
def create_new_videos(videos_info):
videos = []
for video_info in videos_info:
video = Video(yt_id=video_info['id'],
yt_title=video_info['title'],
published_at_yt=video_info['published_at'])
video.title = clean_video_title(video_info['title'])
video.description = clean_video_description(video_info['description'])
video.save()
videos.append(video)
request = requests.get(video_info['thumbnail_url'], stream=True)
if request.status_code == requests.codes.ok:
tempimg = tempfile.NamedTemporaryFile()
for block in request.iter_content(1024 * 8):
if not block:
break
tempimg.write(block)
video.thumbnail = files.File(tempimg)
video.save()
# make thumbnail be "cut" in the right ratio for the video thumbnails
ratio = 275.0 / 154.0
t_width = int(
min(video.thumbnail.width, video.thumbnail.height * ratio))
t_height = int(
min(video.thumbnail.width / ratio, video.thumbnail.height))
#center the cut
y_origin = video.thumbnail.height / 2 - t_height / 2
y_end = y_origin + t_height
t_ratio = '0,' + str(y_origin) + str(t_width) + ',' + str(y_end)
video.thumbnail_ratio = t_ratio
video.save()
if 'tags' in video_info:
for video_tag in video_info['tags']:
tag = Tag.objects.create(video=video, name=video_tag)
return videos
def save(self):
video_url = self.cleaned_data['video_url']
obj, created = Video.get_or_create_for_url(video_url, self._video_type,
self.user)
self.created = created
return obj
