def test_not_secure_token(self, p_urllib2, p_logging):
def mocked_urlopen(request):
return StringIO("""
<?xml version="1.0"?>
<Response>
<Message>Error</Message>
<MessageCode>7.5</MessageCode>
<Errors>
<Error>
<ErrorCode>8.1</ErrorCode>
<ErrorName>Short URL is not protected</ErrorName>
<Description>bla bla</Description>
<Suggestion>ble ble</Suggestion>
</Error>
</Errors>
</Response>
""")
p_urllib2.urlopen = mocked_urlopen
eq_(vidly.tokenize('abc123', 60), '')
# do it a second time and it should be cached
def mocked_urlopen_different(request):
return StringIO("""
Anything different
""")
p_urllib2.urlopen = mocked_urlopen_different
eq_(vidly.tokenize('abc123', 60), '')
def test_secure_token(self, p_urllib2):
event = Event.objects.get(title='Test event')
submission = VidlySubmission.objects.create(event=event, tag='xyz123')
tokenize_calls = [] # globally scope mutable
def mocked_urlopen(request):
tokenize_calls.append(1)
return StringIO("""
<?xml version="1.0"?>
<Response>
<Message>OK</Message>
<MessageCode>7.4</MessageCode>
<Success>
<MediaShortLink>8r9e0o</MediaShortLink>
<Token>MXCsxINnVtycv6j02ZVIlS4FcWP</Token>
</Success>
</Response>
""")
p_urllib2.urlopen = mocked_urlopen
eq_(vidly.tokenize(submission.tag, 60), 'MXCsxINnVtycv6j02ZVIlS4FcWP')
eq_(len(tokenize_calls), 1)
# do it a second time
eq_(vidly.tokenize(submission.tag, 60), 'MXCsxINnVtycv6j02ZVIlS4FcWP')
eq_(len(tokenize_calls), 1) # caching for the win!
submission.token_protection = True
submission.save()
eq_(vidly.tokenize(submission.tag, 60), 'MXCsxINnVtycv6j02ZVIlS4FcWP')
eq_(len(tokenize_calls), 2) # cache got invalidated
def test_secure_token(self, p_urllib2):
event = Event.objects.get(title="Test event")
submission = VidlySubmission.objects.create(event=event, tag="xyz123")
tokenize_calls = [] # globally scope mutable
def mocked_urlopen(request):
tokenize_calls.append(1)
return StringIO(
"""
<?xml version="1.0"?>
<Response>
<Message>OK</Message>
<MessageCode>7.4</MessageCode>
<Success>
<MediaShortLink>8r9e0o</MediaShortLink>
<Token>MXCsxINnVtycv6j02ZVIlS4FcWP</Token>
</Success>
</Response>
"""
)
p_urllib2.urlopen = mocked_urlopen
eq_(vidly.tokenize(submission.tag, 60), "MXCsxINnVtycv6j02ZVIlS4FcWP")
eq_(len(tokenize_calls), 1)
# do it a second time
eq_(vidly.tokenize(submission.tag, 60), "MXCsxINnVtycv6j02ZVIlS4FcWP")
eq_(len(tokenize_calls), 1) # caching for the win!
submission.token_protection = True
submission.save()
eq_(vidly.tokenize(submission.tag, 60), "MXCsxINnVtycv6j02ZVIlS4FcWP")
eq_(len(tokenize_calls), 2) # cache got invalidated
def _set_csp_update(self, response, event):
"""Hack alert!
We need to, potentially, update the CSP at run time if the
video you're trying to watch is a Vid.ly video.
Vid.ly is embedded by simply using `https://vid.ly/:shortcode`
but internally they will redirect to a AWS CloudFront domain
which we might not have prepared in our CSP settings.
So let's update that on the fly.
"""
cache_key = 'custom_csp_update:{}'.format(event.id)
update = cache.get(cache_key)
if update is not None:
# it was set, use that and exit early
if update:
response._csp_update = update
return
if not event.template:
return
if event.is_upcoming() or event.is_live():
return
if 'vid.ly' not in event.template.name.lower():
return
if not event.template_environment.get('tag'):
return
token = None
if not event.is_public():
token = vidly.tokenize(event.template_environment['tag'], 90)
update = {}
webm_url = 'https://vid.ly/{}?content=video&format=webm'.format(
event.template_environment['tag'])
if token:
webm_url += '&token={}'.format(token)
head_response = requests.head(webm_url)
if head_response.status_code == 302:
update['media-src'] = urlparse.urlparse(
head_response.headers['Location']).netloc
poster_url = 'https://vid.ly/{}/poster'.format(
event.template_environment['tag'])
if token:
poster_url += '?token={}'.format(token)
head_response = requests.head(poster_url)
if head_response.status_code == 302:
update['img-src'] = urlparse.urlparse(
head_response.headers['Location']).netloc
cache.set(cache_key, update, 60)
# Now we've figured out what headers to update, set it on the response
if update:
response._csp_update = update
def get_vidly_csp_headers(tag, private=False):
token = None
if private:
token = vidly.tokenize(tag, 90)
headers = {}
def get_netloc(type_, url_format):
netloc = None
try:
found = VidlyTagDomain.objects.get(
tag=tag,
type=type_,
)
if found.private != private:
# The tag has changed!
found.delete()
raise VidlyTagDomain.DoesNotExist
else:
netloc = found.domain
except VidlyTagDomain.DoesNotExist:
url = url_format.format(
tag
)
if token:
url += '&token={}'.format(token)
head_response = requests.head(url)
if head_response.status_code == 302:
netloc = urlparse.urlparse(
head_response.headers['Location']
).netloc
assert netloc, head_response.headers['Location']
VidlyTagDomain.objects.create(
tag=tag,
type=type_,
private=private,
domain=netloc,
)
return netloc
media_netloc = get_netloc('webm', settings.VIDLY_VIDEO_URL_FORMAT)
if media_netloc:
headers['media-src'] = media_netloc
img_netloc = get_netloc('poster', settings.VIDLY_POSTER_URL_FORMAT)
if img_netloc:
headers['img-src'] = img_netloc
return headers
def test_secure_token(self, p_urllib2, p_logging):
def mocked_urlopen(request):
return StringIO("""
<?xml version="1.0"?>
<Response>
<Message>OK</Message>
<MessageCode>7.4</MessageCode>
<Success>
<MediaShortLink>8r9e0o</MediaShortLink>
<Token>MXCsxINnVtycv6j02ZVIlS4FcWP</Token>
</Success>
</Response>
""")
p_urllib2.urlopen = mocked_urlopen
eq_(vidly.tokenize('xyz123', 60),
'MXCsxINnVtycv6j02ZVIlS4FcWP')
def get_vidly_csp_headers(tag, private=False):
token = None
if private:
token = vidly.tokenize(tag, 90)
headers = {}
def get_netloc(type_, url_format):
netloc = None
try:
found = VidlyTagDomain.objects.get(
tag=tag,
type=type_,
)
if found.private != private:
# The tag has changed!
found.delete()
raise VidlyTagDomain.DoesNotExist
else:
netloc = found.domain
except VidlyTagDomain.DoesNotExist:
url = url_format.format(tag)
if token:
url += '&token={}'.format(token)
head_response = requests.head(url)
if head_response.status_code == 302:
netloc = urlparse.urlparse(
head_response.headers['Location']).netloc
assert netloc, head_response.headers['Location']
VidlyTagDomain.objects.create(
tag=tag,
type=type_,
private=private,
domain=netloc,
)
return netloc
media_netloc = get_netloc('webm', settings.VIDLY_VIDEO_URL_FORMAT)
if media_netloc:
headers['media-src'] = media_netloc
img_netloc = get_netloc('poster', settings.VIDLY_POSTER_URL_FORMAT)
if img_netloc:
headers['img-src'] = img_netloc
return headers
def test_invalid_response_token(self, p_urllib2, p_logging):
def mocked_urlopen(request):
return StringIO("""
<?xml version="1.0"?>
<Response>
<Message>Error</Message>
<MessageCode>99</MessageCode>
<Errors>
<Error>
<ErrorCode>0.0</ErrorCode>
<ErrorName>Some other error</ErrorName>
<Description>bla bla</Description>
<Suggestion>ble ble</Suggestion>
</Error>
</Errors>
</Response>
""")
p_urllib2.urlopen = mocked_urlopen
eq_(vidly.tokenize('def123', 60), None)
p_logging.error.asert_called_with(
"Unable fetch token for tag 'abc123'"
)
def get_video_url(event, use_https, save_locally, verbose=False):
if event.upload:
return event.upload.url, None
elif event.template and 'Vid.ly' in event.template.name:
assert event.template_environment.get('tag'), "No Vid.ly tag value"
token_protected = event.privacy != Event.PRIVACY_PUBLIC
hd = False
qs = (VidlySubmission.objects.filter(event=event).filter(
tag=event.template_environment['tag']))
for submission in qs.order_by('-submission_time')[:1]:
hd = submission.hd
token_protected = submission.token_protection
tag = event.template_environment['tag']
video_url = '%s/%s?content=video&format=' % (
settings.VIDLY_BASE_URL,
tag,
)
if hd:
video_url += 'hd_mp4'
else:
video_url += 'mp4'
if token_protected:
video_url += '&token=%s' % vidly.tokenize(tag, 60)
elif event.template and 'Ogg Video' in event.template.name:
assert event.template_environment.get('url'), "No Ogg Video url value"
video_url = event.template_environment['url']
elif event.template and 'YouTube' in event.template.name:
assert event.template_environment.get('id'), "No YouTube ID value"
video_url = 'https://www.youtube.com/watch?v={}'.format(
event.template_environment['id'])
return video_url, None
else:
raise AssertionError("Not valid template")
response = requests.head(video_url)
_count = 0
while response.status_code in (302, 301):
video_url = response.headers['Location']
response = requests.head(video_url)
_count += 1
if _count > 5:
# just too many times
break
response = requests.head(video_url)
assert response.status_code == 200, response.status_code
if verbose: # pragma: no cover
if response.headers.get('Content-Length'):
print("Content-Length: %s" %
(filesizeformat(int(response.headers['Content-Length'])), ))
if not use_https:
video_url = video_url.replace('https://', 'http://')
if save_locally:
# store it in a temporary location
dir_ = tempfile.mkdtemp('videoinfo')
if 'Vid.ly' in event.template.name:
filepath = os.path.join(dir_, '%s.mp4' % tag)
else:
filepath = os.path.join(
dir_, os.path.basename(urlparse.urlparse(video_url).path))
t0 = time.time()
_download_file(video_url, filepath)
t1 = time.time()
if verbose: # pragma: no cover
seconds = int(t1 - t0)
print "Took", show_duration(seconds, include_seconds=True),
print "to download"
video_url = filepath
else:
filepath = None
return video_url, filepath
def _get_video_url(event, use_https, save_locally, verbose=False):
if event.template and 'Vid.ly' in event.template.name:
assert event.template_environment.get('tag'), "No Vid.ly tag value"
token_protected = event.privacy != Event.PRIVACY_PUBLIC
hd = False
qs = (
VidlySubmission.objects
.filter(event=event)
.filter(tag=event.template_environment['tag'])
)
for submission in qs.order_by('-submission_time')[:1]:
hd = submission.hd
token_protected = submission.token_protection
tag = event.template_environment['tag']
video_url = '%s/%s?content=video&format=' % (
settings.VIDLY_BASE_URL,
tag,
)
if hd:
video_url += 'hd_mp4'
else:
video_url += 'mp4'
if token_protected:
video_url += '&token=%s' % vidly.tokenize(tag, 60)
elif event.template and 'Ogg Video' in event.template.name:
assert event.template_environment.get('url'), "No Ogg Video url value"
video_url = event.template_environment['url']
else:
raise AssertionError("Not valid template")
response = requests.head(video_url)
_count = 0
while response.status_code in (302, 301):
video_url = response.headers['Location']
response = requests.head(video_url)
_count += 1
if _count > 5:
# just too many times
break
response = requests.head(video_url)
assert response.status_code == 200, response.status_code
if verbose: # pragma: no cover
if response.headers.get('Content-Length'):
print (
"Content-Length: %s" % (
filesizeformat(int(response.headers['Content-Length'])),
)
)
if not use_https:
video_url = video_url.replace('https://', 'http://')
if save_locally:
# store it in a temporary location
dir_ = tempfile.mkdtemp('videoinfo')
if 'Vid.ly' in event.template.name:
filepath = os.path.join(dir_, '%s.mp4' % tag)
else:
filepath = os.path.join(
dir_,
os.path.basename(urlparse.urlparse(video_url).path)
)
t0 = time.time()
_download_file(video_url, filepath)
t1 = time.time()
if verbose: # pragma: no cover
seconds = int(t1 - t0)
print "Took", show_duration(seconds, include_seconds=True),
print "to download"
video_url = filepath
else:
filepath = None
return video_url, filepath
def get_vidly_csp_headers(tag, private=False):
token = None
if private:
token = vidly.tokenize(tag, 90)
headers = {}
def get_netloc(type_, url_format):
netloc = None
try:
found = VidlyTagDomain.objects.get(
tag=tag,
type=type_,
)
if found.private != private:
# The tag has changed!
found.delete()
raise VidlyTagDomain.DoesNotExist
elif found.domain == 'm.vid.ly': # pragma: no cover
# In a previous life, airmozilla might have attempted to
# look up what the CDN domain was and if it failed,
# Vid.ly would just redirect to 'https://m.vid.ly' which
# is NOT the right CDN domain. We shouldn't have stored
# that.
# This knowledge was added in June 2017 and from now on
# we never save this as the domain so it should cease.
found.delete()
raise VidlyTagDomain.DoesNotExist
else:
netloc = found.domain
except VidlyTagDomain.DoesNotExist:
url = url_format.format(tag)
if token:
url += '&token={}'.format(token)
head_response = requests.head(url)
if head_response.status_code == 302:
if head_response.headers['Location'] == 'https://m.vid.ly':
# Basically, it didn't work.
# When vid.ly can't redirect to the actual file, for
# some reason it instead redirects to the exact
# URL 'https://m.vid.ly'. For example:
#
# curl -v https://vid.ly/l1c2w5/blalbla
# ...
# < HTTP/1.1 302 Found
# ...
# < Location: https://m.vid.ly
#
# Odd right? But it basically means to use that we
# we not able to do the lookup. Sorry.
return
netloc = urlparse.urlparse(
head_response.headers['Location']).netloc
assert netloc, head_response.headers['Location']
VidlyTagDomain.objects.create(
tag=tag,
type=type_,
private=private,
domain=netloc,
)
return netloc
media_netloc = get_netloc('webm', settings.VIDLY_VIDEO_URL_FORMAT)
if media_netloc:
headers['media-src'] = media_netloc
# In almost all cases, the poster image is on the same domain
# as the video. So let's use that.
# Later we're going to try to do a specific lookup for the poster.
# If that's better/different that becomes the added domain
# for 'img-src' instead.
headers['img-src'] = media_netloc
# There is no way to pre-lookup what the actual CDN domain is
# for the webvtt.vtt file is so let's hope for the best and
# reuse the the domain for media on the connect-src too.
headers['connect-src'] = media_netloc
lock_cache_key = 'poster_netloc_failed:{}'.format(tag)
if not cache.get(lock_cache_key):
img_netloc = get_netloc('poster', settings.VIDLY_POSTER_URL_FORMAT)
if img_netloc:
headers['img-src'] = img_netloc
else:
# If that failed, don't bother trying again. For a while.
cache.set(lock_cache_key, True, 60 * 60)
return headers
def fetch_duration(
event, save=False, save_locally=False, verbose=False, use_https=True,
):
assert 'Vid.ly' in event.template.name, "Not a Vid.ly template"
assert event.template_environment.get('tag'), "No Vid.ly tag in template"
hd = False
# This is commented out for the time being because we don't need the
# HD version to just capture the duration.
# qs = VidlySubmission.objects.filter(event=event)
# for submission in qs.order_by('-submission_time')[:1]:
# hd = submission.hd
tag = event.template_environment['tag']
vidly_url = 'https://vid.ly/%s?content=video&format=' % tag
if hd:
vidly_url += 'hd_mp4'
else:
vidly_url += 'mp4'
if event.privacy != Event.PRIVACY_PUBLIC:
vidly_url += '&token=%s' % vidly.tokenize(tag, 60)
response = requests.head(vidly_url)
if response.status_code == 302:
vidly_url = response.headers['Location']
response = requests.head(vidly_url)
assert response.status_code == 200, response.status_code
if verbose: # pragma: no cover
if response.headers['Content-Length']:
print "Content-Length:",
print filesizeformat(int(response.headers['Content-Length']))
if not use_https:
vidly_url = vidly_url.replace('https://', 'http://')
if save_locally:
# store it in a temporary location
dir_ = tempfile.mkdtemp('videoinfo')
filepath = os.path.join(dir_, '%s.mp4' % tag)
t0 = time.time()
_download_file(vidly_url, filepath)
t1 = time.time()
if verbose: # pragma: no cover
seconds = int(t1 - t0)
print "Took", show_duration(seconds, include_seconds=True),
print "to download"
vidly_url = filepath
try:
ffmpeg_location = getattr(
settings,
'FFMPEG_LOCATION',
'ffmpeg'
)
command = [
ffmpeg_location,
'-i',
vidly_url,
]
if verbose: # pragma: no cover
print ' '.join(command)
out, err = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
).communicate()
matches = REGEX.findall(err)
if matches:
found, = matches
hours = int(found[0])
minutes = int(found[1])
minutes += hours * 60
seconds = int(found[2])
seconds += minutes * 60
if save:
event.duration = seconds
event.save()
return seconds
elif verbose: # pragma: no cover
print "No Duration output. Error:"
print err
finally:
if save_locally:
if os.path.isfile(filepath):
shutil.rmtree(os.path.dirname(filepath))
def get_vidly_csp_headers(tag, private=False):
token = None
if private:
token = vidly.tokenize(tag, 90)
headers = {}
def get_netloc(type_, url_format):
netloc = None
try:
found = VidlyTagDomain.objects.get(
tag=tag,
type=type_,
)
if found.private != private:
# The tag has changed!
found.delete()
raise VidlyTagDomain.DoesNotExist
elif found.domain == 'm.vid.ly': # pragma: no cover
# In a previous life, airmozilla might have attempted to
# look up what the CDN domain was and if it failed,
# Vid.ly would just redirect to 'https://m.vid.ly' which
# is NOT the right CDN domain. We shouldn't have stored
# that.
# This knowledge was added in June 2017 and from now on
# we never save this as the domain so it should cease.
found.delete()
raise VidlyTagDomain.DoesNotExist
else:
netloc = found.domain
except VidlyTagDomain.DoesNotExist:
url = url_format.format(
tag
)
if token:
url += '&token={}'.format(token)
head_response = requests.head(url)
if head_response.status_code == 302:
if head_response.headers['Location'] == 'https://m.vid.ly':
# Basically, it didn't work.
# When vid.ly can't redirect to the actual file, for
# some reason it instead redirects to the exact
# URL 'https://m.vid.ly'. For example:
#
# curl -v https://vid.ly/l1c2w5/blalbla
# ...
# < HTTP/1.1 302 Found
# ...
# < Location: https://m.vid.ly
#
# Odd right? But it basically means to use that we
# we not able to do the lookup. Sorry.
return
netloc = urlparse.urlparse(
head_response.headers['Location']
).netloc
assert netloc, head_response.headers['Location']
VidlyTagDomain.objects.create(
tag=tag,
type=type_,
private=private,
domain=netloc,
)
return netloc
media_netloc = get_netloc('webm', settings.VIDLY_VIDEO_URL_FORMAT)
if media_netloc:
headers['media-src'] = media_netloc
# In almost all cases, the poster image is on the same domain
# as the video. So let's use that.
# Later we're going to try to do a specific lookup for the poster.
# If that's better/different that becomes the added domain
# for 'img-src' instead.
headers['img-src'] = media_netloc
# There is no way to pre-lookup what the actual CDN domain is
# for the webvtt.vtt file is so let's hope for the best and
# reuse the the domain for media on the connect-src too.
headers['connect-src'] = media_netloc
lock_cache_key = 'poster_netloc_failed:{}'.format(tag)
if not cache.get(lock_cache_key):
img_netloc = get_netloc('poster', settings.VIDLY_POSTER_URL_FORMAT)
if img_netloc:
headers['img-src'] = img_netloc
else:
# If that failed, don't bother trying again. For a while.
cache.set(lock_cache_key, True, 60 * 60)
return headers