def extendMarkdown(self, md, md_globals):
if self.renderer:
pyembed = PyEmbed(renderer=self.renderer)
else:
pyembed = PyEmbed()
md.inlinePatterns.add(
'pyembed', pattern.PyEmbedPattern(pyembed, md), '_begin')
def fetch_oembed_preview(content, urls):
"""Fetch first oembed content for a list of urls."""
for url in urls:
# See first if recently cached already
if OEmbedCache.objects.filter(url=url,
modified__gte=now() -
datetime.timedelta(days=7)).exists():
oembed = OEmbedCache.objects.get(url=url)
Content.objects.filter(id=content.id).update(oembed=oembed)
return oembed
# Fetch oembed
try:
oembed = PyEmbed(discoverer=OEmbedDiscoverer()).embed(url)
except (PyEmbedError, PyEmbedDiscoveryError, PyEmbedConsumerError,
ValueError):
continue
if not oembed:
continue
# Ensure width is 100% not fixed
oembed = re.sub(r'width="[0-9]*"', 'width="100%"', oembed)
oembed = re.sub(r'height="[0-9]*"', "", oembed)
try:
with transaction.atomic():
oembed = OEmbedCache.objects.create(url=url, oembed=oembed)
except IntegrityError:
# Some other process got ahead of us
oembed = OEmbedCache.objects.get(url=url)
Content.objects.filter(id=content.id).update(oembed=oembed)
return oembed
Content.objects.filter(id=content.id).update(oembed=oembed)
return oembed
return False
def get(url):
''' get obtains the html for an embedded video '''
embed_html = cache.get(url)
if embed_html == None:
start = time.time()
try:
embed_html = PyEmbed().embed(url)
dt = int((time.time() - start) * 1000)
if 'youtube' in url:
statsd.timing('oembed.get.youtube', dt)
elif 'twitch' in url:
statsd.timing('oembed.get.twitch', dt)
elif 'twitter' in url:
statsd.timing('oembed.get.twitter', dt)
else:
statsd.timing('oembed.get.other', dt)
cache.set(url, embed_html)
statsd.incr('oembed.cache.miss')
except KeyError as error:
log.warning("Could not obtain clip from oembed for url: %s", url)
raise
else:
statsd.incr('oembed.cache.hit')
return Markup(embed_html)
def fetch_oembed_preview(content, urls):
"""Fetch first oembed content for a list of urls."""
for url in urls:
# See first if recently cached already
if OEmbedCache.objects.filter(url=url, modified__gte=now()-datetime.timedelta(days=7)).exists():
oembed = OEmbedCache.objects.get(url=url)
Content.objects.filter(id=content.id).update(oembed=oembed)
return oembed
# Fetch oembed
options = {}
if url.startswith("https://twitter.com/"):
# This probably has little effect since we fetch these on the backend...
# But, DNT is always good to communicate if possible :)
options = {"dnt": "true", "omit_script": "true"}
try:
oembed = PyEmbed(discoverer=OEmbedDiscoverer()).embed(url, **options)
except (PyEmbedError, PyEmbedDiscoveryError, PyEmbedConsumerError, ValueError):
continue
if not oembed:
continue
# Ensure width is 100% not fixed
oembed = re.sub(r'width="[0-9]*"', 'width="100%"', oembed)
oembed = re.sub(r'height="[0-9]*"', "", oembed)
try:
with transaction.atomic():
oembed = OEmbedCache.objects.create(url=url, oembed=oembed)
except IntegrityError:
# Some other process got ahead of us
oembed = OEmbedCache.objects.get(url=url)
Content.objects.filter(id=content.id).update(oembed=oembed)
return oembed
Content.objects.filter(id=content.id).update(oembed=oembed)
return oembed
return False
def embed(self):
"""Get the html to embed into the crowdsource"""
try:
# first try to get embed code from oEmbed
return mark_safe(
PyEmbed(
# we don't use the default discoverer because it contains a bug
# that makes it always match spotify
discoverer=ChainingDiscoverer([
FileDiscoverer(
resource_filename(
__name__, 'oembed_providers.json'
)
),
AutoDiscoverer(),
])
).embed(self.url, max_height=400)
)
except PyEmbedConsumerError:
# if this is a private document cloud document, it will not have
# an oEmbed, create the embed manually
doc_match = DOCUMENT_URL_RE.match(self.url)
if doc_match:
return mark_safe(
DOCCLOUD_EMBED.format(doc_id=doc_match.group('doc_id'))
)
else:
# fall back to a simple iframe
return format_html(
'<iframe src="{}" width="100%" height="400px"></iframe>',
self.url,
)
def embed(self):
"""Get the html to embed into the crowdsource"""
if self.url:
try:
# first try to get embed code from oEmbed
return mark_safe(
PyEmbed(
# we don't use the default discoverer because it contains a bug
# that makes it always match spotify
discoverer=ChainingDiscoverer(
[
FileDiscoverer(
resource_filename(__name__, "oembed_providers.json")
),
AutoDiscoverer(),
]
)
).embed(self.url, max_height=400)
)
except PyEmbedConsumerError:
# fall back to a simple iframe
return format_html(
'<iframe src="{}" width="100%" height="400px"></iframe>', self.url
)
else:
return ""
def test_should_throw_error_on_request_error():
with pytest.raises(PyEmbedError):
discoverer = Mock()
discoverer.get_oembed_urls.side_effect = RequestException()
PyEmbed(discoverer, None).embed('http://example.com/',
maxwidth=100,
maxheight=200)
def render(self, *args, **kwargs):
self.rendered_text = markdown.markdown(self.text)
if self.text_type == self.TEXT_TYPE.SIMPLE:
self.rendered_text = self.text
elif self.text_type == self.TEXT_TYPE.TEXTILE:
self.rendered_text = textile.textile(self.text)
soup = BeautifulSoup(self.rendered_text, 'html.parser')
matching_text_nodes = soup.find_all(text = re.compile(URL_REGEX))
pyembed = PyEmbed()
for matching_text_node in matching_text_nodes:
plain_text = six.text_type(matching_text_node)
for each in re.findall(URL_REGEX, plain_text):
plain_text = plain_text.replace(each, '<a href="%s">%s</a>' % (each,each), 1)
if matching_text_node.parent.text == each: # if url on one line, each == url == matching_text_node
try:
oembed = pyembed.embed(matching_text_node)
except PyEmbedError:
matching_text_node.replace_with(plain_text)
else:
matching_text_node.replace_with(oembed)
else:
matching_text_node.replace_with(plain_text)
soup = BeautifulSoup(soup.encode(formatter=None), 'html.parser')
self.rendered_text = soup.encode(formatter=None).decode()
def embed(self):
"""Get the html to embed into the crowdsource"""
try:
# first try to get embed code from oEmbed
return mark_safe(PyEmbed().embed(self.url, max_height=400))
except PyEmbedConsumerError:
# if this is a private document cloud document, it will not have
# an oEmbed, create the embed manually
doc_match = DOCUMENT_URL_RE.match(self.url)
if doc_match:
return mark_safe(
DOCCLOUD_EMBED.format(doc_id=doc_match.group('doc_id')))
else:
# fall back to a simple iframe
return format_html(
'<iframe src="{}" width="100%" height="400px"></iframe>',
self.url,
)
def test_should_embed_xml():
with patch('pyembed.core.consumer.get_oembed_response') as mock_get:
discoverer = Mock()
discoverer.get_oembed_urls.return_value = [
'http://example.com/oembed?format=xml'
]
response = Mock()
mock_get.return_value = response
renderer = Mock()
renderer.render.return_value = '<h1>hi</h1>'
result = PyEmbed(discoverer, renderer).embed('http://example.com/')
assert result == '<h1>hi</h1>'
discoverer.get_oembed_urls.assert_called_with('http://example.com/')
mock_get.assert_called_with('http://example.com/oembed?format=xml')
renderer.render.assert_called_with('http://example.com/', response)
def test_should_embed_with_max_width_and_height():
with patch('pyembed.core.consumer.get_oembed_response') as mock_get:
discoverer = Mock()
discoverer.get_oembed_urls.return_value = [
'http://example.com/oembed?format=json'
]
response = Mock()
mock_get.return_value = response
renderer = Mock()
renderer.render.return_value = '<h1>hi</h1>'
result = PyEmbed(discoverer, renderer).embed('http://example.com/',
100, 200)
assert result == '<h1>hi</h1>'
discoverer.get_oembed_urls.assert_called_with('http://example.com/')
mock_get.assert_called_with('http://example.com/oembed?format=json',
max_width=100,
max_height=200)
renderer.render.assert_called_with('http://example.com/', response)
def resource_change(self):
error_message = None
if self.resource_url:
# Discover embed code using oEmbed
oembed = PyEmbed()
embed_code = None
thumbnail_url = None
try:
oembed_urls = oembed.discoverer.get_oembed_urls(
self.resource_url)
if oembed_urls:
oembed_response = oembed_consumer.get_first_oembed_response(
oembed_urls, max_width=800, max_height=600)
embed_code = oembed.renderer.render(
self.resource_url, oembed_response)
thumbnail_url = getattr(oembed_response, 'thumbnail_url',
'')
if embed_code and thumbnail_url:
self.embed_code = embed_code
self.thumbnail_url = thumbnail_url
else:
error_message = u'''Nie udało się pobrać kodu osadzenia (embed code) tego zasobu!'''
except RequestException:
error_message = u'''Wystąpił problem podczas łączenia z adresem "%s".
Sprawdź czy jest on poprawny.''' % self.resource_url
if error_message:
self.resource_url = ''
self.embed_code = ''
self.thumbnail_url = ''
return {
'warning': {
'title': u'Problem z pobraniem informacji o zasobie!',
'message': error_message,
}
}
def test_should_embed_when_text_xml_returned():
embedding = PyEmbed().embed('https://soundcloud.com/coltonprovias/rush')
assert 'soundcloud.com' in embedding
def test_should_embed_when_no_discovery():
embedding = PyEmbed().embed(
'http://www.rdio.com/artist/Mike_Oldfield/album/Amarok/')
assert 'rd.io' in embedding
def test_should_embed_with_custom_renderer():
embedding = PyEmbed(renderer=DummyRenderer()).embed(
'http://www.youtube.com/watch?v=qrO4YZeyl0I')
assert embedding == \
'Lady Gaga - Bad Romance by LadyGagaVEVO from ' + \
'http://www.youtube.com/watch?v=qrO4YZeyl0I'
def test_should_embed_with_maximum_height():
embedding = PyEmbed().embed('http://www.youtube.com/watch?v=9bZkp7q19f0',
max_height=200)
assert 'height="200"' in embedding
def test_should_get_another_correct_embedding():
embedding = PyEmbed().embed(
'http://www.flickr.com/photos/hansjuul/7899334594')
assert '.jpg' in embedding
def fetch_oembed_preview(content, urls):
"""Fetch first oembed content for a list of urls."""
for url in urls:
# See first if recently cached already
if OEmbedCache.objects.filter(url=url,
modified__gte=now() -
datetime.timedelta(days=7)).exists():
oembed = OEmbedCache.objects.get(url=url)
Content.objects.filter(id=content.id).update(oembed=oembed)
return oembed
# Fetch oembed
options = {}
if url.startswith("https://twitter.com/"):
if len(url.split('/')) < 5:
# Skip, not enough sections to be a tweet
# We don't want to oembed profile streams
continue
# This probably has little effect since we fetch these on the backend...
# But, DNT is always good to communicate if possible :)
options = {"dnt": "true", "omit_script": "true"}
try:
oembed = PyEmbed(discoverer=OEmbedDiscoverer()).embed(
url, **options)
except (PyEmbedError, PyEmbedDiscoveryError, PyEmbedConsumerError,
ValueError):
continue
if not oembed:
continue
# Keep width and height for some sites embedded videos
video_sites = [
"youtube.com", "youtu.be", "vimeo.com", "kickstarter.com"
]
if not any(site in oembed for site in video_sites):
# Keep height if width = 100%
if not re.search(r'\s+width="100%"', oembed):
oembed = re.sub(r'\s+height="[0-9]+"', " ", oembed)
# Ensure width is 100% not fixed
oembed = re.sub(r'\s+width="[0-9]+"', ' width="100%"', oembed)
# Wordpress sites use a random token and message events to identify the right iframe in order
# to set the rendered height. For this to work within a masonry grid, the parent script
# must already be loaded. Since in that context the parent script which updates the iframe
# tag with the token is not called, we add it here.
if "wp-embedded-content" in oembed:
oembed = re.sub(r'<script.*</script>', '', oembed, flags=re.S)
oembed = re.sub(r'<blockquote.*</blockquote>', '', oembed)
ltr = string.ascii_lowercase + string.digits
secret = ''.join(random.choice(ltr) for i in range(10))
oembed = re.sub(
r'(<iframe.*src=".*?)"',
'\g<1>#?secret=' + secret + '" data-secret="' + secret + '"',
oembed)
try:
with transaction.atomic():
oembed = OEmbedCache.objects.create(url=url, oembed=oembed)
except IntegrityError:
# Some other process got ahead of us
oembed = OEmbedCache.objects.get(url=url)
Content.objects.filter(id=content.id).update(oembed=oembed)
return oembed
Content.objects.filter(id=content.id).update(oembed=oembed)
return oembed
return False
def anydownload(request):
# MusicPath = os.path.expanduser("~") + "/Desktop/"
if request.method == 'GET':
ydl_opts = {
# 'logger': MyLogger(),
'quiet': True,
'skip_download': True,
'match_filter': youtube_dl.utils.match_filter_func("!is_live"),
}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
meta = ydl.extract_info(URL_LIST, download=False)
new_url1 = PyEmbed().embed(URL_LIST)
new_url = (re.search("(?P<url>https?://[^\s]+)",
new_url1).group("url"))
context = {
'title': (f"{meta['title']}"),
'uploader': (f"{meta['uploader']}"),
# 'duration': (f"{duration1}"),
'url':
new_url,
'videos_1080':
f"youtube-dl -f 137 --skip-download {URL_LIST} ",
'videos_720':
(f"youtube-dl -f 22 --skip-download {URL_LIST}"),
'videos_normal':
(f"youtube-dl -f best --skip-download {URL_LIST} "),
}
return render(request, 'anyvideo/anydownload.html', context)
if request.method == 'POST':
amd = request.POST['type']
if amd == '1080p':
videos_1080 = os.system(
f"youtube-dl -f 137 --skip-download {URL_LIST} ")
if videos_1080 is None:
(f"youtube-dl -f best --output ~/Downloads/%(title)s.%(ext)s -ik --format mp4 --yes-playlist {URL_LIST} "
)
# (f"youtube-dl -f best --output os.path.expanduser("~")/%(title)s.%(ext)s -ik --format mp4 --yes-playlist {URL_LIST} ")
else:
(f"youtube-dl -f 137 --output ~/Downloads/%(title)s.%(ext)s -ik --format mp4 --yes-playlist {URL_LIST} "
)
return render(request, "anyvideo/anydownload.html")
elif amd == '720p':
videos_720 = (f"youtube-dl -f 22 --skip-download {URL_LIST} ")
if videos_720 is None:
os.system(
f"youtube-dl -f best --output ~/Downloads/%(title)s.%(ext)s -ik --format mp4 --yes-playlist {URL_LIST} "
)
else:
(f"youtube-dl -f 22 --output ~/Downloads/%(title)s.%(ext)s -ik --format mp4 --yes-playlist {URL_LIST} "
)
return render(request, "anyvideo/anydownload.html")
else:
remove_file()
ydl_opts = {
'format': 'best',
'outtmpl': '~/Downloads/video.mp4',
'noplaylist': True,
}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
ydl.download([URL_LIST])
messages.success(request,
'Video has been successfully downloaded !')
return serve_file_helper()
# return redirect('anyhome')
return render(request, "anyvideo/anydownload.html")
def test_should_get_correct_embedding():
embedding = PyEmbed().embed(
'https://twitter.com/BarackObama/status/266031293945503744')
assert 'Four more years.' in embedding
from pyembed.core import PyEmbed
html = PyEmbed().embed('http://www.youtube.com/watch?feature=player_detailpage&v=3oFm4MYbb9o')
print html
Run the above code with Python.
Then paste the output (which will be enclosed in an iframe tag) in an HTML page. This will result in the multimedia content referred to by the URL, being embedded in the page. E.g. If the content is a YouTube video, that video will be embedded in the HTML page.
class OEmbedXBlock(XBlock):
pyembed = PyEmbed()
display_name = String(
display_name="Display Name",
help=
"This name appears in the horizontal navigation at the top of the page.",
scope=Scope.settings,
default="OEmbed",
)
document_url = String(
display_name="Document URL",
help=
"Navigate to the document in your browser and ensure that it is public. Copy its URL and paste it into this field.",
scope=Scope.settings,
default=pyembed.embed(
DEFAULT_DOCUMENT_URL, width=960,
height=569) #EMBED_CODE_TEMPLATE.format(DEFAULT_DOCUMENT_URL)
)
reference_name = String(display_name="Reference Name",
help="The link text.",
scope=Scope.settings,
default="")
output_model = String(
display_name="Ouput Model",
help=
"Currently selected option for how to insert the document into the unit.",
scope=Scope.settings,
default="1")
model1 = String(display_name="Model1 preselection",
help="Previous selection.",
scope=Scope.settings,
default="")
model2 = String(display_name="Model2 preselection",
help="Previous selection.",
scope=Scope.settings,
default="")
output_code = String(display_name="Output Iframe Embed Code",
help="Copy the embed code into this field.",
scope=Scope.settings,
default=pyembed.embed(DEFAULT_DOCUMENT_URL,
width=960,
height=569))
message = String(
display_name="Document display status message",
help="Message to help students in case of errors.",
scope=Scope.settings,
default=
"Note: Some services may require you to be signed into them to access documents stored there."
)
message_display_state = String(
display_name="Whether to display the status message",
help=
"Determines whether to display the message to help students in case of errors.",
scope=Scope.settings,
default="block")
def resource_string(self, path):
"""Handy helper for getting resources from our kit."""
data = pkg_resources.resource_string(__name__, path)
return data.decode("utf8")
def student_view(self, context=None):
"""
The primary view of the OEmbedXBlock, shown to students
when viewing courses.
"""
html = self.resource_string("static/html/oembed.html")
frag = Fragment(html.format(self=self))
frag.add_css(self.resource_string("static/css/oembed.css"))
frag.add_javascript(self.resource_string("static/js/src/oembed.js"))
frag.initialize_js('OEmbedXBlock')
return frag
def studio_view(self, context=None):
"""
he primary view of the OEmbedXBlock, shown to teachers
when viewing courses.
"""
html = self.resource_string("static/html/oembed_edit.html")
frag = Fragment(html.format(self=self))
frag.add_css(self.resource_string("static/css/oembed_edit.css"))
frag.add_javascript(
self.resource_string("static/js/src/oembed_edit.js"))
frag.initialize_js('OEmbedXBlock')
return frag
@XBlock.json_handler
def studio_submit(self, submissions, suffix=''): # pylint: disable=unused-argument
"""
Change the settings for this XBlock given by the Studio user
"""
if not isinstance(submissions, dict):
LOG.error("submissions object from Studio is not a dict - %r",
submissions)
return {'result': 'error'}
self.document_url = submissions['document_url']
self.output_code = Filter.get_embed_code(url=self.document_url)
self.message = "Note: Some services may require you to be signed into them to access documents stored there."
self.message_display_state = "block"
return {'result': 'success'}
@XBlock.json_handler
def check_url(self, data, suffix=''): # pylint: disable=unused-argument,no-self-use
"""
Checks that the given document url is accessible, and therefore assumed to be valid
"""
try:
test_url = data['url']
except KeyError as ex:
LOG.debug("URL not provided - %s", unicode(ex))
return {
'status_code': 400,
}
try:
url_response = requests.head(test_url)
# Catch wide range of request exceptions
except requests.exceptions.RequestException as ex:
LOG.debug("Unable to connect to %s - %s", test_url, unicode(ex))
return {
'status_code': 400,
}
return {
'status_code': url_response.status_code,
}
@staticmethod
def workbench_scenarios():
"""A canned scenario for display in the workbench."""
return [
("OEmbedXBlock", """<vertical_demo>
<oembed/>
<oembed/>
<oembed/>
</vertical_demo>
"""),
]