def _build_provider_list():
"""
Construct the provider registry, using the app settings.
"""
registry = None
if appsettings.FLUENT_OEMBED_SOURCE == 'basic':
registry = bootstrap_basic()
elif appsettings.FLUENT_OEMBED_SOURCE == 'embedly':
params = {}
if appsettings.MICAWBER_EMBEDLY_KEY:
params['key'] = appsettings.MICAWBER_EMBEDLY_KEY
registry = bootstrap_embedly(**params)
elif appsettings.FLUENT_OEMBED_SOURCE == 'noembed':
registry = bootstrap_noembed(nowrap=1)
elif appsettings.FLUENT_OEMBED_SOURCE == 'list':
# Fill list manually in the settings, e.g. to have a fixed set of supported secure providers.
registry = ProviderRegistry()
for regex, provider in appsettings.FLUENT_OEMBED_PROVIDER_LIST:
registry.register(regex, Provider(provider))
else:
raise ImproperlyConfigured("Invalid value of FLUENT_OEMBED_SOURCE, only 'basic', 'list', 'noembed' or 'embedly' is supported.")
# Add any extra providers defined in the settings
for regex, provider in appsettings.FLUENT_OEMBED_EXTRA_PROVIDERS:
registry.register(regex, Provider(provider))
return registry
def media_create_tags(request, match, instance=None, template_name='media_form.html'):
if request.method == 'POST':
form = MediaForm(request.POST, instance=instance if instance is not None else Media())
if form.is_valid():
providers = micawber.bootstrap_basic()
try:
provider_obj = providers.request(form.cleaned_data['url'])
except micawber.exceptions.ProviderNotFoundException as msg:
return render(request, template_name, {'object_form': form, 'error_message': msg})
except micawber.exceptions.ProviderException as msg:
return render(request, template_name, {'object_form': form, 'error_message': "Embedding disabled by request"})
media = form.save(commit=False)
media.title = provider_obj.get('title') if len(provider_obj.get('title')) < 200 else provider_obj.get('title')[:200]
media.media_type = provider_obj['type']
media.thumbnail_url = provider_obj.get('thumbnail_url', provider_obj.get('url'))
media.submitted_by = request.user if instance is None else instance.submitted_by
media.blog_type = 1 # media
media.save()
media.tags.clear()
for tag in form.cleaned_data['tags']:
media.tags.add(tag.lower())
return redirect(media)
else:
if match:
tag_list = ['"%s"' % tag.replace('-', ' ') for tag in match.split('|')]
form = MediaForm(initial={'tags': ','.join(tag_list)})
else:
form = MediaForm(instance=instance)
return render(request, template_name, {'object_form': form})
def youtube_inf(url, base_url):
request = requests.get(url)
soup = BeautifulSoup(request.content, 'html.parser')
youtube_urls = []
for each in soup.findAll('a', {'class': ' yt-uix-sessionlink spf-link '}):
youtube_urls.append(base_url + each['href'])
meta_data = []
#For getting the Youtube links
youtube_video_infos = []
for every_link in youtube_urls:
try:
dd = {}
providers = micawber.bootstrap_basic()
info = providers.request(every_link)
youtube_video_infos.append(
(info['author_name'], info['thumbnail_url'], info['title'],
info['url']))
dd['author'] = info['author_name']
dd['thumbnail_url'] = info['thumbnail_url']
dd['title'] = info['title']
dd['url'] = info['url']
meta_data.append(dd)
except:
pass
#ret_dict = {'meta': meta_data}
#print(youtube_video_infos)
#with open('metadata_youtube.txt', 'w') as file:
# file.write(str(youtube_video_infos))
return meta_data
def bookmark(ctx, url):
providers = micawber.bootstrap_basic()
bookmark = {
'url': url
}
try:
oembed = providers.request(url)
soup = BeautifulSoup(oembed['html'], 'html.parser')
except:
oembed = {}
content = '<!-- TODO Write something here. -->'
if oembed.get('type') == 'video':
iframe_src = soup.iframe.get('src')
content = ' include c_video.html iframe_url="{0}" '.format(iframe_src)
bookmark = oembed
if oembed.get('type') == 'photo':
oembed['image_src'] = soup.img.get('src')
content = (' include c_image.html src="{image_src}" '
'alt="{title}" ')
post = Post(content, title=ctx.obj['TITLE'], layout='post_bookmark',
category='bookmark', date=ctx.obj['NOW'].isoformat(),
tags=ctx.obj.get('TAGS'), bookmark=bookmark)
write_post('bookmark', ctx.obj['NOW'], ctx.obj['SLUG'], post)
def import_posts(self, zipfile, names):
"""Import all posts."""
out_folder = 'posts'
providers = micawber.bootstrap_basic()
for name in names:
with zipfile.open(name, 'r') as post_f:
data = json.load(post_f)
title = data['title']
slug = utils.slugify(title)
if not slug: # should never happen
LOGGER.error("Error converting post:", title)
return
description = ''
post_date = dateutil.parser.parse(data["published"])
content = data["object"]["content"]
for obj in data["object"].get("attachments", []):
content += '\n<div> {} </div>\n'.format(
micawber.parse_text(obj["url"], providers))
tags = []
self.write_metadata(
os.path.join(self.output_folder, out_folder,
slug + '.meta'), title, slug, post_date,
description, tags)
self.write_content(
os.path.join(self.output_folder, out_folder,
slug + '.html'), content)
def _build_provider_list():
"""
Construct the provider registry, using the app settings.
"""
registry = None
if appsettings.FLUENT_OEMBED_SOURCE == "basic":
registry = bootstrap_basic()
elif appsettings.FLUENT_OEMBED_SOURCE == "embedly":
params = {}
if appsettings.MICAWBER_EMBEDLY_KEY:
params["key"] = appsettings.MICAWBER_EMBEDLY_KEY
registry = bootstrap_embedly(**params)
elif appsettings.FLUENT_OEMBED_SOURCE == "noembed":
registry = bootstrap_noembed(nowrap=1)
elif appsettings.FLUENT_OEMBED_SOURCE == "list":
# Fill list manually in the settings, e.g. to have a fixed set of supported secure providers.
registry = ProviderRegistry()
for regex, provider in appsettings.FLUENT_OEMBED_PROVIDER_LIST:
registry.register(regex, Provider(provider))
else:
raise ImproperlyConfigured(
"Invalid value of FLUENT_OEMBED_SOURCE, only 'basic', 'list', 'noembed' or 'embedly' is supported."
)
# Add any extra providers defined in the settings
for regex, provider in appsettings.FLUENT_OEMBED_EXTRA_PROVIDERS:
registry.register(regex, Provider(provider))
return registry
def import_posts(self, zipfile, names):
"""Import all posts."""
out_folder = 'posts'
providers = micawber.bootstrap_basic()
for name in names:
with zipfile.open(name, 'r') as post_f:
data = json.load(post_f)
title = data['title']
slug = utils.slugify(title)
if not slug: # should never happen
LOGGER.error("Error converting post:", title)
return
description = ''
post_date = dateutil.parser.parse(data["published"])
content = data["object"]["content"]
for obj in data["object"].get("attachments", []):
content += '\n<div> {} </div>\n'.format(micawber.parse_text(obj["url"], providers))
tags = []
self.write_metadata(os.path.join(self.output_folder, out_folder, slug + '.meta'), title, slug, post_date, description, tags)
self.write_content(
os.path.join(self.output_folder, out_folder, slug + '.html'),
content)
def run(self):
if micawber is None:
msg = req_missing(['micawber'], 'use the media directive', optional=True)
return [nodes.raw('', '<div class="text-error">{0}</div>'.format(msg), format='html')]
providers = micawber.bootstrap_basic()
return [nodes.raw('', micawber.parse_text(" ".join(self.arguments), providers), format='html')]
def bootstrap(cache=None):
# micawbers own bootstrap basic plus some more
pr = bootstrap_basic(cache=cache)
# add https support for vimeo
pr.register('https://vimeo.com/\S*',
Provider('https://vimeo.com/api/oembed.json'))
return pr
def _build_provider_list():
"""
Construct the provider registry, using the app settings.
"""
if appsettings.FLUENT_OEMBED_SOURCE == 'basic':
registry = micawber.bootstrap_basic()
# make sure http://youtu.be urls are also valid, see https://github.com/coleifer/micawber/pull/7
registry.register('https?://(\S*.)?youtu(\.be/|be\.com/watch)\S*',
Provider('http://www.youtube.com/oembed'))
return registry
elif appsettings.FLUENT_OEMBED_SOURCE == 'embedly':
params = {}
if appsettings.MICAWBER_EMBEDLY_KEY:
params['key'] = appsettings.MICAWBER_EMBEDLY_KEY
return micawber.bootstrap_embedly(**params)
elif appsettings.FLUENT_OEMBED_SOURCE == 'list':
# Fi
registry = ProviderRegistry()
for regex, provider in appsettings.FLUENT_OEMBED_PROVIDER_LIST:
registry.register(regex, Provider(provider))
return registry
else:
raise ImproperlyConfigured(
"Invalid value of FLUENT_OEMBED_SOURCE, only 'basic', 'list' or 'embedly' is supported."
)
def save(self, *args, **kwargs):
providers = micawber.bootstrap_basic()
try:
self.oembed = providers.request(self.url)
except micawber.ProviderException as e:
log.warn(f"error saving oembed for {self}: {e}")
self.oembed = {}
super().save(*args, **kwargs)
def _gen_media_embed(url, *q, **kw):
if micawber is None:
msg = req_missing(['micawber'],
'use the media directive',
optional=True)
return '<div class="text-error">{0}</div>'.format(msg)
providers = micawber.bootstrap_basic()
return micawber.parse_text(url, providers)
def run(self):
"""Run media directive."""
if micawber is None:
msg = req_missing(['micawber'], 'use the media directive', optional=True)
return [nodes.raw('', '<div class="text-error">{0}</div>'.format(msg), format='html')]
providers = micawber.bootstrap_basic()
return [nodes.raw('', micawber.parse_text(" ".join(self.arguments), providers), format='html')]
def get_title_with_oembed(url):
providers = micawber.bootstrap_basic()
try:
res = providers.request(url)
except micawber.ProviderException as e:
logger.info(e)
return ''
return '{} - {}'.format(res.get('title'), res.get('provider_name'))
def getIframeForVideo(videoUrl):
providers = micawber.bootstrap_basic()
try:
embed = providers.request(videoUrl, maxwidth=580, maxheight=377)
except (InvalidResponseException,
ProviderException,
ProviderNotFoundException):
return None
return embed['html']
def run(self):
if micawber is None:
raise Exception("To use the media directive you need to install "
"the micawber module.")
providers = micawber.bootstrap_basic()
return [
nodes.raw('',
micawber.parse_text(" ".join(self.arguments), providers),
format='html')
]
def html_content(self):
"""Parse article bosy for markup and features."""
hilite = CodeHiliteExtension(linenums=False, css_class='highlight')
extras = ExtraExtension()
markdown_content = markdown(self.body, extensions=[hilite, extras])
oembed_providers = bootstrap_basic(OEmbedCache)
oembed_content = parse_html(
markdown_content,
oembed_providers,
urlize_all=True)
return Markup(oembed_content)
def html_content(text):
hilite = CodeHiliteExtension(linenums=False, css_class='highlight')
extras = ExtraExtension()
markdown_content = markdown(text, extensions=[hilite, extras])
oembed_providers = bootstrap_basic(OEmbedCache())
oembed_content = parse_html(markdown_content,
oembed_providers,
urlize_all=True,
maxwidth=SITE_WIDTH)
return Markup(oembed_content)
def getInformationsForVideo(videoUrl):
providers = micawber.bootstrap_basic()
try:
embed = providers.request(videoUrl)
except (InvalidResponseException,
ProviderException,
ProviderNotFoundException):
return {'title': 'Vidéo introuvable',
'thumb': ''}
return {'title': embed['title'],
'thumb': embed['thumbnail_url']}
def run(self):
if micawber is None:
msg = (
"To use the media directive you need to install "
"the micawber module."
)
utils.logger.WARN(msg)
return [nodes.raw('', '<div class="text-error">{0}</div>'.format(msg), format='html')]
providers = micawber.bootstrap_basic()
return [nodes.raw('', micawber.parse_text(" ".join(self.arguments), providers), format='html')]
def test_oembed_services(self):
'''
A test various oembed services and a simple howto use micawber to discover oembed services (soundcloud, vimeo etc) from urls
Note: it doesn't work with ISSUU
'''
import micawber
mic = micawber.bootstrap_basic()
you = mic.request('https://www.youtube.com/watch?v=GGyLP6R4HTE') # youtube
vim = mic.request('http://vimeo.com/17081933') # vimeo
try:
nothing = mic.request('http://en.wikipedia.org/wiki/The_Boat_Race_2000')
except micawber.exceptions.ProviderNotFoundException, e:
pass
def run(self):
if micawber is None:
msg = "To use the media directive, isntall micawber first."
return [nodes.raw('', '<div class="text-error">{0}</div>'.format(msg), format='html')]
url = " ".join(self.arguments)
providers = micawber.bootstrap_basic()
data = providers.request(url)
html = '<h3>{}</h3>{}'.format(
data['title'],
micawber.parse_text(url, providers)
)
return [nodes.raw('', html, format='html')]
def index():
#import shit
providers = micawber.bootstrap_basic()
add_oembed_filters(app, providers)
#connect to db
db_obj = connect_db('data.db')
#assign cursor and connection to variables
cur = db_obj[0]
con = db_obj[1]
#get list of all threads
cur.execute('SELECT * FROM thread;')
threads = cur.fetchall()
#get original post belonging to each thread
cur.execute('SELECT * FROM post WHERE id = 1 ;')
posts = cur.fetchall()
if request.method == 'POST':
#grab form data
subject = request.form['subjectLine']
comment = request.form['comment']
embed = request.form['embed']
if subject == "" or comment == "" or embed == "":
return "need javascript"
#get highest threadID
cur.execute('SELECT MAX(id) FROM thread;')
#current post is 1 higher
threadID = cur.fetchone()[0] + 1
thread = [threadID, subject]
#adds thread to the database by passing it the item
cur.execute('insert into thread values(?,?)', thread)
#create original post
post = [1, threadID, strftime("%I:%M %B %d, %Y"), comment, embed]
cur.execute('insert into post values(?,?,?,?,?)', post)
#save the changes
con.commit()
con.close()
return redirect(url_for('index'))
con.close()
return render_template('listthreads.html',
threads=reversed(threads),
posts=posts)
def save(self, *args, **kwargs):
providers = micawber.bootstrap_basic()
providers.register(WISTIA_REGEX, micawber.Provider("http://fast.wistia.com/oembed/"))
oembed_response = providers.request(self.media_url,maxwidth=600, maxheight=500, width=600)
self.embed_code=oembed_response["html"]
if oembed_response["type"]=="photo":
self.media_type = IMAGE
#request the image again, add full-size link to the embed code
second_oembed_response = providers.request(self.media_url)
link_text = '<div class="viewfullsize-link"><a href="%s">View full-size image</a></div>' % second_oembed_response["url"]
self.embed_code = "".join([self.embed_code,link_text])
elif oembed_response["type"]=="video":
self.media_type = VIDEO
elif oembed_response["type"]=="rich":
self.media_type = RICH
try:
if "thumbnail_url" in oembed_response:
self.thumbnail_url=oembed_response["thumbnail_url"]
elif oembed_response["type"]=="photo":
self.thumbnail_url = oembed_response["url"]
else:
self.thumbnail_url = os.path.join(settings.STATIC_URL,"images/rich.png")
except:
self.thumbnail_url=""
'''
TODO some hacky Wistia-specific additions here
'''
if re.match(WISTIA_REGEX,self.media_url):
self.thumbnail_url = "%s?image_crop_resized=260x180" % self.thumbnail_url.split("?")[0]
if self.media_type == IMAGE:
self.embed_code = re.sub("image_crop_resized","image_resize",self.embed_code)
elif self.media_type == RICH:
if oembed_response["url"]:
link_text = '<div class="viewfullsize-link"><a href="%s">Download this file</a></div>' % oembed_response["url"]
self.embed_code = "".join([self.embed_code,link_text])
super(MediaArtefact, self).save(*args, **kwargs)
def html_content(self):
app = current_app._get_current_object()
# hilite = CodeHiLiteExtension(linenums=False, css_class='highlight')
extras = ExtraExtension()
#TODO why can I not add CodeHiLiteExtension????
# markdown_content = markdown(self.body, extensions=[hilite, extras])
markdown_content = markdown(self.body, extensions=[extras])
oembed_providers = bootstrap_basic(OEmbedCache())
oembed_content = parse_html(
markdown_content,
oembed_providers,
urlize_all=True,
maxwidth=app.config['SITE_WIDTH'])
return Markup(oembed_content)
def updated_provider_list(list):
if list == "OEMBED":
providers = [
[entry[0], entry[1].endpoint] for entry in micawber.bootstrap_basic()
]
providers.extend(
[[entry[0], entry[1].endpoint] for entry in micawber.bootstrap_oembed()]
)
return providers
if list == "NOEMBED":
return [[entry[0], entry[1].endpoint] for entry in micawber.bootstrap_noembed()]
return []
def index():
#import shit
providers = micawber.bootstrap_basic()
add_oembed_filters(app, providers)
#connect to db
db_obj = connect_db('data.db')
#assign cursor and connection to variables
cur = db_obj[0]
con = db_obj[1]
#get list of all threads
cur.execute('SELECT * FROM thread;')
threads = cur.fetchall()
#get original post belonging to each thread
cur.execute('SELECT * FROM post WHERE id = 1 ;')
posts = cur.fetchall()
if request.method == 'POST':
#grab form data
subject = request.form['subjectLine']
comment = request.form['comment']
embed = request.form['embed']
if subject == "" or comment == "" or embed =="":
return "need javascript"
#get highest threadID
cur.execute('SELECT MAX(id) FROM thread;')
#current post is 1 higher
threadID = cur.fetchone()[0] + 1
thread = [threadID, subject]
#adds thread to the database by passing it the item
cur.execute('insert into thread values(?,?)',thread)
#create original post
post = [1, threadID, strftime("%I:%M %B %d, %Y"), comment, embed]
cur.execute('insert into post values(?,?,?,?,?)',post)
#save the changes
con.commit()
con.close()
return redirect(url_for('index'))
con.close()
return render_template('listthreads.html', threads=reversed(threads), posts=posts)
def test_oembed_services(self):
import micawber
mic = micawber.bootstrap_basic()
you = mic.request('https://www.youtube.com/watch?v=GGyLP6R4HTE') # youtube
vim = mic.request('http://vimeo.com/17081933') # vimeo
print self.assertEqual(you['provider_name'], u'YouTube')
print self.assertEqual(vim['provider_name'], u'Vimeo')
document = Document(corpus=self.corpus, name=u'N-L_FR_20140305_.txt')
document.mimetype = "text/html"
document.save()
t1, created = Tag.objects.get_or_create(type=Tag.OEMBED_PROVIDER_NAME, name=vim['provider_name'])
t1, created = Tag.objects.get_or_create(type=Tag.OEMBED_TITLE, name=vim['title'])
t2, created = Tag.objects.get_or_create(type=Tag.OEMBED_THUMBNAIL_URL, name=vim['thumbnail_url'])
def get_oembed_data(url: str, field_name: str) -> Tuple[Dict[str, Any], str]:
"""Get the oembed data from URL or raise an ValidationError."""
providers = micawber.bootstrap_basic()
try:
oembed_data = providers.request(url,
maxwidth=MEDIA_MAX_WIDTH,
maxheight=MEDIA_MAX_HEIGHT)
return oembed_data, SUPPORTED_MEDIA_TYPES[oembed_data["type"]]
except (micawber.exceptions.ProviderException, KeyError):
raise ValidationError({
field_name:
ValidationError(
"Unsupported media provider or incorrect URL.",
code=ProductErrorCode.UNSUPPORTED_MEDIA_PROVIDER.value,
)
})
def process(self, text):
text = striptags(text)
micawber_providers = micawber.bootstrap_basic()
(_, extracted) = micawber.extract(text, micawber_providers)
alt_title = None
for url, data in extracted.items():
if 'thumbnail_url' in data:
replacer = "\\1<a href=\"%(url)s\" target=\"_blank\">s)</a>\\2" % data
else:
replacer = "\\1<a href=\"%(url)s\" target=\"_blank\">%(title)s</a>\\2" % data
text = re.sub("(\s|^)%s(\s|$)" % re.escape(url), replacer, text)
alt_title = data['title']
md = self.render_md(text)
return md, alt_title
def run(self):
if micawber is None:
msg = ("Error: "
"To use the media directive you need to install "
"the micawber module.")
utils.show_msg(msg)
return [
nodes.raw('',
'<div class="text-error">{0}</div>'.format(msg),
format='html')
]
providers = micawber.bootstrap_basic()
return [
nodes.raw('',
micawber.parse_text(" ".join(self.arguments), providers),
format='html')
]
def get_entry_feature(self):
"""
If there is a video_url we want to include that as a feature, otherwise
we fall through to the thumbnail
"""
if self.video_url:
"""
Ensure that the video is from a site that supports oembed
"""
if any(v in self.video_url for v in settings.ALLOWED_OMEMBED_SITES):
"""
Extracting the oembed data using https://github.com/coleifer/micawber
"""
providers = micawber.bootstrap_basic()
entry_oembed = micawber.parse_text(self.video_url, providers)
""" Ensure https, less site warnings the better """
return string.replace(entry_oembed, 'http://', 'https://')
if self.thumbnail:
return '<img src="%s" alt=""/>' % self.get_image_src()
def viewThread(threadID):
#import shit
providers = micawber.bootstrap_basic()
add_oembed_filters(app, providers)
#database shit
db_obj = connect_db('data.db')
cur = db_obj[0]
con = db_obj[1]
#get information about this thread
cur.execute('SELECT * FROM thread WHERE id =' + threadID + ';')
threadInfo = cur.fetchall()
title = threadInfo[0][1]
#get original post belonging to each thread
cur.execute('SELECT * FROM post WHERE threadID =' + threadID + ';')
posts = cur.fetchall()
return render_template('viewthread.html', title=title, posts=posts)
def viewThread(threadID):
#import shit
providers = micawber.bootstrap_basic()
add_oembed_filters(app, providers)
#database shit
db_obj = connect_db('data.db')
cur = db_obj[0]
con = db_obj[1]
#get information about this thread
cur.execute('SELECT * FROM thread WHERE id ='+threadID+';')
threadInfo = cur.fetchall()
title = threadInfo[0][1]
#get original post belonging to each thread
cur.execute('SELECT * FROM post WHERE threadID ='+threadID+';')
posts = cur.fetchall()
return render_template('viewthread.html',title=title, posts=posts)
def md2html(article):
# Markdown拡張
exts = [
'markdown.extensions.extra', 'markdown.extensions.codehilite',
'markdown.extensions.tables', 'markdown.extensions.toc'
]
content = article[3]
# Markdownに使われる[xxx](http://xxx.xxx)とHTMLタグの中のURLを除外する
pattern = re.compile(
r'''(?<!\]\(|.>|=")https?://[\w/:%#\$&\?~\.=\+\-@]+(?!\)|<|">)''',
re.S | re.M)
urls = list(set(re.findall(pattern, content)))
# urlをリッチコンテンツに変換するparser
providers = micawber.bootstrap_basic()
for url in urls:
# urlをリッチurlに変更(youtubeなどの動画が表示される)
rich_url = micawber.parse_text(url, providers)
content = content.replace(url, rich_url)
content = md.markdown(content, extensions=exts)
return article[:3] + (content, )
def save(self, *args, **kwargs):
self.slug = helper_uuslug(model=Document, instance=self, value=self.title)
if self.pk is None:
super(Document, self).save(*args, **kwargs)
if self.permalink:
import micawber
mic = micawber.bootstrap_basic()
try:
oem = mic.request(self.permalink)
except micawber.exceptions.ProviderNotFoundException, e:
pass
else: # store as oembed tags
t1, created = Tag.objects.get_or_create(type=Tag.OEMBED_PROVIDER_NAME, name=oem['provider_name'])
t1, created = Tag.objects.get_or_create(type=Tag.OEMBED_TITLE, name=oem['title'])
t2, created = Tag.objects.get_or_create(type=Tag.OEMBED_THUMBNAIL_URL, name=oem['thumbnail_url'])
self.tags.add(t1)
self.tags.add(t2)
class Embed(EmbeddedTextBlock):
"""
Embed blocks allow HTML from OEmbed providers (e.g. Youtube, Twitter, Vimeo) to be embedded in a report.
"""
_tag = "Embed"
providers = bootstrap_basic(cache=cache.Cache())
def __init__(self,
url: str,
width: int = 640,
height: int = 480,
id: str = None,
label: str = None):
"""
Args:
url: The URL of the resource to be embedded
width: The width of the embedded object (optional)
height: The height of the embedded object (optional)
id: A unique id for the block to aid querying (optional)
label: A label used when displaying the block (optional)
"""
try:
result = self.providers.request(url,
maxwidth=width,
maxheight=height)
except ProviderException:
raise DPError(f"No embed provider found for URL '{url}'")
super().__init__(
id=id,
label=label,
url=url,
title=result.get("title", "Title"),
provider_name=result.get("provider_name", "Embedding"),
)
# if "html" not in result:
# raise DPError(f"Can't embed result from provider for URL '{url}'")
self.content = result["html"]
def _build_provider_list():
"""
Construct the provider registry, using the app settings.
"""
if appsettings.FLUENT_OEMBED_SOURCE == 'basic':
registry = micawber.bootstrap_basic()
# make sure http://youtu.be urls are also valid, see https://github.com/coleifer/micawber/pull/7
registry.register('https?://(\S*.)?youtu(\.be/|be\.com/watch)\S*', Provider('http://www.youtube.com/oembed'))
return registry
elif appsettings.FLUENT_OEMBED_SOURCE == 'embedly':
params = {}
if appsettings.MICAWBER_EMBEDLY_KEY:
params['key'] = appsettings.MICAWBER_EMBEDLY_KEY
return micawber.bootstrap_embedly(**params)
elif appsettings.FLUENT_OEMBED_SOURCE == 'list':
# Fi
registry = ProviderRegistry()
for regex, provider in appsettings.FLUENT_OEMBED_PROVIDER_LIST:
registry.register(regex, Provider(provider))
return registry
else:
raise ImproperlyConfigured("Invalid value of FLUENT_OEMBED_SOURCE, only 'basic', 'list' or 'embedly' is supported.")
def get_random_image(retries=5):
""" Fetch a random image from flickr.
XXX: Change the search term to an argument
"""
import micawber
import lxml.html
from random import choice
# from urllib2 import urlopen
FLICKR = "http://www.flickr.com"
SEARCH = "/search/?q=ultimate frisbee %s&l=cc&ct=0&mt=photos&adv=1&page=%d"
TERMS = ["throw", "catch", "layout", "bid", "sky", "huck"]
# def get_random_image():
term = choice(TERMS)
page = choice(range(1, 6))
search = SEARCH % (term, page)
print "Searching for images on Flickr..."
tree = lxml.html.parse("%s%s" % (FLICKR, search))
photos = [e for e in tree.findall(".//a[@data-track]") if e.attrib["data-track"] == "photo-click"]
found = False
count = 0
print "Getting thumbnail and url for random image...",
while not found and count < retries:
count += 1
the_photo = choice(photos)
photo_url = FLICKR + the_photo.attrib["href"]
# FIXME: Catch any possible errors
providers = micawber.bootstrap_basic()
data = providers.request(photo_url)
found = "thumbnail_url" in data and "url" in data
print "done"
return data
def shutdown_session(response):
db.session.remove()
return response
app = create_app()
app.config.from_object('config')
mail = Mail(app)
cloudinary = Cloudinary(app)
cache = Cache(app)
wtforms_json.init()
bcrypt = Bcrypt()
oembed_providers = bootstrap_basic()
oembed_providers.register('http://www.ustream.tv/channel/\S+', Provider('http://www.ustream.tv/oembed'))
add_oembed_filters(app, oembed_providers)
from services.user_service import UserService
user_service = UserService()
login_manager = LoginManager()
#login_manager.anonymous_user = Anonymous
login_manager.login_view = "login"
login_manager.login_message = u"Please log in to access this page."
import os from flask import Flask from micawber import bootstrap_basic from peewee import SqliteDatabase APP_ROOT = os.path.dirname(os.path.realpath(__file__)) DATABASE = os.path.join(APP_ROOT, 'notes.db') DEBUG = False app = Flask(__name__) app.config.from_object(__name__) db = SqliteDatabase(app.config['DATABASE']) oembed = bootstrap_basic()
# -*- coding: utf-8 -*-
from django.utils.six.moves.urllib.parse import urlencode
from django.utils.six.moves.urllib.parse import parse_qs, urlparse, urlunparse
from bs4 import BeautifulSoup
import micawber
from micawber.exceptions import ProviderNotFoundException, ProviderException
providers = micawber.bootstrap_basic()
default_cms_plugin_table_mapping = (
# (old_name, new_name),
('cmsplugin_oembedvideoplugin', 'aldryn_video_oembedvideoplugin'),
)
def build_html_iframe(response, url_params=None, iframe_attrs=None):
html = response.get('html', '')
if url_params is None:
url_params = {}
if iframe_attrs is None:
iframe_attrs = {}
if html:
# What follows is a pretty nasty looking "hack"
# oEmbed hss not implemented some parameters
# and so for these we need to add them manually to the iframe.
""" Carafe Database """
from flask import Markup
from markdown import markdown
from markdown.extensions.codehilite import CodeHiliteExtension
from markdown.extensions.extra import ExtraExtension
from micawber import bootstrap_basic, parse_html
from micawber.cache import Cache as OEmbedCache
from bs4 import BeautifulSoup
OEMBED_PROVIDERS = bootstrap_basic(OEmbedCache())
class UserContent:
""" UserContent class """
@property
def html_content(self):
""" parses markdown content """
hil = CodeHiliteExtension(linenums=True, css_class='highlight')
extra = ExtraExtension()
mrkdwn_content = markdown(self.text, extensions=[hil, extra])
oembed_content = parse_html(
mrkdwn_content,
OEMBED_PROVIDERS,
urlize_all=True)
return Markup(oembed_content)
@property
def clean_text(self):
""" cleans text of html """
html = markdown(self.text)
def oembed(self, request):
if not request.user.is_authenticated:
raise PermissionDenied()
"""
check if a document url exists in our system;
if not, load
# do a request to intercept 404 requests. Otherwise: go to iframely; or embed.ly
"""
form = URLForm(request.GET)
if not form.is_valid():
raise ValidationError(form.errors)
# check if there is no document in the archive matching the url.
url = form.cleaned_data['url']
ckey = 'oembed:%s' % url
if not request.query_params.get('nocache',
None) and cache.has_key(ckey):
return Response(cache.get(ckey))
# not done? perform requests etc...
from miller.embedder import custom_noembed, perform_request
# only top part of the content to get metadata.
res = perform_request(url, headers={'Range': 'bytes=0-20000'})
content_type = res.headers.get('content-type', '').split(';')[0]
provider_url = self.headers.get('Host', None)
if not provider_url:
from urlparse import urlparse
o = urlparse(url)
provider_url = o.netloc
# enable smart oembedding. cfr settings.MILLER_OEMBEDS_MAPPER
e = custom_noembed(url=url,
content_type=content_type,
provider_url=provider_url)
if e:
cache.set(ckey, e)
return Response(e)
# is it an image or similar?
# https://www.hdg.de/lemo/img_hd/bestand/objekte/biografien/schaeuble-wolfgang_foto_LEMO-F-5-051_bbst.jpg
import micawber
try:
providers = micawber.bootstrap_basic()
d = providers.request(url)
# check noembed! e.g; for flickr. We should check if the url is in the pattern specified.
# noembed = perform_request('https://noembed.com/embed', params={
# 'url': url
# });
# d = noembed.json()
except Exception as e:
# logger.exception(e)
# Normally: provider not found
d = {
'error': 'unknown',
'errorDetails': '%s' % e,
"url": url,
"encoding": res.encoding,
#"provider_name": ogd.get('site_name'),
#"description": ogd.get('description'),
"type": "link",
"html": ''
}
else:
return Response(d)
# if not 'error' in d:
# d.update({
# "info": {
# 'service': 'noembed'
# }
# })
# return Response(d)
# return Response(d)
def quickmeta(doc, name, attrs={}, key='name'):
attrs = {key: name} if not attrs else attrs
try:
m = doc.html.head.findAll('meta', attrs=attrs)
except AttributeError:
return None
#print m, attrs
return None if not m else u"".join(
filter(None, [t.get('content', None) for t in m]))
# get opengraph data
from bs4 import BeautifulSoup
doc = BeautifulSoup(res.text)
d['description'] = quickmeta(doc=doc, name='og:description')
d['title'] = quickmeta(doc=doc, name='og:title')
d['thumbnail_url'] = quickmeta(doc=doc, name='og:image:secure_url')
d['thumbnail_width'] = quickmeta(doc=doc, name='og:image:width')
d['thumbnail_height'] = quickmeta(doc=doc, name='og:image:height')
d['provider_name'] = quickmeta(doc=doc, name='twitter:site')
if not d['description']:
# get normal desxcription tag.
tag = None
try:
tag = doc.html.head.findAll('meta',
attrs={"name": "description"})
if not tag:
tag = doc.html.head.findAll('meta',
attrs={"name": "Description"})
except AttributeError:
pass
d['description'] = '' if not tag else u"".join(
[t['content'] for t in tag])
if not d['title']:
try:
d['title'] = doc.html.head.title.text
except AttributeError:
pass
if not d['thumbnail_url']:
d['thumbnail_url'] = quickmeta(doc=doc, name='og:image')
if not d['thumbnail_url']:
d['thumbnail_url'] = quickmeta(doc=doc,
name='og:image',
key='property')
if not d['provider_name']:
d['provider_name'] = quickmeta(doc=doc,
name='og:site_name',
key='property')
#import opengraph
# og = opengraph.OpenGraph(html=res.text)
# ogd = dict(og.items())
# print "hehehehehehehehehe"
# print og.items()
# d = {
# "url": url,
# "encoding": res.encoding,
# "provider_name": ogd.get('site_name'),
# "title": ogd.get('title'),
# "description": ogd.get('description'),
# "type": "link",
# "html": ''
# }
# print ogd
# if ogd.get('image'):
# d.update({
# "thumbnail_url" : ogd.get('image'),
# "thumbnail_width" : ogd.get('image:width'),
# "thumbnail_height" : ogd.get('image:height'),
# })
# if not d['title']:
print d
cache.set(ckey, d)
# custom from og
return Response(d)
# Some hosts don't like the requests default UA. Use this one instead.
headers = {'User-Agent': 'Mozilla/5.0'}
oembed_providers_reg = micawber.ProviderRegistry()
providers_fname = join(dirname(__file__), 'providers.json')
with open(providers_fname) as providers_file:
providers = json.load(providers_file)
for provider in providers:
for endpoint in provider['endpoints']:
for scheme in endpoint['schemes']:
oembed_providers_reg.register(
scheme, micawber.Provider(endpoint['url']))
oembed_providers = micawber.bootstrap_basic(registry=oembed_providers_reg)
#inspired: https://github.com/phillipsm/pyfav/blob/master/pyfav/pyfav.py
def get_favicon_url(markup, url):
"""
Given markup, parse it for a favicon URL. The favicon URL is adjusted
so that it can be retrieved independently. We look in the markup returned
from the URL first and if we don't find a favicon there, we look for the
default location, e.g., http://example.com/favicon.ico . We retrurn None if
unable to find the file.
Keyword arguments:
markup -- A string containing the HTML markup.
url -- A string containing the URL where the supplied markup can be found.
We use this URL in cases where the favicon path in the markup is relative.
import flask
import flask_sqlalchemy
import micawber
import app.log
import app.moment
import app.jinja_util
from app import config
application = flask.Flask(config.SITE_NAME, template_folder=config.TEMPLATE_DIR)
application.config.from_object('app.config')
db = flask_sqlalchemy.SQLAlchemy(application)
_micawber_cache = micawber.cache.Cache()
oembed_providers = micawber.bootstrap_basic(_micawber_cache)
# Setup jinja function calls
application.jinja_env.globals['momentjs'] = app.moment.moment
# use tha abstract syntax tree to add all pubilc jijja util functions to jinja
for member_name, member in inspect.getmembers(app.jinja_util):
if member_name.startswith('_'):
continue
if inspect.isfunction(member):
application.jinja_env.globals[member_name] = member
application.logger.setLevel(logging.INFO)
app.log.setup_application_handler(application)
application.logger.info('microblog startup')
#!notes/bin/python
"""app module setup """
import os
from flask import Flask
from micawber import bootstrap_basic
from flask.ext.sqlalchemy import SQLAlchemy
from config import basedir
# APP_ROOT = os.path.dirname(os.path.realpath(__file__))
# DEBUG = False
app = Flask(__name__)
app.config.from_object("config")
db = SQLAlchemy(app)
oembed = bootstrap_basic() # URL to video player service
from app import views, models
def element_post_save(sender, instance, created, **kwargs):
#Disconnect signal here so we don't recurse when we save
signals.post_save.disconnect(element_post_save, sender=Element)
if instance.video_url or instance.video_embed:
instance.type = "video"
if instance.image:
instance.image_url = instance.image.url
instance.file_name = os.path.basename(str(instance.image_url))
if not instance.name:
instance.name = instance.file_name
instance.thumbnail_image = instance.image
if instance.thumbnail_image:
instance.thumbnail_image_url = instance.thumbnail_image.url
instance.save()
if instance.video_url:
try:
import micawber
providers = micawber.bootstrap_basic()
oembed = providers.request(instance.video_url)
if "html" in oembed:
instance.video_embed = oembed["html"]
if not instance.thumbnail_image:
if "thumbnail_url" in oembed:
instance.thumbnail_image_url = oembed["thumbnail_url"]
if "thumbnail_width" in oembed:
instance.thumbnail_image_height = oembed["thumbnail_width"]
if "thumbnail_height" in oembed:
instance.thumbnail_image_height = oembed["thumbnail_height"]
except Exception as e:
print traceback.format_exc()
#Process images and thumbnails
try:
if instance.image and instance.file_name != instance.original_file_name:
instance.original_file_name = instance.file_name
instance.save()
from .helpers import ImageHelper
helper = ImageHelper()
rtn = helper.resize(instance)
if rtn["success"]:
if rtn["thumbnail_image_url"]:
instance.thumbnail_image = ""
instance.thumbnail_image_url = rtn["thumbnail_image_url"]
instance.save()
else:
print rtn["message"]
except Exception as e:
print traceback.format_exc()
#If there is still no thumbnail image then use the default
if instance.type == "video" and not instance.thumbnail_image_url:
instance.thumbnail_image_url = settings.DME_VIDEO_THUMBNAIL_DEFAULT_URL
instance.save()
#Reconnect signal
signals.post_save.connect(element_post_save, sender=Element)
return v
v1 = str(v)
return v1.lower() if isinstance(v, bool) else v1
def conv_attribs(**attribs: t.Any) -> t.Dict[str, str]:
# convert attributes, dropping None values
# TODO - uncomment when 3.8+ only
# self.attributes = {str(k): v1 for (k, v) in kwargs.items() if (v1 := _conv_attrib(v)) is not None}
return {
str(k): conv_attrib(v)
for (k, v) in attribs.items() if conv_attrib(v) is not None
}
providers = bootstrap_basic(cache=cache.Cache())
@dc.dataclass(frozen=True)
class Embedded:
html: HTML
title: str
provider: str
def get_embed_url(url: str, width: int = 960, height: int = 540) -> Embedded:
"""Return html for an embeddable URL"""
try:
r = providers.request(url, maxwidth=width, maxheight=height)
return Embedded(html=r["html"],
title=r.get("title", "Title"),
def element_post_save(sender, instance, created, **kwargs):
#Disconnect signal here so we don't recurse when we save
signals.post_save.disconnect(element_post_save, sender=Element)
if instance.video_url or instance.video_embed:
instance.type = "video"
if instance.image:
instance.image_url = instance.image.url
instance.file_name = os.path.basename(str(instance.image_url))
if not instance.name:
instance.name = instance.file_name
instance.thumbnail_image = instance.image
if instance.thumbnail_image:
instance.thumbnail_image_url = instance.thumbnail_image.url
instance.save()
if instance.video_url:
try:
import micawber
providers = micawber.bootstrap_basic()
oembed = providers.request(instance.video_url)
if "html" in oembed:
instance.video_embed = oembed["html"]
if not instance.thumbnail_image:
if "thumbnail_url" in oembed:
instance.thumbnail_image_url = oembed["thumbnail_url"]
if "thumbnail_width" in oembed:
instance.thumbnail_image_height = oembed[
"thumbnail_width"]
if "thumbnail_height" in oembed:
instance.thumbnail_image_height = oembed[
"thumbnail_height"]
except Exception as e:
print traceback.format_exc()
#Process images and thumbnails
try:
if instance.image and instance.file_name != instance.original_file_name:
instance.original_file_name = instance.file_name
instance.save()
from .helpers import ImageHelper
helper = ImageHelper()
rtn = helper.resize(instance)
if rtn["success"]:
if rtn["thumbnail_image_url"]:
instance.thumbnail_image = ""
instance.thumbnail_image_url = rtn["thumbnail_image_url"]
instance.save()
else:
print rtn["message"]
except Exception as e:
print traceback.format_exc()
#If there is still no thumbnail image then use the default
if instance.type == "video" and not instance.thumbnail_image_url:
instance.thumbnail_image_url = settings.DME_VIDEO_THUMBNAIL_DEFAULT_URL
instance.save()
#Reconnect signal
signals.post_save.connect(element_post_save, sender=Element)
# -*- coding: utf-8 -*-
from flask import Flask
from flask.ext.mongoengine import MongoEngine
from micawber import bootstrap_basic
# Create app
app = Flask(__name__)
# MongoDB Config
app.config['MONGODB_DB'] = 'notes'
app.config['MONGODB_HOST'] = 'localhost'
app.config['MONGODB_PORT'] = 27017
app.config['SECRET_KEY'] = 'Ny\xf23`\xeaY\xd8gk\xc6\xa7\xbc\xa8\x14L-\xed\xa3\xc0\x84#\xc0\x80`'
# Create database connection object
db = MongoEngine(app)
oembed = bootstrap_basic()
if __name__ == '__main__':
app.run()
def _gen_media_embed(url, *q, **kw):
if micawber is None:
msg = req_missing(['micawber'], 'use the media directive', optional=True)
return '<div class="text-error">{0}</div>'.format(msg)
providers = micawber.bootstrap_basic()
return micawber.parse_text(url, providers)
def bootstrap(cache=None):
# micawbers own bootstrap basic plus some more
pr = bootstrap_basic(cache=cache)
# add https support for vimeo
pr.register('https://vimeo.com/\S*', Provider('https://vimeo.com/api/oembed.json'))
return pr
# Create a Flask WSGI app and configure it using values from the module.
app = Flask(__name__)
app.config.from_object(__name__)
# FlaskDB is a wrapper for a peewee database that sets up pre/post-request
# hooks for managing database connections.
flask_db = FlaskDB(app)
# The `database` is the actual peewee database, as opposed to flask_db which is
# the wrapper.
database = flask_db.database
# Configure micawber with the default OEmbed providers (YouTube, Flickr, etc).
# We'll use a simple in-memory cache so that multiple requests for the same
# video don't require multiple network requests.
oembed_providers = bootstrap_basic(OEmbedCache())
class Entry(flask_db.Model):
title = CharField()
slug = CharField(unique=True)
content = TextField()
published = BooleanField(index=True)
timestamp = DateTimeField(default=datetime.datetime.now, index=True)
@property
def html_content(self):
"""
Generate HTML representation of the markdown-formatted blog entry,
and also convert any media URLs into rich media objects such as video
players or images.
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy
import micawber
providers = micawber.bootstrap_basic()
def validate_reel_url(value):
"""Check if value is oembeddable and from a valid provider."""
valid_providers = {
'YouTube',
'Vimeo',
}
# Try to extract oembed info from the url
_, extracted_urls = providers.extract(value)
# If no info is found at all, raise an error
if value not in extracted_urls:
raise ValidationError(
gettext_lazy('%(value)s is not a supported url'),
params={'value': value},
)
# Look for a valid provider
provider_name = extracted_urls[value]['provider_name']
if provider_name in valid_providers:
return
raise ValidationError(
gettext_lazy('%(value)s is not a supported provider'),
#one way hash needed
ADMIN_PASSWORD = '******'
APP_DIR = os.path.dirname(os.path.realpath(__file__))
DATABASE = 'sqliteext:///%s' % os.path.join(APP_DIR, 'blog.db')
DEBUG = False
SECRET_KEY = 'secretKiawah'
SITE_WIDTH = 800
app = Flask(__name__)
app.config.from_object(__name__)
flask_db = FlaskDB(app)
database = flask_db.database
oembed_providers = bootstrap_basic(OEmbedCache())
class Entry(flask_db.Model):
title = CharField()
slug = CharField(unique=True)
content = TextField()
published = BooleanField(index=True)
timestamp = DateTimeField(default=datetime.datetime.now, index=True)
def save(self, *args, **kwargs):
if not self.slug:
self.slug = re.sub('[^\w]+', '-', self.title.lower())
ret = super(Entry, self).save(*args, **kwargs)
#Store search content.
return app
def shutdown_session(response):
db.session.remove()
return response
app = create_app()
app.config.from_object('config')
mail = Mail(app)
cloudinary = Cloudinary(app)
cache = Cache(app)
wtforms_json.init()
bcrypt = Bcrypt()
oembed_providers = bootstrap_basic()
oembed_providers.register('http://www.ustream.tv/channel/\S+',
Provider('http://www.ustream.tv/oembed'))
add_oembed_filters(app, oembed_providers)
from services.user_service import UserService
user_service = UserService()
login_manager = LoginManager()
#login_manager.anonymous_user = Anonymous
login_manager.login_view = "login"
login_manager.login_message = u"Please log in to access this page."
login_manager.refresh_view = "reauth"