def testInit(self):
#plain init
ep = oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed')
self.assertEqual(len(ep.getUrlSchemes()), 0)
#init with schemes
ep = oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed',\
['http://*.flickr.com/*',\
'http://flickr.com/*'])
self.assertEqual(len(ep.getUrlSchemes()), 2)
def get_insta_user(short_link, debug=1):
""" Get instagram userid from a link posted on twitter """
print "Fetching instagram user. "
try:
response = urllib2.urlopen(
short_link) # Some shortened url, eg: http://t.co/z8P2xNzT8k
url_destination_https = response.url
url_destination = url_destination_https.replace('https', 'http', 1)
# from link get the media_id
consumer = oembed.OEmbedConsumer()
endpoint = oembed.OEmbedEndpoint(
'https://api.instagram.com/oembed?url=', ['http://instagr.am/p/*'])
consumer.addEndpoint(endpoint)
media_id_code = re.split('/', url_destination)[-2]
url_destination = 'http://instagr.am/p/' + media_id_code
response = consumer.embed(url_destination)
media_id = response['media_id']
api = InstagramAPI(client_id=iconf.client_id,
client_secret=iconf.client_secret)
except:
if debug:
print 'Unable to find picture from link.'
return 'null'
try:
media = api.media(media_id)
return [media.user.id, media.user.username]
except:
if debug:
print 'Unable to fetch instagram ID - most likely private user'
return 'null'
def embed(url=None):
consumer = oembed.OEmbedConsumer()
endpoint = oembed.OEmbedEndpoint('http://www.youtube.com/oembed', [
'http://*.youtube.com/*',
'https://*.youtube.com/*',
'http://*.youtube-nocookie.com/*',
'https://*.youtube-nocookie.com/*',
])
consumer.addEndpoint(endpoint)
try:
html = consumer.embed(url)['html']
except oembed.OEmbedNoEndpoint:
html = None
if html:
# youtube has a trailing ")" at the moment
html = html.rstrip(')')
# convert iframe src from http to https, to avoid mixed security blocking when used on an https page
# and convert to youtube-nocookie.com
html = BeautifulSoup(html)
embed_url = html.find('iframe').get('src')
if embed_url:
embed_url = urlparse(embed_url)
if embed_url.scheme == 'http':
embed_url = urlunparse(['https'] + list(embed_url[1:]))
else:
embed_url = embed_url.geturl()
embed_url = embed_url.replace('www.youtube.com',
'www.youtube-nocookie.com')
html.find('iframe')['src'] = embed_url
return jinja2.Markup('<p>%s</p>' % html)
return '[[embed url=%s]]' % url
def regex_deviantart(self, event):
damatch = re.compile(
'((?:(?:https?|ftp|file)://|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$]))',
re.I)
damatches = damatch.findall(event.message)
for x in damatches:
try:
consumer = oembed.OEmbedConsumer()
endpoint = oembed.OEmbedEndpoint(
'http://backend.deviantart.com/oembed', [
'http://*.deviantart.com/art/*', 'http://fav.me/*',
'http://sta.sh/*', 'http://*.deviantart.com/*#/d*'
])
consumer.addEndpoint(endpoint)
response = consumer.embed(x).getData()
out = []
if 'title' in response:
out.append("Title: {}".format(response[u'title']))
if 'author_name' in response:
out.append("Artist: {}".format(response[u'author_name']))
if 'rating' in response:
out.append("Rating: {}".format(response[u'rating']))
if 'url' in response:
out.append("Direct Url: {}".format(response[u'url']))
self.send_message(event.respond, " | ".join(out).encode(
'utf-8', 'replace'))
except oembed.OEmbedNoEndpoint:
pass
except urllib2.HTTPError:
pass
def oembed_consumer(self):
consumer = oembed.OEmbedConsumer()
endpoint = oembed.OEmbedEndpoint(
self.oembed_endpoint,
self.oembed_schema,
)
consumer.addEndpoint(endpoint)
return consumer
def testBrokenEndpoint(self):
consumer = oembed.OEmbedConsumer()
ep = oembed.OEmbedEndpoint('http://localhost')
ep.addUrlScheme('http://localhost/*')
consumer.addEndpoint(ep)
self.assertRaises(urllib2.URLError, consumer.embed, \
'http://localhost/test')
def embed(url=None):
consumer = oembed.OEmbedConsumer()
endpoint = oembed.OEmbedEndpoint(
'http://www.youtube.com/oembed',
['http://*.youtube.com/*', 'https://*.youtube.com/*'])
consumer.addEndpoint(endpoint)
try:
return jinja2.Markup('<div class="grid-20">%s</div>' %
consumer.embed(url)['html'])
except oembed.OEmbedNoEndpoint as e:
return '[[embed url=%s]]' % url
def get_oembed(url):
""" Get oembed object, convert it to dict and get products list.
:param url: string, page URL
:return: list, list of products
"""
consumer = oembed.OEmbedConsumer()
endpoint = oembed.OEmbedEndpoint(url + '.oembed',
['https://suzyshier.com/*'])
consumer.addEndpoint(endpoint)
response = consumer.embed(url)
return response.getData()['products']
def testGettersAndSetters(self):
consumer = oembed.OEmbedConsumer()
ep1 = oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed')
ep2 = oembed.OEmbedEndpoint(
'http://api.pownce.com/2.1/oembed.{format}')
ep3 = oembed.OEmbedEndpoint('http://www.vimeo.com/api/oembed.{format}')
#adding
consumer.addEndpoint(ep1)
consumer.addEndpoint(ep2)
consumer.addEndpoint(ep3)
self.assertEqual(len(consumer.getEndpoints()), 3)
#removing one
consumer.delEndpoint(ep2)
self.assertEqual(len(consumer.getEndpoints()), 2)
#clearing all!
consumer.clearEndpoints()
self.assertEqual(len(consumer.getEndpoints()), 0)
def testEmbed(self):
consumer = oembed.OEmbedConsumer()
ep1 = oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed')
ep1.addUrlScheme('http://*.flickr.com/*')
ep2 = oembed.OEmbedEndpoint(
'http://api.pownce.com/2.1/oembed.{format}')
ep2.addUrlScheme('http://*.pownce.com/*')
consumer.addEndpoint(ep1)
consumer.addEndpoint(ep2)
#invalid format
self.assertRaises(oembed.OEmbedInvalidRequest, consumer.embed, \
'http://www.flickr.com/photos/wizardbt/2584979382/', \
format='text')
#no matching endpoint for the url
self.assertRaises(oembed.OEmbedNoEndpoint, consumer.embed, \
'http://google.com/123456')
def load_all_endpoints():
endpoints = []
endpoint = WordpressEndPoint()
endpoints.append(endpoint)
providers = REGEX_PROVIDERS
for provider in providers:
endpoint = oembed.OEmbedEndpoint(provider[u'endpoint'],
provider[u'regex'])
endpoints.append(endpoint)
return endpoints
def load_all_endpoints(embedly_apikey=None):
endpoints = []
if embedly_apikey is not None:
endpoint = EmbedlyEndPoint(embedly_apikey)
endpoints.append(endpoint)
endpoint = WordpressEndPoint()
endpoints.append(endpoint)
providers = REGEX_PROVIDERS
for provider in providers:
endpoint = oembed.OEmbedEndpoint(provider[u'endpoint'],
provider[u'regex'])
endpoints.append(endpoint)
return endpoints
def testResponses(self):
consumer = oembed.OEmbedConsumer()
ep = oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed')
ep.addUrlScheme('http://*.flickr.com/*')
consumer.addEndpoint(ep)
#json
resp = consumer.embed(
'http://www.flickr.com/photos/wizardbt/2584979382/', format='json')
#xml
resp = consumer.embed(
'http://www.flickr.com/photos/wizardbt/2584979382/', format='xml')
#resource not found
self.assertRaises(urllib2.HTTPError, consumer.embed, \
'http://www.flickr.com/photos/wizardbt/', \
format='json')
def testUrlScheme(self):
ep = oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed')
#add some schemes
ep.addUrlScheme('http://flickr.com/*')
ep.addUrlScheme('http://*.flickr.com/*')
self.assertEqual(len(ep.getUrlSchemes()), 2)
#add duplicated
ep.addUrlScheme('http://*.flickr.com/*')
self.assertEqual(len(ep.getUrlSchemes()), 2)
#remove url
ep.addUrlScheme('http://*.flickr.com/')
ep.delUrlScheme('http://flickr.com/*')
self.assertEqual(len(ep.getUrlSchemes()), 2)
#clear all
ep.clearUrlSchemes()
self.assertEqual(len(ep.getUrlSchemes()), 0)
def embed(url=None):
consumer = oembed.OEmbedConsumer()
endpoint = oembed.OEmbedEndpoint('http://www.youtube.com/oembed',
[str('http://*.youtube.com/*'), str('https://*.youtube.com/*'),
str('http://*.youtube-nocookie.com/*'), str('https://*.youtube-nocookie.com/*'),
])
consumer.addEndpoint(endpoint)
# workaround for https://github.com/abarmat/python-oembed/pull/9 not being implemented yet
with socket_default_timeout(5):
try:
html = consumer.embed(url)['html']
except oembed.OEmbedNoEndpoint:
html = None
except six.moves.urllib.error.HTTPError as e:
if e.code == 404:
return 'Video not available'
else:
raise
if html:
# youtube has a trailing ")" at the moment
html = html.rstrip(')')
# convert iframe src from http to https, to avoid mixed security blocking when used on an https page
# and convert to youtube-nocookie.com
html = BeautifulSoup(html, 'html.parser')
embed_url = html.find('iframe').get('src')
if embed_url:
embed_url = urlparse(embed_url)
if embed_url.scheme == 'http':
embed_url = urlunparse(['https'] + list(embed_url[1:]))
else:
embed_url = embed_url.geturl()
embed_url = embed_url.replace('www.youtube.com', 'www.youtube-nocookie.com')
html.find('iframe')['src'] = embed_url
return jinja2.Markup('<p>%s</p>' % html)
return '[[embed url=%s]]' % url
if event.command.lower() in self._prefix('implying'):
self.send_message(
event.respond,
format.color(">Implying " + event.params, format.GREEN))
#dA info fetcher
damatch = re.compile(
'((?:(?:https?|ftp|file)://|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$]))',
re.I)
damatches = damatch.findall(event.message)
for x in damatches:
try:
consumer = oembed.OEmbedConsumer()
endpoint = oembed.OEmbedEndpoint(
'http://backend.deviantart.com/oembed', [
'http://*.deviantart.com/art/*', 'http://fav.me/*',
'http://sta.sh/*', 'http://*.deviantart.com/*#/d*'
])
consumer.addEndpoint(endpoint)
response = consumer.embed(x).getData()
out = []
if response.has_key(u'title'):
out.append("Title: {}".format(response[u'title']))
if response.has_key(u'author_name'):
out.append("Artist: {}".format(response[u'author_name']))
if response.has_key(u'rating'):
out.append("Rating: {}".format(response[u'rating']))
if response.has_key(u'url'):
out.append("Direct Url: {}".format(response[u'url']))
self.send_message(event.respond,
" | ".join(out).encode('utf-8', 'replace'))
from markdown import Extension
from markdown.inlinepatterns import Pattern
import oembed
DEFAULT_ENDPOINTS = [
# Youtube
oembed.OEmbedEndpoint('http://www.youtube.com/oembed', [
'https?://(*.)?youtube.com/*',
'https?://youtu.be/*',
]),
# Flickr
oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed/', [
'https?://*.flickr.com/*',
]),
# Vimeo
oembed.OEmbedEndpoint('http://vimeo.com/api/oembed.json', [
'https?://vimeo.com/*',
]),
]
OEMBED_LINK_RE = r'\!\[([^\]]*)\]\(((?:https?:)?//[^\)]*)' \
r'(?<!png)(?<!jpg)(?<!jpeg)(?<!gif)\)'
class OEmbedLinkPattern(Pattern):
def __init__(self, pattern, markdown_instance=None, oembed_consumer=None):
Pattern.__init__(self, pattern, markdown_instance)
self.consumer = oembed_consumer
with information from the APIs Console <https://code.google.com/apis/console>.
""" % os.path.join(os.path.dirname(__file__), CLIENT_SECRETS)
FLOW = flow_from_clientsecrets(CLIENT_SECRETS,
scope='https://www.googleapis.com/auth/plus.me',
message=MISSING_CLIENT_SECRETS_MESSAGE)
# Code to get more information about posted contents using oembed protocol
###############################################################################
OEMBED_CONSUMER = oembed.OEmbedConsumer()
# My customer PicasaWeb/Google+ OEmbed endpoint
OEMBED_CONSUMER.addEndpoint(
oembed.OEmbedEndpoint('http://picasaweb-oembed.appspot.com/oembed', [
'http://picasaweb.google.com/*', 'https://picasaweb.google.com/*',
'http://plus.google.com/photos/*', 'https://plus.google.com/photos/*'
]))
class Embedly(oembed.OEmbedEndpoint):
KEY = False
def __init__(self, key):
oembed.OEmbedEndpoint.__init__(self, 'http://api.embed.ly/1/oembed',
['http://*', 'https://*'])
self.KEY = key
def request(self, *args, **kw):
url = oembed.OEmbedEndpoint.request(self, *args, **kw)
return '%s&key=%s' % (url, self.KEY)
def get_easy_variables(website_rules, url, settings):
"""Stuff that can be found without parsing the DOM -- its easy"""
website_variables = {}
website_variables["URL"] = {
'content': url.full_url,
'strength': 'high',
'type': 'attr'
}
website_variables["DOMAIN"] = {
'content': url.domain,
'strength': 'high',
'type': 'text'
}
website_variables["DOMAIN_CSS"] = {
'content': url.domain.replace('.', '_'),
'strength': 'high',
'type': 'text'
}
website_variables["PROTOCOL"] = {
'content': url.protocol,
'strength': 'high',
'type': 'text'
}
if '_template' in website_rules:
website_variables['_template'] = website_rules['_template']
if '_urlregex' in website_rules:
pattern = website_rules['_urlregex']
match = re.search(pattern, url.full_url)
print(pattern, url.full_url)
if match:
for index, group in enumerate(match.groups()):
if not group:
continue
key = "URLREGEX(" + str(index) + ")"
website_variables[key] = {'content': group, 'type': 'text'}
else:
print("no match")
return None
if '_oembed' in website_rules:
consumer = oembed.OEmbedConsumer()
if '_oembed_schemes' in website_rules:
schemes = website_rules['_oembed_schemes']
else:
schemes = 'http://*/' + url.domain + '/*'
endpoint = oembed.OEmbedEndpoint(website_rules['_oembed'], schemes)
consumer.addEndpoint(endpoint)
try:
response = consumer.embed(url.full_url)
except:
response = None
print('no oembed for URL:', url.full_url)
if response:
for key, value in response.getData().items():
website_variables[key] = {
'content': str(value),
'type': 'text',
'strength': 'high'
}
if 'width' in website_variables and 'height' in website_variables:
try:
height = int(website_variables['height']['content'])
width = int(website_variables['width']['content'])
except ValueError:
# sometimes these are null
pass
else:
prcnt_ratio = (height / width) * 100
prcnt_ratio = str(round(prcnt_ratio, 2))
website_variables['RESPONSIVE_PADDING'] = {
'content': 'padding-bottom: ' + prcnt_ratio + '%;',
'type': 'text',
'strength': 'high'
}
else:
# no response so remove overwritting template
if '_template' in website_variables:
del website_variables["_template"]
del website_rules["_template"]
return website_variables
def setUp(self):
self.consumer = oembed.OEmbedConsumer()
self.endpoint = oembed.OEmbedEndpoint(LIVE, URLS)
self.consumer.addEndpoint(self.endpoint)
self.maxDiff = None
from typing import List
import discord
from discord.utils import get
import oembed
import urlextract
extractor = urlextract.URLExtract()
consumer = oembed.OEmbedConsumer()
endpoint = oembed.OEmbedEndpoint(
'https://www.youtube.com/oembed/',
['*.youtube.com/*', 'youtu.be/*', '*youtube.com/*'])
consumer.addEndpoint(endpoint)
def get_links_from_string(string: str) -> List[str]:
return extractor.find_urls(string)
def get_youtube_links_from_list(links: List[str]) -> List[str]:
return [link for link in links if 'youtu' in link]
def convert_youtube_links_to_format(links: List[str]) -> List[str]:
return [
link.replace('youtu.be/', 'youtube.com/watch?v=') for link in links
]
def get_channel_link_from_link(link: str) -> str:
response = consumer.embed(link)
return response.getData()['author_url']
import oembed
consumer = oembed.OEmbedConsumer()
endpoint = oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed', \
['http://*.flickr.com/*'])
consumer.addEndpoint(endpoint)
response = consumer.embed('http://www.flickr.com/photos/wizardbt/2584979382/')
print response['url']
import pprint
pprint.pprint(response.getData())
oembed_endpoints = [
[
'https://www.youtube.com/oembed',
['https://*.youtube.com/*', 'https://youtu.be/*']
],
[
'http://www.slideshare.net/api/oembed/2',
['https?://www.slideshare.net/*']
],
[
'http://speakerdeck.com/oembed.json',
['https://speakerdeck.com/*'],
],
]
for ent in oembed_endpoints:
e = oembed.OEmbedEndpoint(*ent)
oembed_consumer.addEndpoint(e)
@builderscon.app.template_filter('video_id')
def video_id(url):
mobj = re.search(r'youtube\.com/watch\?v=(.+)', url, flags=re.UNICODE)
if mobj:
return mobj.group(1)
mobj = re.search(r'youtu\.be/(.+)', url, flags=re.UNICODE)
if mobj:
return mobj.group(1)
return ''
def video_oembed(url, **opt):
'http://www.youtube.com/oembed': [
'https?://*.youtube.com/watch*',
'https?://*.youtube.com/v/*',
'https?://youtu.be/*',
'https?://*.youtube.com/user/*',
'https?://*.youtube.com/*#*/*',
'https?://m.youtube.com/index*',
'https?://*.youtube.com/profile*',
'https?://*.youtube.com/view_play_list*',
'https?://*.youtube.com/playlist*'
]
}
consumer = oembed.OEmbedConsumer()
for k, v in PROVIDERS.items():
endpoint = oembed.OEmbedEndpoint(k, v)
consumer.addEndpoint(endpoint)
def parse_oembed_data(oembed_data, data):
"""Parse OEmbed response data to inject into lassie's response dict.
:param oembed_data: OEmbed response data.
:type oembed_data: dict
:param data: Refrence to data variable being updated.
:type data: dict
"""
data.update({
'oembed': oembed_data,
})
def setUp(self):
self.consumer = oembed.OEmbedConsumer()
self.endpoint = oembed.OEmbedEndpoint(LOCAL, URLS)
self.consumer.addEndpoint(self.endpoint)
import oembed
DEBUG = True
LIVE = 'http://picasaweb-oembed.appspot.com/oembed'
LOCAL = 'http://localhost:8080/oembed'
URLS = ['http://picasaweb.google.com/*',
'https://picasaweb.google.com/*',
'http://plus.google.com/photos/*',
'https://plus.google.com/photos/*']
SERVER = [LIVE, LOCAL][DEBUG]
print "Testing against: %s" % SERVER
consumer = oembed.OEmbedConsumer()
endpoint = oembed.OEmbedEndpoint(SERVER, URLS)
consumer.addEndpoint(endpoint)
# * http(s)://picasaweb.google.com/{userid}/{albumname}
assert
# Outputs "rich" element which contains the Picasa album.
# * http(s)://picasaweb.google.com/{userid}/{albumname}#{photoid}
# Outputs "photo" or "video" element which contains the Photo/Video.
# * http(s)://picasaweb.google.com/.*/{userid}/albumid/{albumid}/photoid/{photoid}
# Outputs "photo" or "video" element which contains the Photo/Video.
# * https://plus.google.com/photos/{userid}/albums/{albumid}/{userid}
},
{
"regex": "http://(?:www\\.)?scribd\\.com/.*",
"endpoint": "http://www.scribd.com/services/oembed",
"name": "Scribd"
},
{
"regex": "http://(?:www\\.)?blip\\.tv/.*",
"endpoint": "http://blip.tv/oembed/",
"name": "Blip.tv"
},
{
"regex": "http://.*",
"endpoint": "http://oohembed.com/oohembed/",
"name": "Catchall (OohEmbed)"
},
]
_consumer = oembed.OEmbedConsumer()
for d in config:
_consumer.addEndpoint(
oembed.OEmbedEndpoint(d['endpoint'], ['regex:%s' % d['regex']]))
def oembed_expand(url, **opts):
try:
return _consumer.embed(url, **opts).getData()
except (oembed.OEmbedError, urllib2.HTTPError):
return None
def _VIDEO_COVER_IMAGE(conference, eid, type='front', thumb=False):
return None
VIDEO_COVER_IMAGE = getattr(settings, 'CONFERENCE_VIDEO_COVER_IMAGE',
_VIDEO_COVER_IMAGE)
_OEMBED_PROVIDERS = (('https://www.youtube.com/oembed',
('https://www.youtube.com/*',
'http://www.youtube.com/*')),
('http://vimeo.com/api/oembed.json',
('http://vimeo.com/*', 'https://vimeo.com/*',
'http://vimeo.com/groups/*/videos/*',
'https://vimeo.com/groups/*/videos/*')),
('http://lab.viddler.com/services/oembed/',
('http://*.viddler.com/*', )))
OEMBED_PROVIDERS = getattr(settings, 'CONFERENCE_OEMBED_PROVIDERS',
_OEMBED_PROVIDERS)
import oembed
OEMBED_CONSUMER = oembed.OEmbedConsumer()
for p, urls in OEMBED_PROVIDERS:
endpoint = oembed.OEmbedEndpoint(p, urls)
OEMBED_CONSUMER.addEndpoint(endpoint)
OEMBED_URL_FIX = ((r'https?://vimeopro.com.*/(\d+)$',
r'https://vimeo.com/\1'), )
def endpoint_factory(info):
return oembed.OEmbedEndpoint(info[u'endpoint'], info[u'regex'])
#Verify the tweet has place info before writing (It should, if it got past our place filter)
if tweet.place is not None:
#Write the JSON format to the text file, and add one to the number of tweets we've collected
newTweet = jsonpickle.encode(tweet._json, unpicklable=False)
tweetCount += 1
# response = consumer.embed('http://www.flickr.com/photos/wizardbt/2584979382/')
newTweet = tweet.id
userID = tweet.user.screen_name
text = tweet.text
# embed = consumer.embed('https://twitter.com/realDonaldTrump/status/987463564305797126')
embed = oembed.OEmbedEndpoint(
'ttps://twitter.com/realDonaldTrump/status/987463564305797126',
['http://*.twitter.com/*'])
consumer.addEndpoint(embed)
@app.route('/')
@app.route('/index')
def index():
tweet = {'tweet_display': newTweet}
return render_template('index.html',
title='Index',
tweet=tweet,
userID=userID,
embed=embed,
text=text)
