def testInit(self): #plain init ep = oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed') self.assertEqual(len(ep.getUrlSchemes()), 0) #init with schemes ep = oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed',\ ['http://*.flickr.com/*',\ 'http://flickr.com/*']) self.assertEqual(len(ep.getUrlSchemes()), 2)
def get_insta_user(short_link, debug=1): """ Get instagram userid from a link posted on twitter """ print "Fetching instagram user. " try: response = urllib2.urlopen( short_link) # Some shortened url, eg: http://t.co/z8P2xNzT8k url_destination_https = response.url url_destination = url_destination_https.replace('https', 'http', 1) # from link get the media_id consumer = oembed.OEmbedConsumer() endpoint = oembed.OEmbedEndpoint( 'https://api.instagram.com/oembed?url=', ['http://instagr.am/p/*']) consumer.addEndpoint(endpoint) media_id_code = re.split('/', url_destination)[-2] url_destination = 'http://instagr.am/p/' + media_id_code response = consumer.embed(url_destination) media_id = response['media_id'] api = InstagramAPI(client_id=iconf.client_id, client_secret=iconf.client_secret) except: if debug: print 'Unable to find picture from link.' return 'null' try: media = api.media(media_id) return [media.user.id, media.user.username] except: if debug: print 'Unable to fetch instagram ID - most likely private user' return 'null'
def embed(url=None): consumer = oembed.OEmbedConsumer() endpoint = oembed.OEmbedEndpoint('http://www.youtube.com/oembed', [ 'http://*.youtube.com/*', 'https://*.youtube.com/*', 'http://*.youtube-nocookie.com/*', 'https://*.youtube-nocookie.com/*', ]) consumer.addEndpoint(endpoint) try: html = consumer.embed(url)['html'] except oembed.OEmbedNoEndpoint: html = None if html: # youtube has a trailing ")" at the moment html = html.rstrip(')') # convert iframe src from http to https, to avoid mixed security blocking when used on an https page # and convert to youtube-nocookie.com html = BeautifulSoup(html) embed_url = html.find('iframe').get('src') if embed_url: embed_url = urlparse(embed_url) if embed_url.scheme == 'http': embed_url = urlunparse(['https'] + list(embed_url[1:])) else: embed_url = embed_url.geturl() embed_url = embed_url.replace('www.youtube.com', 'www.youtube-nocookie.com') html.find('iframe')['src'] = embed_url return jinja2.Markup('<p>%s</p>' % html) return '[[embed url=%s]]' % url
def regex_deviantart(self, event): damatch = re.compile( '((?:(?:https?|ftp|file)://|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$]))', re.I) damatches = damatch.findall(event.message) for x in damatches: try: consumer = oembed.OEmbedConsumer() endpoint = oembed.OEmbedEndpoint( 'http://backend.deviantart.com/oembed', [ 'http://*.deviantart.com/art/*', 'http://fav.me/*', 'http://sta.sh/*', 'http://*.deviantart.com/*#/d*' ]) consumer.addEndpoint(endpoint) response = consumer.embed(x).getData() out = [] if 'title' in response: out.append("Title: {}".format(response[u'title'])) if 'author_name' in response: out.append("Artist: {}".format(response[u'author_name'])) if 'rating' in response: out.append("Rating: {}".format(response[u'rating'])) if 'url' in response: out.append("Direct Url: {}".format(response[u'url'])) self.send_message(event.respond, " | ".join(out).encode( 'utf-8', 'replace')) except oembed.OEmbedNoEndpoint: pass except urllib2.HTTPError: pass
def oembed_consumer(self): consumer = oembed.OEmbedConsumer() endpoint = oembed.OEmbedEndpoint( self.oembed_endpoint, self.oembed_schema, ) consumer.addEndpoint(endpoint) return consumer
def testBrokenEndpoint(self): consumer = oembed.OEmbedConsumer() ep = oembed.OEmbedEndpoint('http://localhost') ep.addUrlScheme('http://localhost/*') consumer.addEndpoint(ep) self.assertRaises(urllib2.URLError, consumer.embed, \ 'http://localhost/test')
def embed(url=None): consumer = oembed.OEmbedConsumer() endpoint = oembed.OEmbedEndpoint( 'http://www.youtube.com/oembed', ['http://*.youtube.com/*', 'https://*.youtube.com/*']) consumer.addEndpoint(endpoint) try: return jinja2.Markup('<div class="grid-20">%s</div>' % consumer.embed(url)['html']) except oembed.OEmbedNoEndpoint as e: return '[[embed url=%s]]' % url
def get_oembed(url): """ Get oembed object, convert it to dict and get products list. :param url: string, page URL :return: list, list of products """ consumer = oembed.OEmbedConsumer() endpoint = oembed.OEmbedEndpoint(url + '.oembed', ['https://suzyshier.com/*']) consumer.addEndpoint(endpoint) response = consumer.embed(url) return response.getData()['products']
def testGettersAndSetters(self): consumer = oembed.OEmbedConsumer() ep1 = oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed') ep2 = oembed.OEmbedEndpoint( 'http://api.pownce.com/2.1/oembed.{format}') ep3 = oembed.OEmbedEndpoint('http://www.vimeo.com/api/oembed.{format}') #adding consumer.addEndpoint(ep1) consumer.addEndpoint(ep2) consumer.addEndpoint(ep3) self.assertEqual(len(consumer.getEndpoints()), 3) #removing one consumer.delEndpoint(ep2) self.assertEqual(len(consumer.getEndpoints()), 2) #clearing all! consumer.clearEndpoints() self.assertEqual(len(consumer.getEndpoints()), 0)
def testEmbed(self): consumer = oembed.OEmbedConsumer() ep1 = oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed') ep1.addUrlScheme('http://*.flickr.com/*') ep2 = oembed.OEmbedEndpoint( 'http://api.pownce.com/2.1/oembed.{format}') ep2.addUrlScheme('http://*.pownce.com/*') consumer.addEndpoint(ep1) consumer.addEndpoint(ep2) #invalid format self.assertRaises(oembed.OEmbedInvalidRequest, consumer.embed, \ 'http://www.flickr.com/photos/wizardbt/2584979382/', \ format='text') #no matching endpoint for the url self.assertRaises(oembed.OEmbedNoEndpoint, consumer.embed, \ 'http://google.com/123456')
def load_all_endpoints(): endpoints = [] endpoint = WordpressEndPoint() endpoints.append(endpoint) providers = REGEX_PROVIDERS for provider in providers: endpoint = oembed.OEmbedEndpoint(provider[u'endpoint'], provider[u'regex']) endpoints.append(endpoint) return endpoints
def load_all_endpoints(embedly_apikey=None): endpoints = [] if embedly_apikey is not None: endpoint = EmbedlyEndPoint(embedly_apikey) endpoints.append(endpoint) endpoint = WordpressEndPoint() endpoints.append(endpoint) providers = REGEX_PROVIDERS for provider in providers: endpoint = oembed.OEmbedEndpoint(provider[u'endpoint'], provider[u'regex']) endpoints.append(endpoint) return endpoints
def testResponses(self): consumer = oembed.OEmbedConsumer() ep = oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed') ep.addUrlScheme('http://*.flickr.com/*') consumer.addEndpoint(ep) #json resp = consumer.embed( 'http://www.flickr.com/photos/wizardbt/2584979382/', format='json') #xml resp = consumer.embed( 'http://www.flickr.com/photos/wizardbt/2584979382/', format='xml') #resource not found self.assertRaises(urllib2.HTTPError, consumer.embed, \ 'http://www.flickr.com/photos/wizardbt/', \ format='json')
def testUrlScheme(self): ep = oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed') #add some schemes ep.addUrlScheme('http://flickr.com/*') ep.addUrlScheme('http://*.flickr.com/*') self.assertEqual(len(ep.getUrlSchemes()), 2) #add duplicated ep.addUrlScheme('http://*.flickr.com/*') self.assertEqual(len(ep.getUrlSchemes()), 2) #remove url ep.addUrlScheme('http://*.flickr.com/') ep.delUrlScheme('http://flickr.com/*') self.assertEqual(len(ep.getUrlSchemes()), 2) #clear all ep.clearUrlSchemes() self.assertEqual(len(ep.getUrlSchemes()), 0)
def embed(url=None): consumer = oembed.OEmbedConsumer() endpoint = oembed.OEmbedEndpoint('http://www.youtube.com/oembed', [str('http://*.youtube.com/*'), str('https://*.youtube.com/*'), str('http://*.youtube-nocookie.com/*'), str('https://*.youtube-nocookie.com/*'), ]) consumer.addEndpoint(endpoint) # workaround for https://github.com/abarmat/python-oembed/pull/9 not being implemented yet with socket_default_timeout(5): try: html = consumer.embed(url)['html'] except oembed.OEmbedNoEndpoint: html = None except six.moves.urllib.error.HTTPError as e: if e.code == 404: return 'Video not available' else: raise if html: # youtube has a trailing ")" at the moment html = html.rstrip(')') # convert iframe src from http to https, to avoid mixed security blocking when used on an https page # and convert to youtube-nocookie.com html = BeautifulSoup(html, 'html.parser') embed_url = html.find('iframe').get('src') if embed_url: embed_url = urlparse(embed_url) if embed_url.scheme == 'http': embed_url = urlunparse(['https'] + list(embed_url[1:])) else: embed_url = embed_url.geturl() embed_url = embed_url.replace('www.youtube.com', 'www.youtube-nocookie.com') html.find('iframe')['src'] = embed_url return jinja2.Markup('<p>%s</p>' % html) return '[[embed url=%s]]' % url
if event.command.lower() in self._prefix('implying'): self.send_message( event.respond, format.color(">Implying " + event.params, format.GREEN)) #dA info fetcher damatch = re.compile( '((?:(?:https?|ftp|file)://|www\.|ftp\.)(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[-A-Z0-9+&@#/%=~_|$?!:,.])*(?:\([-A-Z0-9+&@#/%=~_|$?!:,.]*\)|[A-Z0-9+&@#/%=~_|$]))', re.I) damatches = damatch.findall(event.message) for x in damatches: try: consumer = oembed.OEmbedConsumer() endpoint = oembed.OEmbedEndpoint( 'http://backend.deviantart.com/oembed', [ 'http://*.deviantart.com/art/*', 'http://fav.me/*', 'http://sta.sh/*', 'http://*.deviantart.com/*#/d*' ]) consumer.addEndpoint(endpoint) response = consumer.embed(x).getData() out = [] if response.has_key(u'title'): out.append("Title: {}".format(response[u'title'])) if response.has_key(u'author_name'): out.append("Artist: {}".format(response[u'author_name'])) if response.has_key(u'rating'): out.append("Rating: {}".format(response[u'rating'])) if response.has_key(u'url'): out.append("Direct Url: {}".format(response[u'url'])) self.send_message(event.respond, " | ".join(out).encode('utf-8', 'replace'))
from markdown import Extension from markdown.inlinepatterns import Pattern import oembed DEFAULT_ENDPOINTS = [ # Youtube oembed.OEmbedEndpoint('http://www.youtube.com/oembed', [ 'https?://(*.)?youtube.com/*', 'https?://youtu.be/*', ]), # Flickr oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed/', [ 'https?://*.flickr.com/*', ]), # Vimeo oembed.OEmbedEndpoint('http://vimeo.com/api/oembed.json', [ 'https?://vimeo.com/*', ]), ] OEMBED_LINK_RE = r'\!\[([^\]]*)\]\(((?:https?:)?//[^\)]*)' \ r'(?<!png)(?<!jpg)(?<!jpeg)(?<!gif)\)' class OEmbedLinkPattern(Pattern): def __init__(self, pattern, markdown_instance=None, oembed_consumer=None): Pattern.__init__(self, pattern, markdown_instance) self.consumer = oembed_consumer
with information from the APIs Console <https://code.google.com/apis/console>. """ % os.path.join(os.path.dirname(__file__), CLIENT_SECRETS) FLOW = flow_from_clientsecrets(CLIENT_SECRETS, scope='https://www.googleapis.com/auth/plus.me', message=MISSING_CLIENT_SECRETS_MESSAGE) # Code to get more information about posted contents using oembed protocol ############################################################################### OEMBED_CONSUMER = oembed.OEmbedConsumer() # My customer PicasaWeb/Google+ OEmbed endpoint OEMBED_CONSUMER.addEndpoint( oembed.OEmbedEndpoint('http://picasaweb-oembed.appspot.com/oembed', [ 'http://picasaweb.google.com/*', 'https://picasaweb.google.com/*', 'http://plus.google.com/photos/*', 'https://plus.google.com/photos/*' ])) class Embedly(oembed.OEmbedEndpoint): KEY = False def __init__(self, key): oembed.OEmbedEndpoint.__init__(self, 'http://api.embed.ly/1/oembed', ['http://*', 'https://*']) self.KEY = key def request(self, *args, **kw): url = oembed.OEmbedEndpoint.request(self, *args, **kw) return '%s&key=%s' % (url, self.KEY)
def get_easy_variables(website_rules, url, settings): """Stuff that can be found without parsing the DOM -- its easy""" website_variables = {} website_variables["URL"] = { 'content': url.full_url, 'strength': 'high', 'type': 'attr' } website_variables["DOMAIN"] = { 'content': url.domain, 'strength': 'high', 'type': 'text' } website_variables["DOMAIN_CSS"] = { 'content': url.domain.replace('.', '_'), 'strength': 'high', 'type': 'text' } website_variables["PROTOCOL"] = { 'content': url.protocol, 'strength': 'high', 'type': 'text' } if '_template' in website_rules: website_variables['_template'] = website_rules['_template'] if '_urlregex' in website_rules: pattern = website_rules['_urlregex'] match = re.search(pattern, url.full_url) print(pattern, url.full_url) if match: for index, group in enumerate(match.groups()): if not group: continue key = "URLREGEX(" + str(index) + ")" website_variables[key] = {'content': group, 'type': 'text'} else: print("no match") return None if '_oembed' in website_rules: consumer = oembed.OEmbedConsumer() if '_oembed_schemes' in website_rules: schemes = website_rules['_oembed_schemes'] else: schemes = 'http://*/' + url.domain + '/*' endpoint = oembed.OEmbedEndpoint(website_rules['_oembed'], schemes) consumer.addEndpoint(endpoint) try: response = consumer.embed(url.full_url) except: response = None print('no oembed for URL:', url.full_url) if response: for key, value in response.getData().items(): website_variables[key] = { 'content': str(value), 'type': 'text', 'strength': 'high' } if 'width' in website_variables and 'height' in website_variables: try: height = int(website_variables['height']['content']) width = int(website_variables['width']['content']) except ValueError: # sometimes these are null pass else: prcnt_ratio = (height / width) * 100 prcnt_ratio = str(round(prcnt_ratio, 2)) website_variables['RESPONSIVE_PADDING'] = { 'content': 'padding-bottom: ' + prcnt_ratio + '%;', 'type': 'text', 'strength': 'high' } else: # no response so remove overwritting template if '_template' in website_variables: del website_variables["_template"] del website_rules["_template"] return website_variables
def setUp(self): self.consumer = oembed.OEmbedConsumer() self.endpoint = oembed.OEmbedEndpoint(LIVE, URLS) self.consumer.addEndpoint(self.endpoint) self.maxDiff = None
from typing import List import discord from discord.utils import get import oembed import urlextract extractor = urlextract.URLExtract() consumer = oembed.OEmbedConsumer() endpoint = oembed.OEmbedEndpoint( 'https://www.youtube.com/oembed/', ['*.youtube.com/*', 'youtu.be/*', '*youtube.com/*']) consumer.addEndpoint(endpoint) def get_links_from_string(string: str) -> List[str]: return extractor.find_urls(string) def get_youtube_links_from_list(links: List[str]) -> List[str]: return [link for link in links if 'youtu' in link] def convert_youtube_links_to_format(links: List[str]) -> List[str]: return [ link.replace('youtu.be/', 'youtube.com/watch?v=') for link in links ] def get_channel_link_from_link(link: str) -> str: response = consumer.embed(link) return response.getData()['author_url']
import oembed consumer = oembed.OEmbedConsumer() endpoint = oembed.OEmbedEndpoint('http://www.flickr.com/services/oembed', \ ['http://*.flickr.com/*']) consumer.addEndpoint(endpoint) response = consumer.embed('http://www.flickr.com/photos/wizardbt/2584979382/') print response['url'] import pprint pprint.pprint(response.getData())
oembed_endpoints = [ [ 'https://www.youtube.com/oembed', ['https://*.youtube.com/*', 'https://youtu.be/*'] ], [ 'http://www.slideshare.net/api/oembed/2', ['https?://www.slideshare.net/*'] ], [ 'http://speakerdeck.com/oembed.json', ['https://speakerdeck.com/*'], ], ] for ent in oembed_endpoints: e = oembed.OEmbedEndpoint(*ent) oembed_consumer.addEndpoint(e) @builderscon.app.template_filter('video_id') def video_id(url): mobj = re.search(r'youtube\.com/watch\?v=(.+)', url, flags=re.UNICODE) if mobj: return mobj.group(1) mobj = re.search(r'youtu\.be/(.+)', url, flags=re.UNICODE) if mobj: return mobj.group(1) return '' def video_oembed(url, **opt):
'http://www.youtube.com/oembed': [ 'https?://*.youtube.com/watch*', 'https?://*.youtube.com/v/*', 'https?://youtu.be/*', 'https?://*.youtube.com/user/*', 'https?://*.youtube.com/*#*/*', 'https?://m.youtube.com/index*', 'https?://*.youtube.com/profile*', 'https?://*.youtube.com/view_play_list*', 'https?://*.youtube.com/playlist*' ] } consumer = oembed.OEmbedConsumer() for k, v in PROVIDERS.items(): endpoint = oembed.OEmbedEndpoint(k, v) consumer.addEndpoint(endpoint) def parse_oembed_data(oembed_data, data): """Parse OEmbed response data to inject into lassie's response dict. :param oembed_data: OEmbed response data. :type oembed_data: dict :param data: Refrence to data variable being updated. :type data: dict """ data.update({ 'oembed': oembed_data, })
def setUp(self): self.consumer = oembed.OEmbedConsumer() self.endpoint = oembed.OEmbedEndpoint(LOCAL, URLS) self.consumer.addEndpoint(self.endpoint)
import oembed DEBUG = True LIVE = 'http://picasaweb-oembed.appspot.com/oembed' LOCAL = 'http://localhost:8080/oembed' URLS = ['http://picasaweb.google.com/*', 'https://picasaweb.google.com/*', 'http://plus.google.com/photos/*', 'https://plus.google.com/photos/*'] SERVER = [LIVE, LOCAL][DEBUG] print "Testing against: %s" % SERVER consumer = oembed.OEmbedConsumer() endpoint = oembed.OEmbedEndpoint(SERVER, URLS) consumer.addEndpoint(endpoint) # * http(s)://picasaweb.google.com/{userid}/{albumname} assert # Outputs "rich" element which contains the Picasa album. # * http(s)://picasaweb.google.com/{userid}/{albumname}#{photoid} # Outputs "photo" or "video" element which contains the Photo/Video. # * http(s)://picasaweb.google.com/.*/{userid}/albumid/{albumid}/photoid/{photoid} # Outputs "photo" or "video" element which contains the Photo/Video. # * https://plus.google.com/photos/{userid}/albums/{albumid}/{userid}
}, { "regex": "http://(?:www\\.)?scribd\\.com/.*", "endpoint": "http://www.scribd.com/services/oembed", "name": "Scribd" }, { "regex": "http://(?:www\\.)?blip\\.tv/.*", "endpoint": "http://blip.tv/oembed/", "name": "Blip.tv" }, { "regex": "http://.*", "endpoint": "http://oohembed.com/oohembed/", "name": "Catchall (OohEmbed)" }, ] _consumer = oembed.OEmbedConsumer() for d in config: _consumer.addEndpoint( oembed.OEmbedEndpoint(d['endpoint'], ['regex:%s' % d['regex']])) def oembed_expand(url, **opts): try: return _consumer.embed(url, **opts).getData() except (oembed.OEmbedError, urllib2.HTTPError): return None
def _VIDEO_COVER_IMAGE(conference, eid, type='front', thumb=False): return None VIDEO_COVER_IMAGE = getattr(settings, 'CONFERENCE_VIDEO_COVER_IMAGE', _VIDEO_COVER_IMAGE) _OEMBED_PROVIDERS = (('https://www.youtube.com/oembed', ('https://www.youtube.com/*', 'http://www.youtube.com/*')), ('http://vimeo.com/api/oembed.json', ('http://vimeo.com/*', 'https://vimeo.com/*', 'http://vimeo.com/groups/*/videos/*', 'https://vimeo.com/groups/*/videos/*')), ('http://lab.viddler.com/services/oembed/', ('http://*.viddler.com/*', ))) OEMBED_PROVIDERS = getattr(settings, 'CONFERENCE_OEMBED_PROVIDERS', _OEMBED_PROVIDERS) import oembed OEMBED_CONSUMER = oembed.OEmbedConsumer() for p, urls in OEMBED_PROVIDERS: endpoint = oembed.OEmbedEndpoint(p, urls) OEMBED_CONSUMER.addEndpoint(endpoint) OEMBED_URL_FIX = ((r'https?://vimeopro.com.*/(\d+)$', r'https://vimeo.com/\1'), )
def endpoint_factory(info): return oembed.OEmbedEndpoint(info[u'endpoint'], info[u'regex'])
#Verify the tweet has place info before writing (It should, if it got past our place filter) if tweet.place is not None: #Write the JSON format to the text file, and add one to the number of tweets we've collected newTweet = jsonpickle.encode(tweet._json, unpicklable=False) tweetCount += 1 # response = consumer.embed('http://www.flickr.com/photos/wizardbt/2584979382/') newTweet = tweet.id userID = tweet.user.screen_name text = tweet.text # embed = consumer.embed('https://twitter.com/realDonaldTrump/status/987463564305797126') embed = oembed.OEmbedEndpoint( 'ttps://twitter.com/realDonaldTrump/status/987463564305797126', ['http://*.twitter.com/*']) consumer.addEndpoint(embed) @app.route('/') @app.route('/index') def index(): tweet = {'tweet_display': newTweet} return render_template('index.html', title='Index', tweet=tweet, userID=userID, embed=embed, text=text)