def feed(): # Entries are added backwards articles = BlogPost.query.order_by(asc(BlogPost.added)).all() fg = FeedGenerator() fg.title("Mempool | Satoshi Nakamoto Institute") fg.id("https://nakamotoinstitute.org/mempool/feed/") fg.updated(date_to_localized_datetime(articles[0].added)) fg.link(href="https://nakamotoinstitute.org") fg.link(href="https://nakamotoinstitute.org/mempool/feed/", rel="self") fg.language("en") for article in articles: url = url_for("mempool.detail", slug=article.slug, _external=True) page = pages.get(f"mempool/{article.slug}") fe = fg.add_entry() fe.id(url) fe.title(article.title) fe.link(href=url) fe.updated(date_to_localized_datetime(article.added)) fe.published(date_to_localized_datetime(article.date)) fe.author(name=str(article.author[0])) fe.content(page.html) response = make_response(fg.atom_str(encoding="utf-8", pretty=True)) response.headers.set("Content-Type", "application/atom+xml") return response
def build_feed(directory, time_func, base_url, output="atom.xml", n=10, title="", subtitle="", author="", email="", verbose=False): """ Build an Atom feed for all world readable Gemini files in the current directory, and write it to atom.xml. """ # If a title hasn't been provided, try to get one from an index page if not title: title = get_feed_title(directory) # Let user know feed title and URL feed_url = urljoin(base_url, output) if verbose: print('Generating feed "{}", which should be served from {}'.format( title, feed_url)) # Setup feed feed = FeedGenerator() feed.id(base_url) feed.title(title) if subtitle: feed.subtitle(subtitle) author_details = {} if author: author_details["name"] = author if email: author_details["email"] = email if author_details: feed.author(author_details) feed.link(href=feed_url, rel='self') feed.link(href=base_url, rel='alternate') # Add one entry per .gmi file files = find_files(directory, time_func, n) if not files: if verbose: print("No world-readable Gemini content found! :(") return for n, filename in enumerate(files): entry = feed.add_entry() populate_entry_from_file(filename, base_url, entry, time_func) if n == 0: feed.updated(entry.updated()) if verbose: print("Adding {} with title '{}'...".format( os.path.basename(filename), entry.title())) # Write file output = os.path.join(directory, output) feed.atom_file(output, pretty=True) if verbose: print("Wrote Atom feed to {}.".format(output))
def get_feed(atom=False): fg = FeedGenerator() domain = get_domain() items = get_posts({"limit": "10"}, full=True)["results"] fg.id("http://%s/"%domain) fg.title("Blog do MatrUFSC2") fg.description("Feed do blog do MatrUFSC2, onde noticias e novos recursos sao anunciados primeiro!") fg.language('pt-BR') fg.link({"href":"/blog/feed","rel":"self"}) fg.updated(items[0]["posted_at"].replace(tzinfo=pytz.UTC)) for item in items: entry = fg.add_entry() entry.title(item["title"]) tree = html.fromstring(item["summary"]) cleaner = Cleaner(allow_tags=[]) tree = cleaner.clean_html(tree) text = tree.text_content() entry.description(text, True) entry.link({"href":item["link"],"rel":"self"}) entry.content(item["body"]) entry.published(item["posted_at"].replace(tzinfo=pytz.UTC)) entry.updated(item["posted_at"].replace(tzinfo=pytz.UTC)) entry.category({"label": item["category"]["title"], "term": item["category"]["slug"]}) entry.id(item["id"]) if atom: return fg.atom_str(pretty=True) else: return fg.rss_str(pretty=True)
def write_feeds(posts): g = FeedGenerator() g.id('http://peterdowns.com/blog') g.link(href='http://peterdowns.com/blog') g.description('incredibly on-brand') g.title(u'Peter Downs — Posts') first_post = posts[0] updated = first_post['updated'] or first_post['date'] g.updated(_utcstrp(updated, _time_fmt)) for post in posts: e = g.add_entry() post_url = os.path.join(_blog_root, post['html_path']) e.id(post_url) e.link(href=post_url, rel='alternate') e.title(post['title']) e.author(name=post['author'][0]) e.published(_utcstrp(post['date'], _time_fmt)) if post['updated'] is None: e.updated(e.published()) else: e.updated(_utcstrp(post['updated'], _time_fmt)) g.atom_file('{}/atom.xml'.format(_feeds_dir)) g.rss_file('{}/rss.xml'.format(_feeds_dir))
def build_xml_feed(allchapters, verbose=True): if verbose: print print "Generating feeds..." if len(allchapters) == 0: raise CRMangaFeedException("Empty chapter list") crtz = pytz.timezone('America/New_York') fg = FeedGenerator() fg.id('http://utils.senpai.moe/') fg.title('Crunchyroll Manga - Latest Chapters (Unofficial)') fg.author({'name': 'Nosgoroth', 'email': '*****@*****.**'}) fg.link(href='http://utils.senpai.moe/') fg.subtitle( 'Latest manga chapters, updated daily, using undocumented API.') fg.language('en') fg.ttl(15) allchapters = sorted(allchapters, key=itemgetter('updated_t'), reverse=True) first = allchapters[0]["updated_t"].replace(tzinfo=crtz) fg.updated(first) fg.lastBuildDate(first) for chapter in allchapters[0:100]: fe = fg.add_entry() fe.id(chapter["url"]) fe.link({ "href": chapter["url"], "rel": "alternate", "title": "Read online" }) fe.title("%s - %s" % (chapter["series"], chapter["name"])) fe.summary("<p>%s has been added to %s in Crunchyroll Manga.</p>" % (chapter["name"], chapter["series"])) fe.published(chapter["updated_t"].replace(tzinfo=crtz)) chapter_serial = chapter.copy() chapter_serial.pop("updated_t", None) chapter_serial.pop("url", None) chapter_serial.pop("thumb", None) chapter_serial["chapter_id"] = chapter_serial["guid"] chapter_serial.pop("guid", None) content = "<p>%s has been added to %s in Crunchyroll Manga.</p><p>Updated: %s</p><img src=\"%s\" />" % ( chapter["name"], chapter["series"], chapter["updated"], chapter["thumb"]) content += "<!--JSON:[[%s]]-->" % json.dumps(chapter_serial) fe.content(content) fg.rss_file(os.path.join(DESTINATION_FOLDER, 'updates_rss.xml'), pretty=DEBUG) # Write the RSS feed to a file fg.atom_file(os.path.join(DESTINATION_FOLDER, 'updates_atom.xml'), pretty=DEBUG) # Write the ATOM feed to a file
def init_fg(self, repo_info): fg = FeedGenerator() title = 'Recent commits to ' + repo_info['full_name'] fg.title(title) fg.link(href=repo_info['html_url']) fg.updated(repo_info['updated_at']) fg.id(repo_info['html_url']) fg.author(repo_info['author']) return fg
def _get_feed(cls, query: Optional[ClassicAPIQuery] = None) -> FeedGenerator: fg = FeedGenerator() fg.generator("") fg.register_extension("opensearch", OpenSearchExtension) fg.register_extension("arxiv", ArXivExtension, ArXivEntryExtension, rss=False) if query: if query.phrase is not None: query_string = phrase_to_query_string(query.phrase) else: query_string = "" if query.id_list: id_list = ",".join(query.id_list) else: id_list = "" fg.title(f"arXiv Query: {query.to_query_string()}") # From perl documentation of the old site: # search_id is calculated by taking SHA-1 digest of the query # string. Digest is in bytes form and it's 20 bytes long. Then it's # base64 encoded, but perls version returns only 27 characters - # it omits the `=` sign at the end. search_id = base64.b64encode( hashlib.sha1(query.to_query_string().encode( "utf-8")).digest()).decode("utf-8")[:-1] fg.id( cls._fix_url( url_for("classic_api.query").replace( "/query", f"/{search_id}"))) fg.link({ "href": cls._fix_url( url_for( "classic_api.query", search_query=query_string, start=query.page_start, max_results=query.size, id_list=id_list, )), "type": "application/atom+xml", }) else: # TODO: Discuss better defaults fg.title("arXiv Search Results") fg.id("https://arxiv.org/") fg.updated(to_utc(datetime.utcnow())) return fg
class Box: def __init__(self, config=None): if config is None: config = {} self.fg = FeedGenerator() self.fg.title(config['title'] if 'title' in config else 'PyFeed') self.fg.description(config['description'] if 'description' in config else 'github.com/BugenZhao/PyFeed') self.fg.author({'name': 'Bugen Zhao', 'email': '*****@*****.**'}) self.fg.link(href=config['url'] if 'url' in config else 'github.com/BugenZhao/PyFeed') self.fg.language('zh-CN') self.dict = {} self.version = 'v0.3' def add_article(self, article: Article): fe = self.fg.add_entry() fe.id(article.link) fe.link(href=article.link) fe.title(article.title) fe.description(article.description) fe.pubDate(article.date.replace(tzinfo=gettz("Asia/Shanghai"))) self.fg.updated() def update_article(self, article: Article): fe = FeedEntry() fe.id(article.link) fe.link(href=article.link) fe.title(article.title) fe.description(article.description) fe.pubDate(article.date.replace(tzinfo=gettz("Asia/Shanghai"))) self.fg.entry(fe, replace=True) self.fg.updated() def article(self, article: Article): if article.link not in self.dict: logging.info('New article: ' + article.title) self.add_article(article) self.dict[article.link] = article elif article != self.dict[article.link]: logging.info('Update article: ' + article.title) self.update_article(article) self.dict[article.link] = article else: logging.info('Article already existed: ' + article.title) def rss_file(self, filename): self.fg.rss_file(filename, pretty=True) def size(self): return len(self.dict)
def news_feed(): feed = FeedGenerator() feed.id("https://jazzband.co/news/feed") feed.link(href="https://jazzband.co/", rel="alternate") feed.title("Jazzband News Feed") feed.subtitle("We are all part of this.") feed.link(href=full_url(request.url), rel="self") # the list of updates of all news for setting the feed's updated value updates = [] for page in news_pages: if page.path == "index": continue # make the datetime timezone aware if needed published = page.meta.get("published", None) if published and published.tzinfo is None: published = pytz.utc.localize(published) updated = page.meta.get("updated", published) if updated: if updated.tzinfo is None: updated = pytz.utc.localize(updated) updates.append(updated) summary = page.meta.get("summary", None) author = page.meta.get("author", None) author_link = page.meta.get("author_link", None) url = full_url(url_for("content.news", path=page.path)) entry = feed.add_entry() entry.id(url) entry.title(page.meta["title"]) entry.summary(summary) entry.content(content=str(page.html), type="html") if author is not None: author = {"name": author} if author_link is not None: author["uri"] = author_link entry.author(author) entry.link(href=url) entry.updated(updated) entry.published(published) sorted_updates = sorted(updates) feed.updated(sorted_updates and sorted_updates[-1] or datetime.utcnow()) return Response(feed.atom_str(pretty=True), mimetype="application/atom+xml")
def render_feed(text_paths, outpath): # http://rhodesmill.org/brandon/feed # http://rhodesmill.org/brandon/category/python/feed # http://rhodesmill.org/brandon/feed/atom/ t0 = datetime.min.time() def fix(d): dt = datetime.combine(d, t0) return timezone('US/Eastern').localize(dt) posts = [post_info(path) for path in text_paths if date_of(path)] posts = sorted(posts, key=lambda post: post['date']) posts = posts[-1:] most_recent_date = max(post['date'] for post in posts) def full(url): return 'http://rhodesmill.org/' + url.lstrip('/') fg = FeedGenerator() fg.id(full('/')) fg.author({'name': 'Brandon Rhodes'}) fg.language('en') fg.link(href=full('/brandon/'), rel='alternate') if 'python' in outpath: fg.link(href=full('/brandon/category/python/feed/'), rel='self') else: fg.link(href=full('/brandon/feed/'), rel='self') fg.subtitle('Thoughts and ideas from Brandon Rhodes') fg.title("Let's Discuss the Matter Further") fg.updated(fix(most_recent_date)) for post in posts: url = full(post['url_path']) excerpt = truncate_at_more(post['body_html'], url) fe = fg.add_entry() fe.content(excerpt, type='html') fe.guid(url, permalink=True) fe.id(url) fe.link({'href': url}) fe.published(fix(post['date'])) fe.title(post['title']) fe.updated(fix(post['date'])) rss = fg.rss_str(pretty=True) fg.link(href=full('/brandon/feed/atom/'), rel='self', replace=True) atom = fg.atom_str(pretty=True) return rss, atom
def __init__( self, id_: str, links: List[dict], title: str, updated: Optional[str], ): fg = FeedGenerator() fg.id(id_) fg.link(links) fg.title(title) fg.updated(updated) fg.generator("") self._feed_generator = fg
def build(): feed = FeedGenerator() call_dict_as_methods(feed, FEED_SETTINGS) paths = sorted(glob("editions/*.md"), reverse=True)[:10] for i, path in enumerate(paths): entry = feed.add_entry(order="append") data = create_item(path) call_dict_as_methods(entry, data) if i == 0: feed.updated(data["updated"]) return feed
def saveFeed(listings, title, path): url = githubRepoURL + title + ".xml" # Create a feed generator fg = FeedGenerator() # Create the feed's title fg.id(url) fg.title(title) fg.author({'name': 'Ben Snell'}) fg.description("NYC 2BR Apartment Listings in " + title) fg.link(href=url, rel='alternate') fg.language('en') time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + "-05:00" fg.pubDate(time) fg.updated(time) for apt in listings: e = fg.add_entry() e.id(apt[0]) e.title("$" + apt[1] + " // " + apt[4]) e.link(href=apt[0]) text = "" if apt[5] != "": imgs = apt[5].split(" ") for i in range(len(imgs)): text += "<img src=\"" + imgs[i] + "\" /> " if i == 0: text += "<p>" + apt[8] + "</p>" else: text += "<p>" + apt[8] + "</p>" e.content(type="html", content=text) # This doesn't seem to work: e.pubDate(datetime2RSSString(clDate(apt[2]))) e.updated(datetime2RSSString(clDate(apt[2]))) fg.atom_str(pretty=True) fg.atom_file(path)
def saveFeed(listings, title, path): url = githubRepoURL + title + ".xml" # Create a feed generator fg = FeedGenerator() # Create the feed's title fg.id(url) fg.title(title) fg.author({'name': 'Ben Snell'}) fg.description("Art Show Open Call Opportunities") fg.link(href=url, rel='alternate') fg.language('en') time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + "-05:00" fg.pubDate(time) fg.updated(time) for item in listings: e = fg.add_entry() e.id(item["ID"]) # Get a clearer title thisTitle = getShortDate(item["Application Deadline"]) + item["Title"] e.title(thisTitle) # for key, value in item.items(): # print(key, value); # print(item["url"]) # if "url" in item: e.link(href=item["url"]) text = getHtmlFormattedListing(item) e.content(type="html", content=text) # This doesn't seem to work: # e.pubDate( datetime2RSSString(clDate(apt[2])) ) # e.updated( datetime2RSSString(clDate(apt[2])) ) fg.atom_str(pretty=True) fg.atom_file(path)
def feed(posts: Tuple[Post], kind: str) -> FeedGenerator: log.debug('generating %s feed', kind) fg = FeedGenerator() fg.title('beepb00p') fg.author(name='karlicoss', email='*****@*****.**') # TODO better description? fg.description('feed') bb = lambda x: f'https://beepb00p.xyz{x}' fg.id(bb(f'/{kind}.xml')) fg.link(rel='self', href=bb(f'/{kind}.xml')) fg.link(href=bb('')) if len(posts) > 0: dates = (p.date for p in posts) fg.updated( max(tz.localize(d) if d is not None else throw() for d in dates)) # eh, apparnetly in adds items to the feed from bottom to top... for post in reversed(posts): fe = fg.add_entry() # not sure why id() doesn't allow to set permalink=True fe.guid(bb(post.url), permalink=True) fe.link(href=bb(post.url)) fe.title(post.title) # TOOD FIXME meh. d = post.date assert d is not None td = tz.localize(d) fe.published(td) fe.updated(td) # TODO meh, later use proper update date... # # TODO use type=text/html for comparisons? # TODO meh if post.upid == 'infra_diagram': content = "Sorry, this post contains a huge diagram and isn't RSS friendly. It's best viewed on the website" else: content = post.body fe.content(content, type='html') return fg
def display_feed(feed_name): if feed_name not in config['feeds']: abort(404) f = Feed(config['feeds'][feed_name]) f.load() f.parse() fg = FeedGenerator() fg.generator(**_generator) fg.id(request.base_url) fg.link( href=request.base_url, rel='self', ) fg.title(f.properties.get('title', feed_name)) fg.author(name=f.properties.get('author', '')) fg.updated( timezone.localize( _round_date(max([e['updated'] for e in f.entries]), config.get('date_rounding', None)))) for entry in f.entries: fe = fg.add_entry() fe.id(entry['url']) fe.title(entry['title']) fe.link(href=entry['url']) fe.updated( timezone.localize( _round_date(entry['updated'], config.get('date_rounding', None)))) fe.content(entry['content']) atomfeed = fg.atom_str() resp = make_response(atomfeed) resp.headers['content-type'] = 'application/xml' return resp
def generate_feed(posts): fg = FeedGenerator() fg.id('%s/' % SITE_URL) fg.title("jml's notebook") fg.author({'name': 'Jonathan M. Lange', 'email': '*****@*****.**'}) fg.link(href=SITE_URL, rel='alternate') fg.link(href='%s/feed.xml' % (SITE_URL, ), rel='self') fg.language('en') dates = [] for post in reversed(posts): fe = fg.add_entry() fe.id(SITE_URL + post.url) fe.link(href=SITE_URL + post.url) fe.title(post.title or post.name) fe.content(post.body) updated = subprocess.check_output([ "git", "log", "-1", '--date=iso8601', '--format="%ad"', "--", post.original_file, ]).decode('ascii').strip().strip('"') if updated: updated = dateutil.parser.parse(updated) else: updated = datetime.strptime(post.name.replace( '.html', ''), POST_DATE_FORMAT).replace(tzinfo=tz.gettz()) dates.append(updated) fe.updated(updated) fg.updated(max(dates)) fg.atom_file(os.path.join(HTML_ROOT, 'feed.xml'), pretty=True)
def newsfeed(): def makedate(strdate): dt = datetime.datetime.strptime(strdate, '%Y-%m-%d %H:%M') return datetime.datetime( year=dt.year, month=dt.month, day=dt.day, hour=dt.hour, minute=dt.minute, tzinfo=datetime.timezone.utc, ) news = load_news() feed = FeedGenerator() feed.icon(make_external('static/favicon.ico')) feed.id(request.url) feed.language('en-US') feed.link(href=make_external('news')) feed.link(href=request.url, rel='self') feed.title('The State of Taisei') feed.updated(makedate(news[0][0])) for article in news: date = makedate(article[0]) url = make_external("/news/" + article[3]) entry = feed.add_entry() entry.author(name='Taisei team') entry.content(article[2], type='html') entry.id(url) entry.link(href=make_external("/news/" + article[3])) entry.published(date) entry.title(article[1]) entry.updated(date) return Response(feed.atom_str(), mimetype='text/xml')
from datetime import datetime import pytz from algoliasearch.search_client import SearchClient from feedgen.feed import FeedGenerator if __name__ == '__main__': client = SearchClient.create(os.environ['APP_ID'], os.environ['API_KEY']) index = client.init_index('interviews_publishedAt_desc') articles = index.search('')['hits'] fg = FeedGenerator() fg.title('IH Interviews') fg.id('ih-interviews-20201123-205642') pubs = [] for article in articles: pub = datetime.fromtimestamp(article['publishedAt'] / 1000).replace(tzinfo=pytz.timezone('UTC')) pubs.append(pub) fe = fg.add_entry() fe.id(article['interviewId']) fe.published(pub) fe.pubDate(pub) fe.updated(pub) fe.title(article['title']) fe.link( href= f"https://www.indiehackers.com/interview/{article['interviewId']}") fg.updated(max(pubs)) print(fg.atom_str(pretty=True).decode())
async def playlist(request, playlist_id, return_type='video'): log.info(f'Playlist: {playlist_id}') playlist_name = f'{playlist_id}/{return_type}' if playlist_name in playlist_feed and playlist_feed[playlist_name][ 'expire'] > datetime.now(): return raw(playlist_feed[playlist_name]['feed'], content_type='application/rss+xml') calls = 0 payload = {'part': 'snippet', 'id': playlist_id, 'key': KEY} log.debug('Downloaded Playlist Information') response = json.loads(await get( 'https://www.googleapis.com/youtube/v3/playlists', params=payload)) calls += 1 fg = FeedGenerator() fg.load_extension('podcast') fg.generator('PodTube', __version__, 'https://github.com/aquacash5/PodTube') snippet = response['items'][0]['snippet'] icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) fg.title(snippet['title']) fg.id(f'http://{request.headers["host"]}{request.url}') fg.description(snippet['description'] or ' ') fg.author(name=snippet['channelTitle']) fg.image(snippet['thumbnails'][icon]['url']) fg.link(href=f'https://www.youtube.com/playlist?list={playlist_id}') fg.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fg.podcast.itunes_summary(snippet['description']) fg.podcast.itunes_category('Technology', 'Podcasting') fg.updated(f'{str(datetime.utcnow())}Z') response = {'nextPageToken': ''} while 'nextPageToken' in response.keys(): payload = { 'part': 'snippet', 'maxResults': 50, 'playlistId': playlist_id, 'key': KEY, 'pageToken': response['nextPageToken'] } response = json.loads(await get( 'https://www.googleapis.com/youtube/v3/playlistItems', params=payload)) calls += 1 for item in response['items']: snippet = item['snippet'] current_video = snippet['resourceId']['videoId'] if 'Private' in snippet['title']: continue log.debug(f'PlaylistVideo: {current_video} {snippet["title"]}') fe = fg.add_entry() fe.title(snippet['title']) fe.id(current_video) icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) fe.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fe.updated(snippet['publishedAt']) if return_type == 'audio': fe.enclosure( url= f'http://{request.headers["host"]}/audio/{current_video}', type="audio/mpeg") else: fe.enclosure( url= f'http://{request.headers["host"]}/video/{current_video}', type="video/mp4") fe.author(name=snippet['channelTitle']) fe.podcast.itunes_author(snippet['channelTitle']) fe.podcast.itunes_author(snippet['channelTitle']) fe.pubdate(snippet['publishedAt']) fe.link(href='http://www.youtube.com/watch?v=' + current_video, title=snippet['title']) fe.podcast.itunes_summary(snippet['description']) fe.description(snippet['description']) await sleep(0) feed = { 'feed': fg.rss_str(), 'expire': datetime.now() + timedelta(hours=calls) } playlist_feed[playlist_name] = feed return raw(feed['feed'], content_type='application/rss+xml')
def get(self, playlist): playlist = playlist.split('/') if len(playlist) < 2: playlist.append('video') playlist_name = '/'.join(playlist) self.set_header('Content-type', 'application/rss+xml') if playlist_name in playlist_feed and playlist_feed[playlist_name][ 'expire'] > datetime.datetime.now(): self.write(playlist_feed[playlist_name]['feed']) self.finish() return calls = 0 payload = {'part': 'snippet', 'id': playlist[0], 'key': key} request = requests.get( 'https://www.googleapis.com/youtube/v3/playlists', params=payload) calls += 1 response = request.json() if request.status_code == 200: logging.debug('Downloaded Playlist Information') else: logging.error('Error Downloading Playlist: %s', request.reason) self.send_error(reason='Error Downloading Playlist') return fg = FeedGenerator() fg.load_extension('podcast') fg.generator('PodTube (python-feedgen)', __version__, 'https://github.com/aquacash5/PodTube') snippet = response['items'][0]['snippet'] icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) logging.info('Playlist: %s (%s)', playlist[0], snippet['title']) fg.title(snippet['title']) fg.id('http://' + self.request.host + self.request.uri) fg.description(snippet['description'] or ' ') fg.author(name='Podtube', email='*****@*****.**', uri='https://github.com/aquacash5/PodTube') fg.podcast.itunes_author(snippet['channelTitle']) fg.image(snippet['thumbnails'][icon]['url']) fg.link(href=f'http://youtube.com/playlist/?list={playlist}', rel='self') fg.language('en-US') fg.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fg.podcast.itunes_explicit('no') fg.podcast.itunes_owner(name='Podtube', email='*****@*****.**') fg.podcast.itunes_summary(snippet['description']) fg.podcast.itunes_category(cat='Technology') fg.updated(str(datetime.datetime.utcnow()) + 'Z') video = None response = {'nextPageToken': ''} while 'nextPageToken' in response.keys(): payload = { 'part': 'snippet', 'maxResults': 50, 'playlistId': playlist[0], 'key': key, 'pageToken': response['nextPageToken'] } request = requests.get( 'https://www.googleapis.com/youtube/v3/playlistItems', params=payload) calls += 1 response = request.json() if request.status_code == 200: logging.debug('Downloaded Playlist Information') else: logging.error('Error Downloading Playlist: %s', request.reason) self.send_error(reason='Error Downloading Playlist Items') return for item in response['items']: snippet = item['snippet'] current_video = snippet['resourceId']['videoId'] if 'Private' in snippet['title']: continue logging.debug('PlaylistVideo: %s (%s)', current_video, snippet['title']) fe = fg.add_entry() fe.title(snippet['title']) fe.id(current_video) icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) fe.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fe.updated(snippet['publishedAt']) if playlist[1] == 'video': fe.enclosure( url=f'http://{self.request.host}/video/{current_video}', type="video/mp4") elif playlist[1] == 'audio': fe.enclosure( url=f'http://{self.request.host}/audio/{current_video}', type="audio/mpeg") fe.author(name=snippet['channelTitle']) fe.podcast.itunes_author(snippet['channelTitle']) fe.pubDate(snippet['publishedAt']) fe.link(href=f'http://www.youtube.com/watch?v={current_video}', title=snippet['title']) fe.podcast.itunes_summary(snippet['description']) fe.description(snippet['description']) if not video or video['expire'] < fe.pubDate(): video = {'video': fe.id(), 'expire': fe.pubDate()} feed = { 'feed': fg.rss_str(), 'expire': datetime.datetime.now() + datetime.timedelta(hours=calls) } playlist_feed[playlist_name] = feed self.write(feed['feed']) self.finish() video = video['video'] mp3_file = 'audio/{}.mp3'.format(video) if playlist[1] == 'audio' and not os.path.exists( mp3_file) and video not in conversion_queue.keys(): conversion_queue[video] = { 'status': False, 'added': datetime.datetime.now() }
def get(self, channel): channel = channel.split('/') if len(channel) < 2: channel.append('video') channel_name = ['/'.join(channel)] self.set_header('Content-type', 'application/rss+xml') if channel_name[0] in channel_feed and channel_feed[ channel_name[0]]['expire'] > datetime.datetime.now(): self.write(channel_feed[channel_name[0]]['feed']) self.finish() return fg = None video = None calls = 0 response = {'nextPageToken': ''} while 'nextPageToken' in response.keys(): next_page = response['nextPageToken'] payload = { 'part': 'snippet,contentDetails', 'maxResults': 50, 'channelId': channel[0], 'key': key, 'pageToken': next_page } request = requests.get( 'https://www.googleapis.com/youtube/v3/activities', params=payload) calls += 1 if request.status_code != 200: payload = { 'part': 'snippet', 'maxResults': 1, 'forUsername': channel[0], 'key': key } request = requests.get( 'https://www.googleapis.com/youtube/v3/channels', params=payload) response = request.json() channel[0] = response['items'][0]['id'] channel_name.append('/'.join(channel)) payload = { 'part': 'snippet,contentDetails', 'maxResults': 50, 'channelId': channel[0], 'key': key, 'pageToken': next_page } request = requests.get( 'https://www.googleapis.com/youtube/v3/activities', params=payload) calls += 2 response = request.json() if request.status_code == 200: logging.debug('Downloaded Channel Information') else: logging.error('Error Downloading Channel: %s', request.reason) self.send_error(reason='Error Downloading Channel') return if not fg: fg = FeedGenerator() fg.load_extension('podcast') fg.generator('PodTube (python-feedgen)', __version__, 'https://github.com/aquacash5/PodTube') for item in response['items']: if item['snippet']['type'] != 'upload': continue elif 'Private' in item['snippet']['title']: continue else: snippet = item['snippet'] break logging.info('Channel: %s (%s)', channel[0], snippet['channelTitle']) icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) fg.title(snippet['channelTitle']) fg.id('http://' + self.request.host + self.request.uri) fg.description(snippet['description'] or ' ') fg.author(name='Podtube', email='*****@*****.**', uri='https://github.com/aquacash5/PodTube') fg.podcast.itunes_author(snippet['channelTitle']) fg.image(snippet['thumbnails'][icon]['url']) fg.link(href=f'http://youtube.com/channel/{channel}', rel='self') fg.language('en-US') fg.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fg.podcast.itunes_explicit('no') fg.podcast.itunes_owner(name='Podtube', email='*****@*****.**') fg.podcast.itunes_summary(snippet['description']) fg.podcast.itunes_category(cat='Technology') fg.updated(str(datetime.datetime.utcnow()) + 'Z') for item in response['items']: snippet = item['snippet'] if snippet['type'] != 'upload': continue if 'private' in snippet['title'].lower(): continue current_video = item['contentDetails']['upload']['videoId'] logging.debug('ChannelVideo: %s (%s)', current_video, snippet['title']) fe = fg.add_entry() fe.title(snippet['title']) fe.id(current_video) icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) fe.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fe.updated(snippet['publishedAt']) if channel[1] == 'video': fe.enclosure( url=f'http://{self.request.host}/video/{current_video}', type="video/mp4") elif channel[1] == 'audio': fe.enclosure( url=f'http://{self.request.host}/audio/{current_video}', type="audio/mpeg") fe.author(name=snippet['channelTitle']) fe.podcast.itunes_author(snippet['channelTitle']) fe.pubDate(snippet['publishedAt']) fe.link(href=f'http://www.youtube.com/watch?v={current_video}', title=snippet['title']) fe.podcast.itunes_summary(snippet['description']) fe.description(snippet['description']) if not video or video['expire'] < fe.pubDate(): video = {'video': fe.id(), 'expire': fe.pubDate()} feed = { 'feed': fg.rss_str(), 'expire': datetime.datetime.now() + datetime.timedelta(hours=calls) } for chan in channel_name: channel_feed[chan] = feed self.write(feed['feed']) self.finish() video = video['video'] mp3_file = 'audio/{}.mp3'.format(video) if channel[1] == 'audio' and not os.path.exists( mp3_file) and video not in conversion_queue.keys(): conversion_queue[video] = { 'status': False, 'added': datetime.datetime.now() }
def do_build(rebuild=False, full=True, name=""): only = name try: os.makedirs(HTML_POSTS) except FileExistsError: pass for name in tqdm(post_names()): source = os.path.join(POSTS, name + ".md") if not name.startswith(only): continue dest = os.path.join(HTML_POSTS, name + ".html") if not (rebuild or not os.path.exists(dest) or os.path.getmtime(source) > os.path.getmtime(dest)): continue with open(source) as i: source_text = i.read() with open(dest, "w") as o: o.write(post_html(template_cache_key(), name, source_text)) if not full: return for post in glob(os.path.join(HTML_POSTS, "*.html")): source = os.path.join(POSTS, os.path.basename(post).replace(".html", ".md")) if not os.path.exists(source): os.unlink(post) posts = [post_object(name) for name in tqdm(post_names())] posts.sort(key=lambda p: p.name, reverse=True) new_count = 0 new_posts = posts[:new_count] old_posts = [] for post in posts[new_count:]: date = dateutil.parser.parse(post.date) date = f"{date.year}-{date.month:02d}" if not old_posts or date != old_posts[-1][0]: old_posts.append((date, [])) old_posts[-1][-1].append(post) with open(INDEX_PAGE, "w") as o: o.write( TEMPLATE_LOOKUP.get_template("index.html").render( new_posts=new_posts, old_posts=old_posts, title="Thoughts from David R. MacIver", )) fg = FeedGenerator() fg.id("https://notebook.drmaciver.com/") fg.title("DRMacIver's notebook") fg.author({"name": "David R. MacIver", "email": "*****@*****.**"}) fg.link(href="https://notebook.drmaciver.com", rel="alternate") fg.link(href="https://notebook.drmaciver.com/feed.xml", rel="self") fg.language("en") dates = [] for post in sorted(posts, key=lambda p: p.date, reverse=True)[:10]: fe = fg.add_entry() fe.id("https://notebook.drmaciver.com" + post.url) fe.link(href="https://notebook.drmaciver.com" + post.url) fe.title(post.title or post.name) fe.content(post.body, type="html") updated = (subprocess.check_output([ "git", "log", "-1", "--date=iso8601", '--format="%ad"', "--", post.original_file, ]).decode("ascii").strip().strip('"')) if updated: updated = dateutil.parser.parse(updated) else: updated = datetime.strptime(post.name.replace( ".html", ""), POST_DATE_FORMAT).replace(tzinfo=tz.gettz()) dates.append(updated) fe.updated(updated) fg.updated(max(dates)) fg.atom_file(os.path.join(HTML_ROOT, "feed.xml"), pretty=True)
def render_atom(datasette, request, sql, columns, rows, database, table, query_name, view_name, data): from datasette.views.base import DatasetteError if not REQUIRED_COLUMNS.issubset(columns): raise DatasetteError( "SQL query must return columns {}".format( ", ".join(REQUIRED_COLUMNS)), status=400, ) fg = FeedGenerator() fg.generator( generator="Datasette", version=__version__, uri="https://github.com/simonw/datasette", ) fg.id(request.url) fg.link(href=request.url, rel="self") fg.updated(max(row["atom_updated"] for row in rows)) title = request.args.get("_feed_title", sql) if table: title += "/" + table if data.get("human_description_en"): title += ": " + data["human_description_en"] # If this is a canned query the configured title for that over-rides all others if query_name: try: title = datasette.metadata( database=database)["queries"][query_name]["title"] except (KeyError, TypeError): pass fg.title(title) clean_function = clean if query_name: # Check allow_unsafe_html_in_canned_queries plugin_config = datasette.plugin_config("datasette-atom") if plugin_config: allow_unsafe_html_in_canned_queries = plugin_config.get( "allow_unsafe_html_in_canned_queries") if allow_unsafe_html_in_canned_queries is True: clean_function = lambda s: s elif isinstance(allow_unsafe_html_in_canned_queries, dict): allowlist = allow_unsafe_html_in_canned_queries.get( database) or [] if query_name in allowlist: clean_function = lambda s: s # And the rows for row in reversed(rows): entry = fg.add_entry() entry.id(str(row["atom_id"])) if "atom_content_html" in columns: entry.content(clean_function(row["atom_content_html"]), type="html") elif "atom_content" in columns: entry.content(row["atom_content"], type="text") entry.updated(row["atom_updated"]) entry.title(str(row["atom_title"])) # atom_link is optional if "atom_link" in columns: entry.link(href=row["atom_link"]) if "atom_author_name" in columns and row["atom_author_name"]: author = { "name": row["atom_author_name"], } for key in ("uri", "email"): colname = "atom_author_{}".format(key) if colname in columns and row[colname]: author[key] = row[colname] entry.author(author) return Response( fg.atom_str(pretty=True), content_type="application/xml; charset=utf-8", status=200, )
def setUp(self): fg = FeedGenerator() self.nsAtom = "http://www.w3.org/2005/Atom" self.nsRss = "http://purl.org/rss/1.0/modules/content/" self.feedId = 'http://lernfunk.de/media/654321' self.title = 'Some Testfeed' self.authorName = 'John Doe' self.authorMail = '*****@*****.**' self.author = {'name': self.authorName, 'email': self.authorMail} self.linkHref = 'http://example.com' self.linkRel = 'alternate' self.logo = 'http://ex.com/logo.jpg' self.subtitle = 'This is a cool feed!' self.link2Href = 'http://larskiesow.de/test.atom' self.link2Rel = 'self' self.language = 'en' self.categoryTerm = 'This category term' self.categoryScheme = 'This category scheme' self.categoryLabel = 'This category label' self.cloudDomain = 'example.com' self.cloudPort = '4711' self.cloudPath = '/ws/example' self.cloudRegisterProcedure = 'registerProcedure' self.cloudProtocol = 'SOAP 1.1' self.icon = "http://example.com/icon.png" self.contributor = {'name': "Contributor Name", 'uri': "Contributor Uri", 'email': 'Contributor email'} self.copyright = "The copyright notice" self.docs = 'http://www.rssboard.org/rss-specification' self.managingEditor = '*****@*****.**' self.rating = '(PICS-1.1 "http://www.classify.org/safesurf/" ' + \ '1 r (SS~~000 1))' self.skipDays = 'Tuesday' self.skipHours = 23 self.textInputTitle = "Text input title" self.textInputDescription = "Text input description" self.textInputName = "Text input name" self.textInputLink = "Text input link" self.ttl = 900 self.webMaster = '*****@*****.**' fg.id(self.feedId) fg.title(self.title) fg.author(self.author) fg.link(href=self.linkHref, rel=self.linkRel) fg.logo(self.logo) fg.subtitle(self.subtitle) fg.link(href=self.link2Href, rel=self.link2Rel) fg.language(self.language) fg.cloud(domain=self.cloudDomain, port=self.cloudPort, path=self.cloudPath, registerProcedure=self.cloudRegisterProcedure, protocol=self.cloudProtocol) fg.icon(self.icon) fg.category(term=self.categoryTerm, scheme=self.categoryScheme, label=self.categoryLabel) fg.contributor(self.contributor) fg.copyright(self.copyright) fg.docs(docs=self.docs) fg.managingEditor(self.managingEditor) fg.rating(self.rating) fg.skipDays(self.skipDays) fg.skipHours(self.skipHours) fg.textInput(title=self.textInputTitle, description=self.textInputDescription, name=self.textInputName, link=self.textInputLink) fg.ttl(self.ttl) fg.webMaster(self.webMaster) fg.updated('2017-02-05 13:26:58+01:00') fg.pubDate('2017-02-05 13:26:58+01:00') fg.generator('python-feedgen', 'x', uri='http://github.com/lkie...') fg.image(url=self.logo, title=self.title, link=self.link2Href, width='123', height='123', description='Example Inage') self.fg = fg
def get_xml(self: Serializer, response: Response) -> Tuple[str, int]: """ Serialize the provided response data into Atom, version 1.0. Parameters ---------- response : Response The search response data to be serialized. Returns ------- data : str The serialized XML results. status The HTTP status code for the operation. """ fg = FeedGenerator() fg.register_extension("arxiv", ArxivExtension, ArxivEntryExtension, rss=False) fg.id("http://arxiv.org/rss/version=atom_1.0") archive = response.hits[0]["primary_classification"]["archive"] fg.title(archive["id"] + " updates on arXiv.org") fg.link(href='http://arxiv.org/rss/version=atom_1.0', rel='self', type='application/atom+xml') fg.updated(datetime.utcnow().replace(tzinfo=utc)) # TODO - Try to remove generator element? This doesn't work - code ignores "None" # fg.generator(None) # TODO - We don't currently set "subtitle", but could do it like this # fg.subtitle( # f"{archive['name']} ({archive['id']}) updates on the arXiv.org e-print archive") # Add each search result "hit" to the feed for hit in response: entry = fg.add_entry() entry.id("http://arxiv.org/abs/" + hit['id']) entry.title(hit['title']) entry.summary(hit['abstract']) entry.published(hit['submitted_date']) entry.updated(hit['updated_date']) entry.link({ "href": url_for("abs_by_id", paper_id=hit['id']), "type": "text/html" }) pdf_link = { "title": "pdf", "rel": "related", "type": "application/pdf" } pdf_link["href"] = url_for("pdf_by_id", paper_id=hit['id']) entry.link(pdf_link) # Add categories categories = [hit['primary_classification'].to_dict()['category']] for dict in hit['secondary_classification']: categories += [dict['category'].to_dict()] for cat in categories: label = cat['name'] + " (" + cat['id'] + ")" category = { "term": cat['id'], "scheme": "http://arxiv.org/schemas/atom", "label": label } entry.category(category) # Add arXiv-specific element "comment" if not hit['comments'].strip(): entry.arxiv.comment(hit['comments']) # Add arXiv-specific element "journal_ref" if not hit['journal_ref'].strip(): entry.arxiv.journal_ref(hit['journal_ref']) # Add arXiv-specific element "primary_category" prim_cat = hit['primary_classification'].to_dict()['category'] label = prim_cat['name'] + " (" + prim_cat['id'] + ")" category = { "term": prim_cat['id'], "scheme": "http://arxiv.org/schemas/atom", "label": label } entry.arxiv.primary_category(category) # Add arXiv-specific element "doi" if hit['doi']: entry.arxiv.doi(hit['doi']) # Add each author for author in hit['authors']: author_list = {"name": author['full_name']} entry.author(author_list) # TODO - How can arxiv-specific affiliation elements be added to authors? data = fg.atom_str(pretty=True) status_code = status.HTTP_200_OK return data, status_code
def main (args): """ main functions """ logger = logging.getLogger(sys._getframe().f_code.co_name) with open(args.config) as f: config = json.load(f) logger.debug(pformat(config, indent=4)) # override any loaded config with items specified on command line for arg, argv in vars(args).iteritems(): if argv is not None: config[arg] = argv if 'maximum' not in config.keys(): config['maximum'] = DEFAULTMAXIMUM logger.debug(pformat(config, indent=4)) # get metadata about the collection context = '{0}/collections/{1}'.format(config['zotero_account'], config['zotero_collection']) url = '/'.join((ZOT_BASE, context, '?format=json')) response = zot_get(url) alt_html = json.loads(response['content'])['links']['alternate']['href'] # get list of items in collection context = '{0}/collections/{1}/items/top'.format(config['zotero_account'], config['zotero_collection']) url = '/'.join((ZOT_BASE, context, '?format=keys&sort=dateModified&direction=desc&limit={0}'.format(config['maximum']))) logger.debug('fetching: {0}'.format(url)) response = zot_get(url) if int(response['length']) > 0: keys = response['content'].split('\n') else: print "boom" if len(keys) > config['maximum']+1: logger.error("gigantic: {0}".format(len(keys))) raise Exception fg = FeedGenerator() feed_id = u'tag:{domain},{date}:{slug}'.format( domain=config['tag_domain'], date=config['tag_date'], slug=config['tag_slug']) fg.id(feed_id) fg.title(config['title']) fg.author( {'name':config['author_name'],'email':config['author_email']} ) fg.link( href=config['self'], rel='self' ) fg.link( href=alt_html, rel='alternate' ) fg.logo('https://www.zotero.org/support/_media/logo/zotero_256x256x32.png') fg.language('en') fg.updated(datetime.now(pytz.utc)) context = '{0}/items'.format(config['zotero_account']) entries = {} for key in [k for k in keys if len(k.strip()) > 0]: logger.info(u'zotero key: "{0}"'.format(key)) url = '/'.join((ZOT_BASE, context, key)) response = zot_get(url) data = json.loads(response['content']) zot_link_html = data['links']['alternate']['href'] zot_link_json = data['links']['self']['href'] data = data['data'] logger.info(u'zotero itemType: "{0}"'.format(data['itemType'])) if data['itemType'] == 'note': logger.warning('ignored note (key="{0}")'.format(key)) elif data['itemType'] == 'attachment': if data['linkMode'] == u'linked_url': fe = entries[data['parentItem']] fe.link(href=data['url'], title=data['title'], rel='alternate') else: raise NotImplemented('Zotero attachment (key="{0}") with unhandled linkMode="{1}"'.format(key, data['linkMode'])) else: fe = fg.add_entry() entries[key] = fe entry_id = u'tag:{domain},{date}:{slug}'.format( domain='zotero.org', date=data['dateAdded'].split('T')[0], slug='/'.join((context, key))) fe.id(entry_id) try: fe.title(data['title']) except KeyError: logger.warning("unexpected lack of title in zotero record") logger.debug(pformat(data, indent=2)) raise try: creators = data['creators'] except KeyError: pass else: authors = [c for c in data['creators'] if c['creatorType'] == u'author'] for a in authors: if 'name' in a.keys(): fe.author({'name':a['name']}) else: fe.author({'name':u'{0} {1}'.format(a['firstName'], a['lastName']), }) try: fe.link(href=data['url'], rel='alternate', title='link to resource') except KeyError: pass fe.link(href=zot_link_html, rel='alternate', title='link to zotero record (html)') #fe.link(href=zot_link_json, rel='alternate', title='link to zotero record (json)') try: fe.description(data['abstractNote'], isSummary=True) except KeyError: pass url = '/'.join((ZOT_BASE, context, key, '?format=bib')) bib = zot_get(url) logger.debug(pformat(bib, indent=4)) bib = bib['content'].split('\n')[2].strip() logger.debug("bib: '{0}'".format(bib)) fe.content(content=bib, type='xhtml') fe.published(data['dateAdded']) fe.updated(data['dateModified']) #fe.updated(datetime.now(pytz.utc)) with open(config['out_path'], 'w') as f: fg.atom_file(f)
class Spider: def __init__(self): self.config = toml.load("config.toml") self.aws_session = boto3.session.Session() self.cloudwatch = self.aws_session.client( service_name="cloudwatch", region_name=self.config["secretsmanager"]["region"], ) self._report_execution() self.metric_retry_count = 0 self.environment = getenv("APP_ENVIRONMENT", "development") with open("version.txt", "r") as f: self.version = f.readline().strip() loglevel = logging.DEBUG if self.environment == "production": loglevel = logging.ERROR logging.basicConfig( stream=stdout, level=loglevel, format="%(asctime)s - %(levelname)s - %(message)s", ) sentry_sdk.init( "https://[email protected]/1509686", environment=self.environment, release=self.version, ) self.feed = FeedGenerator() self.session = HTMLSession() self.s3 = self.aws_session.client(service_name="s3") def _anti_hammer_sleep(self): logging.debug("zzzZZzzzZZZZZzzzzz") sleep(randrange(1, self.config["anti_hammer_sleep"])) def _secrets(self): logging.debug("fetching secrets from AWS") try: client = self.aws_session.client( service_name="secretsmanager", region_name=self.config["secretsmanager"]["region"], ) get_secret_value_response = client.get_secret_value( SecretId=self.config["secretsmanager"]["secret_name"]) except ClientError as e: capture_exception(e) if e.response["Error"]["Code"] == "DecryptionFailureException": # Secrets Manager can't decrypt the protected secret text using the provided KMS key. # Deal with the exception here, and/or rethrow at your discretion. raise e elif e.response["Error"][ "Code"] == "InternalServiceErrorException": # An error occurred on the server side. # Deal with the exception here, and/or rethrow at your discretion. raise e elif e.response["Error"]["Code"] == "InvalidParameterException": # You provided an invalid value for a parameter. # Deal with the exception here, and/or rethrow at your discretion. raise e elif e.response["Error"]["Code"] == "InvalidRequestException": # You provided a parameter value that is not valid for the current state of the resource. # Deal with the exception here, and/or rethrow at your discretion. raise e elif e.response["Error"]["Code"] == "ResourceNotFoundException": # We can't find the resource that you asked for. # Deal with the exception here, and/or rethrow at your discretion. raise e else: # Decrypts secret using the associated KMS CMK. # Depending on whether the secret is a string or binary, one of these fields will be populated. if "SecretString" in get_secret_value_response: return json.loads(get_secret_value_response["SecretString"]) def crawl(self): self._login() self.feed.id(f"{self.version}.vadviktor.xyz") self.feed.updated(datetime.utcnow().isoformat("T") + "Z") self.feed.author({ "name": "Viktor (Ikon) VAD", "email": "*****@*****.**", "uri": "https://www.github.com/vadviktor", }) self.feed.title("Animetorrents.me feed") self.feed.link( href=self.config["s3"]["object_url"].format( bucket=self.config["s3"]["bucket"], region=self.config["s3"]["region"], filekey=self.config["s3"][f"feed_filename_{self.environment}"], ), rel="self", ) for profile_url in self._torrent_profile_links(self._max_pages()): profile_data = self._parse_profile(profile_url) if profile_data is None: continue fe = self.feed.add_entry(order="append") fe.id(profile_url) fe.title(profile_data["title"]) fe.link(href=profile_url, rel="self") cover_image_url = None if profile_data["cover_image_src"] is not None: cover_image_url = self._cover_image_upload_and_get_url( profile_data["cover_image_src"]) thumbnail_small_image_urls = self._thumbnail_small_image_upload_and_get_urls( profile_data["thumbnail_small_image_srcs"]) thumbnail_large_image_urls = self._thumbnail_large_image_upload_and_get_urls( profile_data["thumbnail_large_image_srcs"]) torrent_public_url = self._torrent_upload_and_get_url( profile_data["torrent_download_url"], profile_data["torid"], slugify(profile_data["title"]), profile_data["publish_date"], ) content_lines = [] if cover_image_url is not None: content_lines.append(f'<p><img src="{cover_image_url}" /></p>') content_lines.append(f'<p>[{profile_data["category"]}]</p>') content_lines.append(f'<p>Tags: {profile_data["tags"]}</p>') content_lines.append( f'<p>Published: {profile_data["publish_date"]}</p>') content_lines.append( f'<p><a href="{profile_url}" target="blank">{profile_url}</a></p>' ) content_lines.append( f'<p style="white-space: pre-wrap;">{profile_data["description"]}</p>' ) content_lines.append(f"<p>") for k, v in enumerate(thumbnail_small_image_urls): content_lines.append(f""" <a href="{thumbnail_large_image_urls[k]}" target="blank"> <img src="{v}" width="200" height="100" /> </a>""") content_lines.append(f"</p>") content_lines.append( f'<p><a href="{torrent_public_url}" target="blank">Download</a></p>' ) content_lines.append(f'<p>{profile_data["torrent_details"]}</p>') content_lines.append(f'<p>{profile_data["file_list"]}</p>') if profile_data["media_info"] is not None: content_lines.append(f'<p>{profile_data["media_info"]}</p>') fe.content(self._valid_xhtml_content(content_lines), type="xhtml") self._upload_feed() self._report_retry_count() @staticmethod def _valid_xhtml_content(content_lines: List) -> str: broken_html = "".join(content_lines) # parse as HTML parser = etree.HTMLParser() tree = etree.parse(StringIO(broken_html), parser) # output as valid XML result = etree.tostring(tree.getroot(), pretty_print=True, method="xml") return result.decode("utf-8") def _upload_feed(self): logging.debug("construct and upload feed") atomfeed = self.feed.atom_str() bucket = self.config["s3"]["bucket"] key = self.config["s3"][f"feed_filename_{self.environment}"].format( version=getenv("FEED_VERSION", "v0")) self.s3.upload_fileobj(BytesIO(atomfeed), bucket, key) resp = self.s3.put_object_acl(ACL="public-read", Bucket=bucket, Key=key) if resp is None: capture_message(f"Failed to set object ACL for {bucket}/{key}") def _parse_profile(self, profile_url): logging.debug(f"processing profile {profile_url}") resp = self._get(profile_url) if ("Error 404: Torrent not found" in resp.text or "Torrent not found" in resp.text): msg = f"No torrent found for {profile_url}" logging.info(msg) capture_message(msg) return None profile_data = {} profile_data["category"] = resp.html.find("h1.headline img", first=True).attrs["alt"] if any(category in profile_data["category"] for category in self.config["exclude_categories"]): return None profile_data["torid"] = re.match(r".*=(\d+)$", profile_url)[1] try: profile_data["torrent_download_url"] = next( l for l in resp.html.links if "download.php?torid=" in l) except StopIteration: msg = f"did not find download link for {profile_url}" capture_message(msg) raise RuntimeError(msg) profile_data["hashid"] = re.match( r".*torid=([a-z0-9]+)$", profile_data["torrent_download_url"]).group(1) profile_data["title"] = resp.html.find("h1.headline", first=True).text profile_data["description"] = resp.html.find("#torDescription", first=True).text profile_data["tags"] = resp.html.find("#tagLinks", first=True).text profile_data["publish_date"] = self._parse_publish_date( resp.html.find("div.ribbon span.blogDate", first=True).text) profile_data["torrent_details"] = resp.html.find( "#tabs-1 table.dataTable", first=True).html profile_data["media_info"] = self._download_media_info( profile_data["torid"]) profile_data["file_list"] = self._download_file_list( profile_data["hashid"]) try: profile_data["cover_image_src"] = next( link.attrs["src"] for link in resp.html.find("div.contentArea img") if "imghost/covers/" in link.attrs["src"]) except StopIteration: logging.debug(f"did not find cover image for {profile_url}") profile_data["cover_image_src"] = None profile_data["thumbnail_small_image_srcs"] = [ i.attrs["src"] for i in resp.html.find("#torScreens img") ] profile_data["thumbnail_large_image_srcs"] = [ i.attrs["href"] for i in resp.html.find("#torScreens a") ] return profile_data @retry((TimeOutException, ConnectionError), tries=5, delay=3, backoff=2) def _get(self, url, **kwargs) -> Response: self._anti_hammer_sleep() resp = self.session.get(url, **kwargs) if resp.status_code in TIMEOUT_STATUS_CODES: self.metric_retry_count += 1 raise TimeOutException return resp @staticmethod def _parse_publish_date(text) -> datetime: return datetime.fromtimestamp( mktime(strptime(text, "%d %b, %Y [%I:%M %p]"))) def _torrent_profile_links(self, max_pages) -> List: links = [] for page in range(1, self.config["torrent_pages_to_scan"] + 1): resp = self._torrent_list_response(page, max_pages) [ links.append(l) for l in resp.html.links if "torrent-details.php?torid=" in l ] return links @retry(TimeOutException, tries=5, delay=3, backoff=2) def _torrent_list_response(self, current_page: int, max_pages: int) -> Response: logging.debug(f"getting torrent list page no. {current_page}") headers = {"X-Requested-With": "XMLHttpRequest"} url = self.config["site"]["torrent_list_url"].format( max=max_pages, current=current_page) resp = self._get(url=url, headers=headers) if resp.status_code in TIMEOUT_STATUS_CODES: self.metric_retry_count += 1 raise TimeOutException logging.debug(f"response status code {resp.status_code}") logging.debug(f"response length {len(resp.text)}") if "Access Denied!" in resp.text: raise RuntimeError("AJAX request was denied") return resp @retry(TimeOutException, tries=5, delay=3, backoff=2) def _login(self): login_url = self.config["site"]["login_url"] username = self._secrets()["username"] password = self._secrets()["password"] self._get(login_url) resp = self.session.post( login_url, data={ "form": "login", "username": username, "password": password }, ) if resp.status_code in TIMEOUT_STATUS_CODES: self.metric_retry_count += 1 raise TimeOutException if "Error: Invalid username or password." in resp.text: raise RuntimeError("login failed because of invalid credentials") else: logging.debug("logged in") @retry(TimeOutException, tries=5, delay=3, backoff=2) def _max_pages(self): logging.debug("finding out torrents max page number") try: resp = self._get(self.config["site"]["torrents_url"]) if resp.status_code in TIMEOUT_STATUS_CODES: self.metric_retry_count += 1 raise TimeOutException if resp.status_code != 200: raise RuntimeError( "the torrents page is not responding correctly") pattern = r"ajax/torrents_data\.php\?total=(?P<max>\d+)&page=1" match = re.search(pattern, resp.text) if match is None: raise RuntimeError("could not find max page number") max_page = match.group("max") logging.debug(f"max pages figured out: {max_page}") return int(max_page) except ConnectionError as e: capture_exception(e) raise RuntimeError("failed to get the torrents page") def _download_media_info(self, torid) -> Optional[str]: logging.debug(f"getting torrent media info for {torid}") headers = {"X-Requested-With": "XMLHttpRequest"} url = self.config["site"]["torrent_techspec_url"].format(torid) resp = self._get(url=url, headers=headers) logging.debug(f"response status code {resp.status_code}") logging.debug(f"response length {len(resp.text)}") if len(resp.text) == 0: return None if "Access Denied!" in resp.text: raise RuntimeError("AJAX request was denied") return resp.html.html def _download_file_list(self, hashid) -> str: logging.debug(f"getting torrent file list for {hashid}") headers = {"X-Requested-With": "XMLHttpRequest"} url = self.config["site"]["torrent_filelist_url"].format(hashid) resp = self._get(url=url, headers=headers) logging.debug(f"response status code {resp.status_code}") logging.debug(f"response length {len(resp.text)}") if "Access Denied!" in resp.text: raise RuntimeError("AJAX request was denied") return resp.html.html def _cover_image_upload_and_get_url(self, url) -> str: matches = re.match(r".*/covers/(\d{4})/(\d{2})/(.*)", url) year = matches[1] month = matches[2] filename = matches[3] key = f"covers/{year}/{month}/{filename}" return self._upload(key, url) def _upload(self, key, url) -> str: """ Check if key exists in the bucket. If not, then download it from url and upload it to S3 as key. Set the object ACL to public readable. Return the public URL for the object. Args: key (str): S3 object key url (str): source URL to download the data from Returns: (str): the public URL in S3 """ bucket = self.config["s3"]["bucket"] try: self.s3.head_object(Bucket=bucket, Key=key) except ClientError: resp = self._get(url) self.s3.upload_fileobj( BytesIO(resp.content), bucket, key, ExtraArgs={"StorageClass": "STANDARD_IA"}, ) resp = self.s3.put_object_acl(ACL="public-read", Bucket=bucket, Key=key) if resp is None: capture_message(f"Failed to set object ACL for {bucket}/{key}") return self.config["s3"]["object_url"].format( bucket=self.config["s3"]["bucket"], region=self.config["s3"]["region"], filekey=key, ) def _thumbnail_small_image_upload_and_get_urls(self, urls) -> List: pub_urls = [] for url in urls: matches = re.match(r".*/screenthumb/(\d{4})/(\d{2})/(.*)", url) year = matches[1] month = matches[2] filename = matches[3] key = f"screenthumbs/small/{year}/{month}/{filename}" pub_urls.append(self._upload(key, url)) return pub_urls def _thumbnail_large_image_upload_and_get_urls(self, urls) -> List: pub_urls = [] for url in urls: matches = re.match(r".*/screens/(\d{4})/(\d{2})/(.*)", url) year = matches[1] month = matches[2] filename = matches[3] key = f"screenthumbs/large/{year}/{month}/{filename}" pub_urls.append(self._upload(key, url)) return pub_urls def _torrent_upload_and_get_url(self, url, torid, filename, publish_date) -> str: """ Args: url (str): Source URL to torrent torid (str): Torrent ID filename (str): The filename to use in the S3 key publish_date (datetime): Torrent publish date Returns: (str) S3 public URL for the file """ key = f"torrents/{publish_date.year}/{publish_date.month}/{filename}_{torid}.torrent" return self._upload(key, url) def _report_execution(self): self.cloudwatch.put_metric_data( Namespace="Animetorrents", MetricData=[{ "MetricName": "execution", "Value": 0.0 }], ) def _report_retry_count(self): self.cloudwatch.put_metric_data( Namespace="Animetorrents", MetricData=[{ "MetricName": "retries", "Value": self.metric_retry_count }], )
# # You should have received a copy of the GNU General Public License # along with PyCeed. If not, see <http://www.gnu.org/licenses/>. import feedparser from feedgen.feed import FeedGenerator out_feed = FeedGenerator() url = 'http://www.courantpositif.fr/feed/' d = feedparser.parse(url) print("~~ %s ~~" % d.feed.title) out_feed.title(d.feed.title) out_feed.subtitle(d.feed.subtitle) out_feed.id(d.feed.get("id", "no id")) out_feed.updated(d.feed.updated) for e in d.entries: print(" * [%s] %s" % (e.published, e.title)) out_entry = out_feed.add_entry() out_entry.title(e.title) out_entry.published(e.published) out_entry.updated(e.updated) out_entry.id(e.id) out_entry.summary(e.summary) for c in e.content: out_entry.content(content=c.value, type=c.type) #, src=c.base for l in e.links: print(" > [%s] %s" % (l.rel, l.href)) out_entry.link(link=l)
return [arg] parser = argparse.ArgumentParser() parser.add_argument('condition', type=condition) parser.add_argument('currency', type=currency) parser.add_argument('minimum_discount', type=int) parser.add_argument('outfile') args = parser.parse_args() try: fg = FeedGenerator() fg.id(FEED_URL) fg.title('Discogs Deals') fg.updated(now()) fg.link(href=FEED_URL, rel='self') fg.author(FEED_AUTHOR) with httpx.Client() as client: for deal in get_deals( client, args.condition, args.currency, args.minimum_discount ): fe = fg.add_entry() fe.id(deal['id']) fe.title(deal['title']) fe.updated(deal['updated'])
async def channel(request, channel_id, return_type='video'): log.info(f'Channel: {channel_id}') channel_name = [f'{channel_id}/{return_type}'] if channel_name[0] in channel_feed and channel_feed[ channel_name[0]]['expire'] > datetime.now(): return raw(channel_feed[channel_name[0]]['feed'], content_type='application/rss+xml') fg = None calls = 0 response = {'nextPageToken': ''} while 'nextPageToken' in response: next_page = response['nextPageToken'] payload = { 'part': 'snippet,contentDetails', 'maxResults': 50, 'channelId': channel_id, 'key': KEY, 'pageToken': next_page } response = json.loads( await get('https://www.googleapis.com/youtube/v3/activities', params=payload)) calls += 1 if 'error' in response: payload = { 'part': 'snippet', 'maxResults': 1, 'forUsername': channel_id, 'key': KEY } response = json.loads(await get( 'https://www.googleapis.com/youtube/v3/channels', params=payload)) channel_id = response['items'][0]['id'] channel_name.append(f'{channel_id}/{return_type}') payload = { 'part': 'snippet,contentDetails', 'maxResults': 50, 'channelId': channel_id, 'key': KEY, 'pageToken': next_page } response = json.loads(await get( 'https://www.googleapis.com/youtube/v3/activities', params=payload)) calls += 2 if not fg: fg = FeedGenerator() fg.load_extension('podcast') fg.generator('PodTube', __version__, 'https://github.com/aquacash5/PodTube') snippet = response['items'][0]['snippet'] if 'Private' in snippet['title']: continue icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) fg.title(snippet['title']) fg.id(f'http://{request.headers["host"]}{request.url}') fg.description(snippet['description'] or ' ') fg.author(name=snippet['channelTitle']) fg.image(snippet['thumbnails'][icon]['url']) fg.link(href=f'https://www.youtube.com/playlist?list={channel_id}') fg.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fg.podcast.itunes_summary(snippet['description']) fg.podcast.itunes_category('Technology', 'Podcasting') fg.updated(f'{str(datetime.utcnow())}Z') for item in response['items']: snippet = item['snippet'] if snippet['type'] != 'upload': continue current_video = item['contentDetails']['upload']['videoId'] log.debug(f'ChannelVideo: {current_video} {snippet["title"]}') fe = fg.add_entry() fe.title(snippet['title']) fe.id(current_video) icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) fe.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fe.updated(snippet['publishedAt']) if return_type == 'audio': fe.enclosure( url= f'http://{request.headers["host"]}/audio/{current_video}', type="audio/mpeg") else: fe.enclosure( url= f'http://{request.headers["host"]}/video/{current_video}', type="video/mp4") fe.author(name=snippet['channelTitle']) fe.podcast.itunes_author(snippet['channelTitle']) fe.podcast.itunes_author(snippet['channelTitle']) fe.pubdate(snippet['publishedAt']) fe.link(href=f'http://www.youtube.com/watch?v={current_video}', title=snippet['title']) fe.podcast.itunes_summary(snippet['description']) fe.description(snippet['description']) await sleep(0) feed = { 'feed': fg.rss_str(), 'expire': datetime.now() + timedelta(hours=calls) } for _name in channel_name: channel_feed[_name] = feed return raw(feed['feed'], content_type='application/rss+xml')
class Feed(): """ Instanceable class to manage a named feed including storage, retrieval and genration functions. :param feed_name: a string containg a feed name present in config.Feed.names() """ def __init__(self, feed_name: str) -> None: self.feed_name = feed_name self.alternates = {} self.added_mail_uuids = [] self.written_mail_uuids = None # Retrieve fg from shelf is it exists otherwise create it using config options with shelve.open( str(Path( config.paths["data"]).joinpath('feeds.shelf'))) as shelf: try: self.fg = shelf[self.feed_name] except KeyError as e: if not self.feed_name in shelf: # Mandatory ATOM values fg_config = config.ParseFeed.info(self.feed_name) self.fg = FeedGenerator() self.fg.id('tag:{},{}/feeds/{}.xml'.format( fg_config['fqdn'], date.today(), feed_name)) href_ = '{}{}/feeds/{}.xml'.format(fg_config['protocol'], fg_config['fqdn'], feed_name) self.fg.link(rel='self', type='application/atom+xml', href=href_) self.fg.title(feed_name) self.fg.subtitle( 'Feed generated from mail messages recieved at {} by refeed' .format(config.ParseFeed.account_name(self.feed_name))) self.fg.author(name=fg_config['author-name']) # Optional values try: self.fg.logo( str( Path(config.paths["static"]).joinpath( fg_config['logo']))) except KeyError: pass try: self.fg.language(fg_config['language']) except KeyError: pass else: raise KeyError(e) # context manager def __enter__(self) -> Feed: return self def __exit__(self, exc_type, exc_value, exc_traceback) -> None: self._dump_shelves() def add_entries_from_dict_if_new(self, mails: Dict[int, MailParser]) -> bool: try: for uuid, mail in mails: if FeedTools.uuid_not_in_feed(self.feed_name, uuid): self.add_entry((uuid, mail)) except (TypeError, ValueError): logging.error( 'Given NoneType as mailobject to Feed, some error in mail with IMAP.', exc_info=True) except Exception: logging.error('Unexpected error', exc_info=True) def add_entry(self, mail: Tuple[int, MailParser]) -> None: random.seed(None, 2) fe = self.fg.add_entry(order='prepend') fg_config = config.ParseFeed.info(self.feed_name) # id try: fe.id('tag:{},{}/feeds/{}.xml:{}'.format(fg_config['fqdn'], date.today(), self.feed_name, mail[0])) except (AttributeError, MailParserReceivedParsingError): fe.id('tag:{},{}/feeds/{}.xml:ID_NOT_FOUND-{}'.format( fg_config['fqdn'], date.today(), self.feed_name, ''.join( random.choices(string.ascii_lowercase + string.digits, k=10)))) # title try: fe.title(mail[1].subject) except (AttributeError, MailParserReceivedParsingError): fe.title('SUBJECT_NOT_FOUND-{}'.format(''.join( random.choices(string.ascii_lowercase + string.digits, k=10)))) # alt link and body contents try: alt_id = FeedTools.generate_unique_alt_id() self.alternates[alt_id] = mail[1].body alt_link = '{}{}/alt-html/{}.html'.format(fg_config['protocol'], fg_config['fqdn'], alt_id) fe.link(rel='alternate', type='text/html', href=alt_link) fe.contents(content=mail[1].body, src=alt_link, type='text/html') except (AttributeError, MailParserReceivedParsingError): fe.contents(content='MAIL_BODY_NOT_FOUND', type='text/plain') #update time now = datetime.now( ) # entry and feed should match exactly, not be a few seconds off. fe.updated(now) self.fg.updated(now) # cache uuids added to feed self.added_mail_uuids.append(mail[0]) def generate_feed(self) -> None: # generate htmls if self.alternates != {}: try: for alt_id, body in self.alternates.items(): with Path(config.paths["static"]).joinpath( 'alt', '{}.html'.format(str(alt_id))).open(mode='w') as f: f.write(body) except Exception: # Exception gets *most* inbuilt exceptions, except KeyboardInterrupt, SystemInterrupt and some others which are out of scope logging.error( 'Failed to write some html alt pages to file for new entries for feed {}' .format(self.feed_name), exc_info=True) finally: logging.info( 'Successfully generated html alt pages: {} for feed {}'. format(list(self.alternates.keys()), self.feed_name)) FeedTools.cleanup_alts( self.feed_name, config.ParseFeed.alternate_cache(self.feed_name)) # generate xml try: self.fg.atom_file( str( Path(config.paths["static"]).joinpath( 'feed', '{}.xml'.format(self.feed_name)))) except Exception: # TODO: Find out what f*****g exceptions that feedgen actually raises, if any(not documented - check source) logging.error( 'Failed to generate and write new copy of feed {} to file'. format(self.feed_name)) finally: self.written_mail_uuids = self.added_mail_uuids def _dump_shelves(self) -> None: with shelve.open( str(Path( config.paths["data"]).joinpath('feeds.shelf'))) as shelf: shelf[self.feed_name] = self.fg logging.info('Atom data for feed {} stored to disk'.format( self.feed_name)) with shelve.open( str( Path(config.paths["data"]).joinpath( 'alternate_ids.shelf'))) as shelf: try: shelf[self.feed_name] = shelf[self.feed_name].extend( list(self.alternates.keys())) except (KeyError, AttributeError): # feed alternates list does not exist yet shelf[self.feed_name] = list(self.alternates.keys()) logging.info( 'Alt id data for feed {} stored to disk for first time'. format(self.feed_name)) finally: logging.info( 'Alt id data for feed {} stored back to disk'.format( self.feed_name)) with shelve.open( str(Path(config.paths["data"]).joinpath( 'mail_uuids.shelf'))) as shelf: try: shelf[self.feed_name] = shelf[self.feed_name].extend( self.written_mail_uuids) except (KeyError, AttributeError): # feed id list does not exist yet shelf[self.feed_name] = self.written_mail_uuids logging.info( 'Mail UUID data for feed {} stored to disk for first time'. format(self.feed_name)) except TypeError: if self.written_mail_uuids is None: logging.info( 'Failed to write mail UUIDs to shelf file for feed {}: Newly written mail UUID data is None. Feed._dump_shelves() was likely called without any new items beeing added to feed' .format(self.feed_name), exc_info=True) else: logging.error( 'Failed to write mail UUIDs to shelf file for feed {}: Newly written mail UUID is not None, some unexpected error has occured. ' .format(self.feed_name), exc_info=True) finally: logging.info( 'Mail UUID data for feed {} stored back to disk'.format( self.feed_name))
def setUp(self): fg = FeedGenerator() self.nsAtom = "http://www.w3.org/2005/Atom" self.nsRss = "http://purl.org/rss/1.0/modules/content/" self.feedId = 'http://lernfunk.de/media/654321' self.title = 'Some Testfeed' self.authorName = 'John Doe' self.authorMail = '*****@*****.**' self.author = {'name': self.authorName, 'email': self.authorMail} self.linkHref = 'http://example.com' self.linkRel = 'alternate' self.logo = 'http://ex.com/logo.jpg' self.subtitle = 'This is a cool feed!' self.link2Href = 'http://larskiesow.de/test.atom' self.link2Rel = 'self' self.language = 'en' self.categoryTerm = 'This category term' self.categoryScheme = 'This category scheme' self.categoryLabel = 'This category label' self.cloudDomain = 'example.com' self.cloudPort = '4711' self.cloudPath = '/ws/example' self.cloudRegisterProcedure = 'registerProcedure' self.cloudProtocol = 'SOAP 1.1' self.icon = "http://example.com/icon.png" self.contributor = { 'name': "Contributor Name", 'uri': "Contributor Uri", 'email': 'Contributor email' } self.copyright = "The copyright notice" self.docs = 'http://www.rssboard.org/rss-specification' self.managingEditor = '*****@*****.**' self.rating = '(PICS-1.1 "http://www.classify.org/safesurf/" ' + \ '1 r (SS~~000 1))' self.skipDays = 'Tuesday' self.skipHours = 23 self.textInputTitle = "Text input title" self.textInputDescription = "Text input description" self.textInputName = "Text input name" self.textInputLink = "Text input link" self.ttl = 900 self.webMaster = '*****@*****.**' fg.id(self.feedId) fg.title(self.title) fg.author(self.author) fg.link(href=self.linkHref, rel=self.linkRel) fg.logo(self.logo) fg.subtitle(self.subtitle) fg.link(href=self.link2Href, rel=self.link2Rel) fg.language(self.language) fg.cloud(domain=self.cloudDomain, port=self.cloudPort, path=self.cloudPath, registerProcedure=self.cloudRegisterProcedure, protocol=self.cloudProtocol) fg.icon(self.icon) fg.category(term=self.categoryTerm, scheme=self.categoryScheme, label=self.categoryLabel) fg.contributor(self.contributor) fg.copyright(self.copyright) fg.docs(docs=self.docs) fg.managingEditor(self.managingEditor) fg.rating(self.rating) fg.skipDays(self.skipDays) fg.skipHours(self.skipHours) fg.textInput(title=self.textInputTitle, description=self.textInputDescription, name=self.textInputName, link=self.textInputLink) fg.ttl(self.ttl) fg.webMaster(self.webMaster) fg.updated('2017-02-05 13:26:58+01:00') fg.pubDate('2017-02-05 13:26:58+01:00') fg.generator('python-feedgen', 'x', uri='http://github.com/lkie...') fg.image(url=self.logo, title=self.title, link=self.link2Href, width='123', height='123', description='Example Inage') self.fg = fg