class YoutubeFeed: ydl_opts = { 'format': 'bestaudio/best', 'outtmpl': '%(id)s.%(ext)s', 'postprocessors': [{ 'key': 'FFmpegExtractAudio', 'preferredcodec': 'mp3', 'preferredquality': '192', }] } def __init__(self, name): self.name = name self.ydl = youtube_dl.YoutubeDL(self.ydl_opts) self.fg = FeedGenerator() self.fg.title(name) self.fg.author({"name": "Youtube Audio Feed", "email": ""}) self.fg.link(href="http://www.foo.bar.baz.com", rel="alternate") self.fg.description("Personalized Youtube audio feed") self.fg.generator("") self.fg.docs("") def add_video(self, url): info = self.ydl.extract_info(url, download=True) entry = self.fg.add_entry() entry.id(info['id']) entry.title(info['title']) entry.description(info['description']) entry.enclosure(info['id'] + ".mp3", str(info['duration']), 'audio/mpeg') def save(self): self.fg.rss_file(name + '.xml')
def make_feedgenerator(conf): feedgen = FeedGenerator() feedgen.title('Lojban twitter feed in {lang}'.format(lang=conf['long'])) feedgen.description('Twitter Atom feed in {lang} about the constructed language Lojban'.format(lang=conf['long'])) feedgen.language(conf['short']) feedgen.link(href='{}.atom.xml'.format(conf['short'])) feedgen.id('{}.atom.xml'.format(conf['short'])) feedgen.generator(generator='bano', version='0.0.0', uri='https://github.com/kyrias/bano') return feedgen
async def create_rss(channel_alias: str, request: Request): """ Get posts from the channel and return rss-feed """ global channel_hash, client channel_alias = channel_alias.lstrip('@') private_channel = channel_alias[:8] == 'joinchat' if private_channel: private_hash = channel_alias[8:] channel_alias = 't.me/joinchat/' + private_hash try: await client.start() if channel_alias not in channel_hash: if private_channel: await client(ImportChatInviteRequest(private_hash)) channel = await client.get_entity(channel_alias) ch_full = await client(GetFullChannelRequest(channel=channel)) username = channel.username or channel.id channel_hash[channel_alias] = { 'username': username, 'title': channel.title, 'id': channel.id, 'about': ch_full.full_chat.about or str(username), } logging.info(f"Adding to the hash '{channel_alias}'") with open('hash.pickle', 'wb') as f: pickle.dump(channel_hash, f) ch = channel_hash[channel_alias] messages = [ m async for m in client.iter_messages( ch['username'], limit=int(config['RSS']['RECORDS'])) ] except Exception as e: warn = f"{str(e)}, request: '{channel_alias}'" logging.warning(warn) return warn fg = FeedGenerator() fg.title(f"{ch['title']} (@{ch['username']}, id:{ch['id']})") fg.subtitle(ch['about']) link = channel_alias if private_channel else f"t.me/s/{ch['username']}" fg.link(href=f'https://{link}', rel='alternate') fg.generator(config['RSS']['GENERATOR']) fg.language(config['RSS']['LANGUAGE']) for m in messages: if not (config['RSS'].getboolean('SKIP_EMPTY') and not m.text): fe = fg.add_entry(order='append') link = 'https://t.me/' + ('c/' if private_channel else '') fe.guid(guid=f"{link}{ch['username']}/{m.id}", permalink=True) fe.content(markdown(m.text)) fe.published(m.date) logging.debug(f"Successfully requested '{ch['username']}'") return Response(content=fg.rss_str(), media_type='application/xml')
def _get_feed(query: Optional[ClassicAPIQuery] = None) -> FeedGenerator: fg = FeedGenerator() fg.generator("") fg.register_extension("opensearch", OpenSearchExtension) fg.register_extension("arxiv", ArXivExtension, ArXivEntryExtension, rss=False) if query: if query.phrase is not None: query_string = phrase_to_query_string(query.phrase) else: query_string = "" if query.id_list: id_list = ",".join(query.id_list) else: id_list = "" fg.title(f"arXiv Query: {query.to_query_string()}") # From perl documentation of the old site: # search_id is calculated by taking SHA-1 digest of the query # string. Digest is in bytes form and it's 20 bytes long. Then it's # base64 encoded, but perls version returns only 27 characters - # it omits the `=` sign at the end. search_id = base64.b64encode( hashlib.sha1(query.to_query_string().encode( "utf-8")).digest()).decode("utf-8")[:-1] fg.id( url_for("classic_api.query").replace("/query", f"/{search_id}")) fg.link({ "href": url_for( "classic_api.query", search_query=query_string, start=query.page_start, max_results=query.size, id_list=id_list, ), "type": "application/atom+xml", }) else: # TODO: Discuss better defaults fg.title("arXiv Search Results") fg.id("https://arxiv.org/") fg.updated(to_utc(datetime.utcnow())) return fg
def __init__( self, id_: str, links: List[dict], title: str, updated: Optional[str], ): fg = FeedGenerator() fg.id(id_) fg.link(links) fg.title(title) fg.updated(updated) fg.generator("") self._feed_generator = fg
def build_feed(uri, path, title, description, articles): uri = uri.rstrip("/") blog_uri = f"{uri}/{path.split('/')[1]}" feed = FeedGenerator() feed.generator("Python Feedgen") feed.title(title) feed.description(description) feed.link(href=f"{uri}{path}", rel="self") for article in articles: feed.add_entry(build_entry(article, blog_uri), order="append") return feed
def notices_feed(feed_type): if feed_type not in ["atom", "rss"]: flask.abort(404) url_root = flask.request.url_root base_url = flask.request.base_url feed = FeedGenerator() feed.generator("Feedgen") feed.id(url_root) feed.copyright( f"{datetime.now().year} Canonical Ltd. " "Ubuntu and Canonical are registered trademarks of Canonical Ltd." ) feed.title("Ubuntu security notices") feed.description("Recent content on Ubuntu security notices") feed.link(href=base_url, rel="self") def feed_entry(notice, url_root): _id = notice.id title = f"{_id}: {notice.title}" description = notice.details published = notice.published notice_path = flask.url_for(".notice", notice_id=notice.id).lstrip("/") link = f"{url_root}{notice_path}" entry = FeedEntry() entry.id(link) entry.title(title) entry.description(description) entry.link(href=link) entry.published(f"{published} UTC") entry.author({"name": "Ubuntu Security Team"}) return entry notices = ( db_session.query(Notice) .order_by(desc(Notice.published)) .limit(10) .all() ) for notice in notices: feed.add_entry(feed_entry(notice, url_root), order="append") payload = feed.atom_str() if feed_type == "atom" else feed.rss_str() return flask.Response(payload, mimetype="text/xml")
def get(self, request, user_pk, feed_token): user = get_object_or_404(User, pk=user_pk, feed_token=feed_token, is_active=True) with user.localized(): gen = FeedGenerator() gen.generator(generator="MatShare", version=__version__, uri=settings.MS_URL) gen.author(name="MatShare", email=settings.MS_CONTACT_EMAIL) gen.link(href=settings.MS_ROOT_URL + reverse("user_dashboard"), rel="alternate") self.populate_feed(user, gen) return HttpResponse(gen.atom_str(), content_type="application/atom+xml")
def _build_feed( self, blog_url, feed_url, feed_title, feed_description, articles ): """ Build the content for the feed :blog_url: string blog url :feed_url: string url :feed_title: string title :feed_description: string description :param articles: Articles to create feed from """ feed = FeedGenerator() feed.generator("Python Feedgen") feed.title(feed_title) feed.description(feed_description) feed.link(href=feed_url, rel="self") for article in articles: title = article["title"]["rendered"] slug = article["slug"] author = article["_embedded"]["author"][0] description = article["excerpt"]["rendered"] content = article["content"]["rendered"] published = f'{article["date_gmt"]} GMT' updated = f'{article["modified_gmt"]} GMT' link = f"{blog_url}/{slug}" categories = [] if "wp:term" in article["_embedded"]: for category in article["_embedded"]["wp:term"][1]: categories.append( dict(term=category["slug"], label=category["name"]) ) entry = FeedEntry() entry.title(title) entry.description(description) entry.content(content) entry.author(name=author["name"], email=author["name"]) entry.link(href=link) entry.category(categories) entry.published(published) entry.updated(updated) feed.add_entry(entry, order="append") return feed
async def read_item(channel_alias: str, request: Request): try: channel = await client.get_entity(channel_alias) except Exception as e: logger.error('channel ERROR: {}'.format(e)) channel = None return channel else: logger.info('Запросили канал {}'.format(channel.username)) fg = FeedGenerator() fg.title('Recent posts from @' + channel_alias) fg.author({'name': 'Nik Parotikov', 'email': '*****@*****.**'}) fg.link(href=str(request.url), rel='alternate') fg.subtitle(channel.title + ' ' + 'https://t.me/' + channel.username + ' (id ' + str(channel.id) + ')') fg.generator( 'http://tg2rss.prosto-tak.ru/ - Convert any Telegram channel to RSS feed' ) fg.link(href=str(request.url), rel='self') fg.language('ru') # return channel # messages = await client.get_messages(channel, 20) # logger.info('сообщения из канала {}'.format(messages)) async for message in client.iter_messages(channel, 20): # if hasattr(message.photo, 'file_reference'): # logger.info('Сообщение {}'.format(message.photo.sizes[-1].location)) fe = fg.add_entry() fe.guid(guid='https://t.me/' + channel_alias + '/' + str(message.id), permalink=True) # fe.link(href='https://t.me/' + channel_alias + '/' + str(message.id)) # fe.title(message.text) # fe.description(message.text) fe.content(markdown2.markdown(message.text)) # fe.enclosure() fe.published(message.date) # rssfeed = fg.rss_str(pretty=True) rssfeed = fg.rss_str() # return rssfeed.decode() return Response(content=rssfeed, media_type="application/xml")
def rss(): fg = FeedGenerator() fg.title('CloudWalk DevSecOps test') fg.description('A RSS Feed for HTTP and TCP service') fg.docs('') fg.generator('') fg.link(href='http://example.com') with open('log.txt') as f: for line in f.readlines(): info = line.replace('\n', '').split(';') fe = fg.add_entry() fe.title(f"{info[1]}") fe.pubDate(f"{info[0]} GTM-3") fe.description(f"server: {info[2]} port:{info[3]}") response = make_response(fg.rss_str()) response.headers.set('Content-type', 'application/rss+xml') return response
def user_rss(service, id): cursor = get_cursor() query = "SELECT * FROM posts WHERE \"user\" = %s AND service = %s " params = (id, service) query += "ORDER BY added desc " query += "LIMIT 10" cursor.execute(query, params) results = cursor.fetchall() cursor3 = get_cursor() query3 = "SELECT * FROM lookup WHERE id = %s AND service = %s" params3 = (id, service) cursor3.execute(query3, params3) results3 = cursor.fetchall() name = results3[0]['name'] if len(results3) > 0 else '' fg = FeedGenerator() fg.title(name) fg.description('Feed for posts from ' + name + '.') fg.id(f'http://{request.headers.get("host")}/{service}/user/{id}') fg.link(href=f'http://{request.headers.get("host")}/{service}/user/{id}') fg.generator(generator='Kemono') fg.ttl(ttl=40) for post in results: fe = fg.add_entry() fe.title(post['title']) fe.id( f'http://{request.headers.get("host")}/{service}/user/{id}/post/{post["id"]}' ) fe.link( href= f'http://{request.headers.get("host")}/{service}/user/{id}/post/{post["id"]}' ) fe.content(content=post["content"]) fe.pubDate(pytz.utc.localize(post["added"])) response = make_response(fg.atom_str(pretty=True), 200) response.headers['Content-Type'] = 'application/rss+xml' return response
def gen_feed(title, base_url, feed_url, num_cutoff, entries): fg = FeedGenerator() if feed_url: fg.id(feed_url) fg.link(href=feed_url, rel="self") else: fg.id(base_url) fg.title(title) fg.generator(generator="dir2feed", uri="https://github.com/pR0Ps/dir2feed") all_entries = sorted(entries, key=attrgetter("date", "title")) for e in all_entries[-max(0, num_cutoff or 0):]: fe = fg.add_entry() fe.id(e.url) fe.title(e.title) fe.link(rel="alternate", href=e.url) fe.updated(e.date) for l in e.links(): fe.link(**l) fe.summary(summary=e.summary(), type="html") return fg
def build_feed(self, commits: List[Commit]): log.info("build feed page %d" % len(commits)) feed = FeedGenerator() feed.id("") feed.title("AWS API Changes") feed.author({ "name": "AWSPIChanges", "email": "https://github.com/awslabs/aws-sdk-api-changes", }) feed.link(href=self.site_url, rel="alternate") feed.link(href="%s/feed/" % self.site_url, rel="self") feed.description("AWS API ChangeLog") feed.language("en-US") feed.generator("artisan-sdk-gitops") feed.image( url= "https://a0.awsstatic.com/main/images/logos/aws_logo_smile_179x109.png" ) # noqa for c in commits: for s in c.service_changes: fe = feed.add_entry(order="append") fe.title("{} - {}{}methods".format( s.title, s.count_new and "%d new " % s.count_new or "", s.count_updated and "%d updated " % s.count_updated or "", )) fe.id("{}-{}".format(c.id, s.name)) fe.description(s.change_log) fe.link({ "href": self.link("archive/changes/%s-%s.html" % (c.id[:6], s.name)) }) fe.published(c.created) self.render_page( "feed/feed.rss", force=True, content=feed.rss_str(pretty=True).decode("utf8"), )
def display_feed(feed_name): if feed_name not in config['feeds']: abort(404) f = Feed(config['feeds'][feed_name]) f.load() f.parse() fg = FeedGenerator() fg.generator(**_generator) fg.id(request.base_url) fg.link( href=request.base_url, rel='self', ) fg.title(f.properties.get('title', feed_name)) fg.author(name=f.properties.get('author', '')) fg.updated( timezone.localize( _round_date(max([e['updated'] for e in f.entries]), config.get('date_rounding', None)))) for entry in f.entries: fe = fg.add_entry() fe.id(entry['url']) fe.title(entry['title']) fe.link(href=entry['url']) fe.updated( timezone.localize( _round_date(entry['updated'], config.get('date_rounding', None)))) fe.content(entry['content']) atomfeed = fg.atom_str() resp = make_response(atomfeed) resp.headers['content-type'] = 'application/xml' return resp
def render(self, data, accepted_media_type=None, renderer_context=None): """ Renders *data* into serialized XML. """ if data is None: return '' fg = FeedGenerator() feed_url = settings.API_URL + reverse('api:veroeffentlichung-list') feed_title = 'OffeneGesetze.de' feed_description = 'Feed für Veröffentlichungen des Bundesgesetzblatts' fg.id(feed_url) fg.title(feed_title) fg.subtitle(feed_description) fg.link(href=feed_url, rel='alternate') fg.logo('https://offenegesetze.de/apple-touch-icon.png') fg.link(href=feed_url + '?format=rss', rel='self') fg.language('de') fg.generator('') if not isinstance(data, list): data = [data] results = reversed(data[0].get('results', [])) for item in results: fe = fg.add_entry() fe.id('%s/%s' % (settings.SITE_URL, item['id'])) fe.pubDate(item['date']) fe.title(item['title']) fe.link({'href': item['url']}) if 'content' in item: fe.description(item['content'] if isinstance( item['content'], str) else ''.join(item['content'])) return fg.rss_str(pretty=True)
def to_atom(self, deployment_uri, use_summary=False): fg = FeedGenerator() fg.load_extension('podcast') fg.id(deployment_uri) fg.title(self.title) fg.author({"name": "Atomizer/1.0"}) fg.generator("Atomizer") fg.link(href=self.uri, rel='alternate', type="text/html") fg.link(href=deployment_uri, rel='self') fg.description(self.title) for entry in self.entries: feed_item = fg.add_entry(order='append') feed_item.id(entry.link) feed_item.title(entry.title) feed_item.updated(self.ensure_tz_utc(entry.date)) author = {"name": entry.author} if entry.author_uri: author['uri'] = entry.author_uri feed_item.author(author) feed_item.published(self.ensure_tz_utc(entry.date)) feed_item.link(link={ "href": entry.link, "rel": "alternate", "type": "text/html" }) if entry.enclosures: for enclosure in entry.enclosures: if enclosure.get("href"): feed_item.enclosure(url=enclosure.get("href"), length=enclosure.get("length", 0), type=enclosure.get("type", "")) if use_summary and entry.summary: feed_item.summary(entry.summary_html) feed_item.content(content=entry.content_html, type="html") return fg.atom_str(pretty=True)
class SteamCommentsRss: def __init__(self, app_id): self.app_id = app_id self.steam_comments = CommentsFeed(app_id) self.feed = FeedGenerator() self.feed.id( "https://steamcommunity.com/games/{app_id}/allnews/".format( app_id=self.app_id)) self.feed.title("PULSAR: Lost Colony Developer Comments on Steam") self.feed.link(href="https://pulsar.wiki/leafytracker/") self.feed.description( "Comments by leafygamesdev on PULSAR news articles.") self.feed.language("en") self.feed.generator("https://pulsar.wiki/leafytracker/" ) # TODO: Automate project name and version def append_comments(self, news_ids, user_ids): news_ids = set(int(x) for x in news_ids) user_ids = set(int(x) for x in user_ids) for nid in news_ids: for comment in self.steam_comments.get(nid, user_ids): entry = self.feed.add_entry() entry.id(str(comment.cid)) entry.link({"href": comment.url}) entry.title("{} commented on {}".format( comment.author.name, comment.title)) entry.author({"name": comment.author.name}) entry.published(comment.datetime) entry.content(comment.body) def to_atom(self, output_path, pretty=True): return self.feed.atom_file(output_path, pretty=pretty) def to_rss(self, output_path, pretty=True): return self.feed.rss_file(output_path, pretty=pretty)
class Feed: def __init__(self, url: str, name: str, email: str, title: str = None, generator: str = None, generator_version: str = None, logo: str = None, icon: str = None, description: str = None, language: str = None) -> None: self.name = name self.email = email self.fg = FeedGenerator() self.fg.id(url + "feed.atom") self.fg.link(href=url + "feed.xml", rel="self") self.fg.link(href=url, rel="alternate") self.fg.author(name=name, email=email) self.fg.contributor(name=name, email=email) self.fg.managingEditor(email) self.fg.webMaster(email) self.fg.title(title) self.fg.generator(generator=generator, version=generator_version) self.fg.logo(logo) self.fg.icon(icon) self.fg.description(description) self.fg.language(language) def add(self, article: Article) -> None: feed_entry = self.fg.add_entry() feed_entry.id(article.url) feed_entry.title(article.title) feed_entry.link(href=article.url) feed_entry.guid(guid=article.url, permalink=True) feed_entry.author(name=self.name, email=self.email) feed_entry.summary(article.description or article.snippet) feed_entry.content(content=article.content, type="CDATA") feed_entry.published(article.date) if article.date: feed_entry.published(article.date) feed_entry.updated(article.date) else: epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) feed_entry.published(epoch) feed_entry.updated(epoch) def add_from_blog(self, url: str) -> None: blog = Blog(url) if not self.fg.title(): self.fg.title(blog.title) for article in blog.articles: self.add(article) def atom(self) -> bytes: return self.fg.atom_str(pretty=True) def rss(self) -> bytes: return self.fg.rss_str(pretty=True) def atom_file(self, filename: str = "feed.atom") -> None: self.fg.atom_file(filename, pretty=True) def rss_file(self, filename: str = "feed.xml") -> None: self.fg.rss_file(filename, pretty=True)
def get(self, mc, db, pkey): def check_encoding(string): data = string if string is not unicode: data = unicode(string) return ud.normalize('NFKD', data).encode('ascii', 'xmlcharrefreplace') try: # host URL urlparts = request.urlparts host_url = '%s://%s/feeds/%s' % (urlparts.scheme, urlparts.netloc, pkey) # get feed data cfg = self._app.config obj = FeedService.get_feed_activities(db, mc, cfg, pkey) activities = obj['activities'] user_id = obj['user_id'] # main element channel = FeedGenerator() channel.title('Plus Channel feed') channel.description('Google+ List of Activities for %s' % obj['name']) channel.generator('Plus Channel %s' % cfg.get('main.version')) channel.id('https://plus.google.com/' + user_id) channel.link(href=host_url, rel='self') channel.docs('') if 'photo_url' in obj and obj['photo_url'] is not None: channel.image(url=obj['photo_url'], title='Plus Channel feed', link='https://plus.google.com/' + user_id, width=str(cfg.get('feed.photo_size.database')), height=str(cfg.get('feed.photo_size.database'))) # additional namespaces channel.register_extension('media', MediaExtension, MediaEntryExtension) channel.register_extension('geo', GeoExtension, GeoEntryExtension) # compose items h = HTMLParser.HTMLParser() for activity in activities: title = activity['title'] content = activity['content'] url = activity['url'] # check content if content is None or content == title: content = '' # check title if title is None: title = 'notitle' # reformat strings title = h.unescape(title) title = re.sub('<[^>]*>', '', title) title = escape(title) content = h.unescape(content) content = re.sub('<[^>]*>', '', content) content = escape(content) # log activity logging.debug('--- activity ---') logging.debug(title) logging.debug(content) logging.debug(url) logging.debug('----------------') # create item item = channel.add_entry() item.title(check_encoding(title)) item.pubdate(activity['datePublished']) # process content c_content = check_encoding(content) item.description(c_content) item.content(content=c_content, type='CDATA') # # check image presence if 'imageUrl' in activity and activity['imageUrl'] != '': item.media.media_thumbnail_url(activity['imageUrl']) # check size if 'imageWidth' in activity and 'imageHeight' in activity: item.media.media_thumbnail_width(activity['imageWidth']) item.media.media_thumbnail_height(activity['imageHeight']) # check coordinates if activity['hasCoordinates']: item.geo.geo_lat(activity['latitude']) item.geo.geo_long(activity['longitude']) # check link if url is None or url == '': url = activity['url'] item.link(href=escape(url), rel='alternate') item.guid(escape(activity['id'])) # return created feed response.set_header('content-type', 'application/rss+xml; charset=utf-8') out = channel.rss_str(pretty=True) del channel, activities, user_id, obj return out except FeedService.FeedNotFoundException: abort(404) except FeedService.UserIdNotFoundException: abort(410)
def get(self, channel): channel = channel.split('/') if len(channel) < 2: channel.append('video') channel_name = ['/'.join(channel)] self.set_header('Content-type', 'application/rss+xml') if channel_name[0] in channel_feed and channel_feed[ channel_name[0]]['expire'] > datetime.datetime.now(): self.write(channel_feed[channel_name[0]]['feed']) self.finish() return fg = None video = None calls = 0 response = {'nextPageToken': ''} while 'nextPageToken' in response.keys(): next_page = response['nextPageToken'] payload = { 'part': 'snippet,contentDetails', 'maxResults': 50, 'channelId': channel[0], 'key': key, 'pageToken': next_page } request = requests.get( 'https://www.googleapis.com/youtube/v3/activities', params=payload) calls += 1 if request.status_code != 200: payload = { 'part': 'snippet', 'maxResults': 1, 'forUsername': channel[0], 'key': key } request = requests.get( 'https://www.googleapis.com/youtube/v3/channels', params=payload) response = request.json() channel[0] = response['items'][0]['id'] channel_name.append('/'.join(channel)) payload = { 'part': 'snippet,contentDetails', 'maxResults': 50, 'channelId': channel[0], 'key': key, 'pageToken': next_page } request = requests.get( 'https://www.googleapis.com/youtube/v3/activities', params=payload) calls += 2 response = request.json() if request.status_code == 200: logging.debug('Downloaded Channel Information') else: logging.error('Error Downloading Channel: %s', request.reason) self.send_error(reason='Error Downloading Channel') return if not fg: fg = FeedGenerator() fg.load_extension('podcast') fg.generator('PodTube (python-feedgen)', __version__, 'https://github.com/aquacash5/PodTube') for item in response['items']: if item['snippet']['type'] != 'upload': continue elif 'Private' in item['snippet']['title']: continue else: snippet = item['snippet'] break logging.info('Channel: %s (%s)', channel[0], snippet['channelTitle']) icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) fg.title(snippet['channelTitle']) fg.id('http://' + self.request.host + self.request.uri) fg.description(snippet['description'] or ' ') fg.author(name='Podtube', email='*****@*****.**', uri='https://github.com/aquacash5/PodTube') fg.podcast.itunes_author(snippet['channelTitle']) fg.image(snippet['thumbnails'][icon]['url']) fg.link(href=f'http://youtube.com/channel/{channel}', rel='self') fg.language('en-US') fg.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fg.podcast.itunes_explicit('no') fg.podcast.itunes_owner(name='Podtube', email='*****@*****.**') fg.podcast.itunes_summary(snippet['description']) fg.podcast.itunes_category(cat='Technology') fg.updated(str(datetime.datetime.utcnow()) + 'Z') for item in response['items']: snippet = item['snippet'] if snippet['type'] != 'upload': continue if 'private' in snippet['title'].lower(): continue current_video = item['contentDetails']['upload']['videoId'] logging.debug('ChannelVideo: %s (%s)', current_video, snippet['title']) fe = fg.add_entry() fe.title(snippet['title']) fe.id(current_video) icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) fe.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fe.updated(snippet['publishedAt']) if channel[1] == 'video': fe.enclosure( url=f'http://{self.request.host}/video/{current_video}', type="video/mp4") elif channel[1] == 'audio': fe.enclosure( url=f'http://{self.request.host}/audio/{current_video}', type="audio/mpeg") fe.author(name=snippet['channelTitle']) fe.podcast.itunes_author(snippet['channelTitle']) fe.pubDate(snippet['publishedAt']) fe.link(href=f'http://www.youtube.com/watch?v={current_video}', title=snippet['title']) fe.podcast.itunes_summary(snippet['description']) fe.description(snippet['description']) if not video or video['expire'] < fe.pubDate(): video = {'video': fe.id(), 'expire': fe.pubDate()} feed = { 'feed': fg.rss_str(), 'expire': datetime.datetime.now() + datetime.timedelta(hours=calls) } for chan in channel_name: channel_feed[chan] = feed self.write(feed['feed']) self.finish() video = video['video'] mp3_file = 'audio/{}.mp3'.format(video) if channel[1] == 'audio' and not os.path.exists( mp3_file) and video not in conversion_queue.keys(): conversion_queue[video] = { 'status': False, 'added': datetime.datetime.now() }
def get(self, playlist): playlist = playlist.split('/') if len(playlist) < 2: playlist.append('video') playlist_name = '/'.join(playlist) self.set_header('Content-type', 'application/rss+xml') if playlist_name in playlist_feed and playlist_feed[playlist_name][ 'expire'] > datetime.datetime.now(): self.write(playlist_feed[playlist_name]['feed']) self.finish() return calls = 0 payload = {'part': 'snippet', 'id': playlist[0], 'key': key} request = requests.get( 'https://www.googleapis.com/youtube/v3/playlists', params=payload) calls += 1 response = request.json() if request.status_code == 200: logging.debug('Downloaded Playlist Information') else: logging.error('Error Downloading Playlist: %s', request.reason) self.send_error(reason='Error Downloading Playlist') return fg = FeedGenerator() fg.load_extension('podcast') fg.generator('PodTube (python-feedgen)', __version__, 'https://github.com/aquacash5/PodTube') snippet = response['items'][0]['snippet'] icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) logging.info('Playlist: %s (%s)', playlist[0], snippet['title']) fg.title(snippet['title']) fg.id('http://' + self.request.host + self.request.uri) fg.description(snippet['description'] or ' ') fg.author(name='Podtube', email='*****@*****.**', uri='https://github.com/aquacash5/PodTube') fg.podcast.itunes_author(snippet['channelTitle']) fg.image(snippet['thumbnails'][icon]['url']) fg.link(href=f'http://youtube.com/playlist/?list={playlist}', rel='self') fg.language('en-US') fg.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fg.podcast.itunes_explicit('no') fg.podcast.itunes_owner(name='Podtube', email='*****@*****.**') fg.podcast.itunes_summary(snippet['description']) fg.podcast.itunes_category(cat='Technology') fg.updated(str(datetime.datetime.utcnow()) + 'Z') video = None response = {'nextPageToken': ''} while 'nextPageToken' in response.keys(): payload = { 'part': 'snippet', 'maxResults': 50, 'playlistId': playlist[0], 'key': key, 'pageToken': response['nextPageToken'] } request = requests.get( 'https://www.googleapis.com/youtube/v3/playlistItems', params=payload) calls += 1 response = request.json() if request.status_code == 200: logging.debug('Downloaded Playlist Information') else: logging.error('Error Downloading Playlist: %s', request.reason) self.send_error(reason='Error Downloading Playlist Items') return for item in response['items']: snippet = item['snippet'] current_video = snippet['resourceId']['videoId'] if 'Private' in snippet['title']: continue logging.debug('PlaylistVideo: %s (%s)', current_video, snippet['title']) fe = fg.add_entry() fe.title(snippet['title']) fe.id(current_video) icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) fe.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fe.updated(snippet['publishedAt']) if playlist[1] == 'video': fe.enclosure( url=f'http://{self.request.host}/video/{current_video}', type="video/mp4") elif playlist[1] == 'audio': fe.enclosure( url=f'http://{self.request.host}/audio/{current_video}', type="audio/mpeg") fe.author(name=snippet['channelTitle']) fe.podcast.itunes_author(snippet['channelTitle']) fe.pubDate(snippet['publishedAt']) fe.link(href=f'http://www.youtube.com/watch?v={current_video}', title=snippet['title']) fe.podcast.itunes_summary(snippet['description']) fe.description(snippet['description']) if not video or video['expire'] < fe.pubDate(): video = {'video': fe.id(), 'expire': fe.pubDate()} feed = { 'feed': fg.rss_str(), 'expire': datetime.datetime.now() + datetime.timedelta(hours=calls) } playlist_feed[playlist_name] = feed self.write(feed['feed']) self.finish() video = video['video'] mp3_file = 'audio/{}.mp3'.format(video) if playlist[1] == 'audio' and not os.path.exists( mp3_file) and video not in conversion_queue.keys(): conversion_queue[video] = { 'status': False, 'added': datetime.datetime.now() }
def render_atom(datasette, request, sql, columns, rows, database, table, query_name, view_name, data): from datasette.views.base import DatasetteError if not REQUIRED_COLUMNS.issubset(columns): raise DatasetteError( "SQL query must return columns {}".format( ", ".join(REQUIRED_COLUMNS)), status=400, ) fg = FeedGenerator() fg.generator( generator="Datasette", version=__version__, uri="https://github.com/simonw/datasette", ) fg.id(request.url) fg.link(href=request.url, rel="self") fg.updated(max(row["atom_updated"] for row in rows)) title = request.args.get("_feed_title", sql) if table: title += "/" + table if data.get("human_description_en"): title += ": " + data["human_description_en"] # If this is a canned query the configured title for that over-rides all others if query_name: try: title = datasette.metadata( database=database)["queries"][query_name]["title"] except (KeyError, TypeError): pass fg.title(title) clean_function = clean if query_name: # Check allow_unsafe_html_in_canned_queries plugin_config = datasette.plugin_config("datasette-atom") if plugin_config: allow_unsafe_html_in_canned_queries = plugin_config.get( "allow_unsafe_html_in_canned_queries") if allow_unsafe_html_in_canned_queries is True: clean_function = lambda s: s elif isinstance(allow_unsafe_html_in_canned_queries, dict): allowlist = allow_unsafe_html_in_canned_queries.get( database) or [] if query_name in allowlist: clean_function = lambda s: s # And the rows for row in reversed(rows): entry = fg.add_entry() entry.id(str(row["atom_id"])) if "atom_content_html" in columns: entry.content(clean_function(row["atom_content_html"]), type="html") elif "atom_content" in columns: entry.content(row["atom_content"], type="text") entry.updated(row["atom_updated"]) entry.title(str(row["atom_title"])) # atom_link is optional if "atom_link" in columns: entry.link(href=row["atom_link"]) if "atom_author_name" in columns and row["atom_author_name"]: author = { "name": row["atom_author_name"], } for key in ("uri", "email"): colname = "atom_author_{}".format(key) if colname in columns and row[colname]: author[key] = row[colname] entry.author(author) return Response( fg.atom_str(pretty=True), content_type="application/xml; charset=utf-8", status=200, )
def generate_atom_feeds(app): """ Generate archive pages for all posts, categories, tags, authors, and drafts. """ if not ablog.builder_support(app): return blog = Blog(app) url = blog.blog_baseurl if not url: return feeds = [ ( blog.posts, blog.blog_path, os.path.join(app.builder.outdir, blog.blog_path, feed_root + ".xml"), blog.blog_title, os_path_join(url, blog.blog_path, feed_root + ".xml"), feed_templates, ) for feed_root, feed_templates in blog.blog_feed_templates.items() ] if blog.blog_feed_archives: for header, catalog in [ (_("Posts by"), blog.author), (_("Posts from"), blog.location), (_("Posts in"), blog.language), (_("Posts in"), blog.category), (_("Posted in"), blog.archive), (_("Posts tagged"), blog.tags), ]: for coll in catalog: # skip collections containing only drafts if not len(coll): continue folder = os.path.join(app.builder.outdir, coll.path) if not os.path.isdir(folder): os.makedirs(folder) for feed_root, feed_templates in blog.blog_feed_templates.items(): feeds.append( ( coll, coll.path, os.path.join(folder, feed_root + ".xml"), blog.blog_title + " - " + header + " " + text_type(coll), os_path_join(url, coll.path, feed_root + ".xml"), feed_templates, ) ) # Config options feed_length = blog.blog_feed_length feed_fulltext = blog.blog_feed_fulltext for feed_posts, pagename, feed_path, feed_title, feed_url, feed_templates in feeds: feed = FeedGenerator() feed.id(blog.blog_baseurl) feed.title(feed_title) feed.link(href=url) feed.subtitle(blog.blog_feed_subtitle) feed.link(href=feed_url, rel="self") feed.language(app.config.language) feed.generator("ABlog", ablog.__version__, "https://ablog.readthedocs.org/") for i, post in enumerate(feed_posts): if feed_length and i == feed_length: break post_url = os_path_join(url, app.builder.get_target_uri(post.docname)) if post.section: post_url += "#" + post.section if blog.blog_feed_titles: content = None else: content = post.to_html(pagename, fulltext=feed_fulltext) feed_entry = feed.add_entry() feed_entry.id(post_url) feed_entry.link(href=post_url) feed_entry.author({"name": author.name for author in post.author}) feed_entry.pubDate(post.date.astimezone()) feed_entry.updated(post.update.astimezone()) for tag in post.tags: feed_entry.category( dict( term=tag.name.strip().replace(" ", ""), label=tag.label, ) ) # Entry values that support templates title = post.title summary = "".join(paragraph.astext() for paragraph in post.excerpt) template_values = {} for element in ("title", "summary", "content"): if element in feed_templates: template_values[element] = jinja2.Template(feed_templates[element]).render(**locals()) feed_entry.title(template_values.get("title", title)) summary = template_values.get("summary", summary) if summary: feed_entry.summary(summary) content = template_values.get("content", content) if content: feed_entry.content(content=content, type="html") parent_dir = os.path.dirname(feed_path) if not os.path.isdir(parent_dir): os.makedirs(parent_dir) with open(feed_path, "w", encoding="utf-8") as out: feed_str = feed.atom_str(pretty=True) out.write(feed_str.decode()) if 0: # this is to make the function a generator # and make work for Sphinx 'html-collect-pages' yield
def write_rss(self, audio=False): """Write podcast feeds to files.""" print("playlist self.info", flush=True) pp.pprint(self.info) prefix = "audio-" if audio else "" feed_url = self.controller.base_url + self.folder + '/' + prefix + 'podcast.xml' feedgen = FeedGenerator() feedgen.load_extension('podcast') feedgen.generator('Adafruit-Podcast') feedgen.id(feed_url) feedgen.title(self.info['title']) feedgen.subtitle(self.info['itunesSubtitle']) feedgen.author({'name': self.info['author']}) for category in self.info['categories']: feedgen.category(term=category) feedgen.webMaster(self.info['webMaster']) feedgen.managingEditor(self.info['managingEditor']) feedgen.link(href=feed_url, rel='self') # Link to a chosen URL as an alternate, if set. if 'htmlUrl' in self.info: feedgen.link(href=self.info['htmlUrl'], rel='alternate') else: # Otherwise link to the original YouTube playlist as an alternate: if isinstance(self.url, list): for url in self.url: feedgen.link(href=url, rel='alternate') else: feedgen.link(href=self.url, rel='alternate') feedgen.language('en') # feedgen.logo('http://ex.com/logo.jpg') # pylint: disable=no-member feedgen.podcast.itunes_category(self.info['itunesCategory']['text']) feedgen.podcast.itunes_subtitle(self.info['itunesSubtitle']) feedgen.podcast.itunes_summary(self.info['description']) feedgen.podcast.itunes_owner(email=self.info['itunesOwner']['email'], name=self.info['itunesOwner']['name']) feedgen.podcast.itunes_author(self.info['itunesOwner']['name']) feedgen.podcast.itunes_image(self.controller.base_url + self.folder + '/image.jpg') feedgen.podcast.itunes_explicit('clean') for vid in self.videos: print("vid:\n", flush=True) pp.pprint(vid) print("\n", flush=True) vid_filename = vid['_filename'].split('.')[0] + (".mp3" if audio else ".mp4") vid_url = self.video_url(vid_filename) # Size of enclosed file in bytes: vid_size = os.path.getsize(vid_filename) # Date of upload (from the youtube-dl JSON data) eastern = pytz.timezone('US/Eastern') vid_date = eastern.localize( datetime.datetime.strptime(vid['upload_date'], '%Y%m%d')) entry = feedgen.add_entry() entry.id(vid_url) entry.title(vid['fulltitle']) entry.published(vid_date) for category in vid['categories']: entry.category(term=category) entry.description(vid['description']) entry.enclosure(vid_url, str(vid_size), ('audio/mp3' if audio else 'video/mp4')) entry.podcast.itunes_image(self.controller.base_url + self.folder + '/image.jpg') entry.podcast.itunes_author(self.info['author']) entry.podcast.itunes_summary(vid['description']) entry.podcast.itunes_duration(vid['duration']) feedgen.rss_str(pretty=True) # Ensure output folder for this podcast exists: os.makedirs(os.path.join(self.controller.output_dir, self.folder), exist_ok=True) # Generate RSS file in output folder: feedgen.rss_file( os.path.join(self.controller.output_dir, self.folder, prefix + 'podcast.xml'))
def generate_atom_feeds(app): """ Generate archive pages for all posts, categories, tags, authors, and drafts. """ if not ablog.builder_support(app): return blog = Blog(app) url = blog.blog_baseurl if not url: return feed_path = os.path.join(app.builder.outdir, blog.blog_path, "atom.xml") feeds = [( blog.posts, blog.blog_path, feed_path, blog.blog_title, os_path_join(url, blog.blog_path, "atom.xml"), )] if blog.blog_feed_archives: for header, catalog in [ (_("Posts by"), blog.author), (_("Posts from"), blog.location), (_("Posts in"), blog.language), (_("Posts in"), blog.category), (_("Posted in"), blog.archive), (_("Posts tagged"), blog.tags), ]: for coll in catalog: # skip collections containing only drafts if not len(coll): continue folder = os.path.join(app.builder.outdir, coll.path) if not os.path.isdir(folder): os.makedirs(folder) feeds.append(( coll, coll.path, os.path.join(folder, "atom.xml"), blog.blog_title + " - " + header + " " + text_type(coll), os_path_join(url, coll.path, "atom.xml"), )) # Config options feed_length = blog.blog_feed_length feed_fulltext = blog.blog_feed_fulltext for feed_posts, pagename, feed_path, feed_title, feed_url in feeds: feed = FeedGenerator() feed.id("http://lernfunk.de/media/654321") feed.title(feed_title) feed.link(href=url) feed.subtitle(blog.blog_feed_subtitle) feed.link(href=feed_url) feed.language("en") feed.generator("ABlog", ablog.__version__, "https://ablog.readthedocs.org") for i, post in enumerate(feed_posts): if feed_length and i == feed_length: break post_url = os_path_join(url, app.builder.get_target_uri(post.docname)) if post.section: post_url += "#" + post.section if blog.blog_feed_titles: content = None else: content = post.to_html(pagename, fulltext=feed_fulltext) feed_entry = feed.add_entry() feed_entry.id(post_url) feed_entry.title(post.title) feed_entry.link(href=post_url) feed_entry.author({"name": author.name for author in post.author}) feed_entry.pubDate(post.date.astimezone()) feed_entry.updated(post.update.astimezone()) feed_entry.content(content=content, type="html") parent_dir = os.path.dirname(feed_path) if not os.path.isdir(parent_dir): os.makedirs(parent_dir) with open(feed_path, "w", encoding="utf-8") as out: feed_str = feed.atom_str(pretty=True) out.write(feed_str.decode()) if 0: # this is to make the function a generator # and make work for Sphinx 'html-collect-pages' yield
def setUp(self): fg = FeedGenerator() self.nsAtom = "http://www.w3.org/2005/Atom" self.nsRss = "http://purl.org/rss/1.0/modules/content/" self.feedId = 'http://lernfunk.de/media/654321' self.title = 'Some Testfeed' self.authorName = 'John Doe' self.authorMail = '*****@*****.**' self.author = {'name': self.authorName, 'email': self.authorMail} self.linkHref = 'http://example.com' self.linkRel = 'alternate' self.logo = 'http://ex.com/logo.jpg' self.subtitle = 'This is a cool feed!' self.link2Href = 'http://larskiesow.de/test.atom' self.link2Rel = 'self' self.language = 'en' self.categoryTerm = 'This category term' self.categoryScheme = 'This category scheme' self.categoryLabel = 'This category label' self.cloudDomain = 'example.com' self.cloudPort = '4711' self.cloudPath = '/ws/example' self.cloudRegisterProcedure = 'registerProcedure' self.cloudProtocol = 'SOAP 1.1' self.icon = "http://example.com/icon.png" self.contributor = {'name': "Contributor Name", 'uri': "Contributor Uri", 'email': 'Contributor email'} self.copyright = "The copyright notice" self.docs = 'http://www.rssboard.org/rss-specification' self.managingEditor = '*****@*****.**' self.rating = '(PICS-1.1 "http://www.classify.org/safesurf/" ' + \ '1 r (SS~~000 1))' self.skipDays = 'Tuesday' self.skipHours = 23 self.textInputTitle = "Text input title" self.textInputDescription = "Text input description" self.textInputName = "Text input name" self.textInputLink = "Text input link" self.ttl = 900 self.webMaster = '*****@*****.**' fg.id(self.feedId) fg.title(self.title) fg.author(self.author) fg.link(href=self.linkHref, rel=self.linkRel) fg.logo(self.logo) fg.subtitle(self.subtitle) fg.link(href=self.link2Href, rel=self.link2Rel) fg.language(self.language) fg.cloud(domain=self.cloudDomain, port=self.cloudPort, path=self.cloudPath, registerProcedure=self.cloudRegisterProcedure, protocol=self.cloudProtocol) fg.icon(self.icon) fg.category(term=self.categoryTerm, scheme=self.categoryScheme, label=self.categoryLabel) fg.contributor(self.contributor) fg.copyright(self.copyright) fg.docs(docs=self.docs) fg.managingEditor(self.managingEditor) fg.rating(self.rating) fg.skipDays(self.skipDays) fg.skipHours(self.skipHours) fg.textInput(title=self.textInputTitle, description=self.textInputDescription, name=self.textInputName, link=self.textInputLink) fg.ttl(self.ttl) fg.webMaster(self.webMaster) fg.updated('2017-02-05 13:26:58+01:00') fg.pubDate('2017-02-05 13:26:58+01:00') fg.generator('python-feedgen', 'x', uri='http://github.com/lkie...') fg.image(url=self.logo, title=self.title, link=self.link2Href, width='123', height='123', description='Example Inage') self.fg = fg
async def channel(request, channel_id, return_type='video'): log.info(f'Channel: {channel_id}') channel_name = [f'{channel_id}/{return_type}'] if channel_name[0] in channel_feed and channel_feed[ channel_name[0]]['expire'] > datetime.now(): return raw(channel_feed[channel_name[0]]['feed'], content_type='application/rss+xml') fg = None calls = 0 response = {'nextPageToken': ''} while 'nextPageToken' in response: next_page = response['nextPageToken'] payload = { 'part': 'snippet,contentDetails', 'maxResults': 50, 'channelId': channel_id, 'key': KEY, 'pageToken': next_page } response = json.loads( await get('https://www.googleapis.com/youtube/v3/activities', params=payload)) calls += 1 if 'error' in response: payload = { 'part': 'snippet', 'maxResults': 1, 'forUsername': channel_id, 'key': KEY } response = json.loads(await get( 'https://www.googleapis.com/youtube/v3/channels', params=payload)) channel_id = response['items'][0]['id'] channel_name.append(f'{channel_id}/{return_type}') payload = { 'part': 'snippet,contentDetails', 'maxResults': 50, 'channelId': channel_id, 'key': KEY, 'pageToken': next_page } response = json.loads(await get( 'https://www.googleapis.com/youtube/v3/activities', params=payload)) calls += 2 if not fg: fg = FeedGenerator() fg.load_extension('podcast') fg.generator('PodTube', __version__, 'https://github.com/aquacash5/PodTube') snippet = response['items'][0]['snippet'] if 'Private' in snippet['title']: continue icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) fg.title(snippet['title']) fg.id(f'http://{request.headers["host"]}{request.url}') fg.description(snippet['description'] or ' ') fg.author(name=snippet['channelTitle']) fg.image(snippet['thumbnails'][icon]['url']) fg.link(href=f'https://www.youtube.com/playlist?list={channel_id}') fg.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fg.podcast.itunes_summary(snippet['description']) fg.podcast.itunes_category('Technology', 'Podcasting') fg.updated(f'{str(datetime.utcnow())}Z') for item in response['items']: snippet = item['snippet'] if snippet['type'] != 'upload': continue current_video = item['contentDetails']['upload']['videoId'] log.debug(f'ChannelVideo: {current_video} {snippet["title"]}') fe = fg.add_entry() fe.title(snippet['title']) fe.id(current_video) icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) fe.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fe.updated(snippet['publishedAt']) if return_type == 'audio': fe.enclosure( url= f'http://{request.headers["host"]}/audio/{current_video}', type="audio/mpeg") else: fe.enclosure( url= f'http://{request.headers["host"]}/video/{current_video}', type="video/mp4") fe.author(name=snippet['channelTitle']) fe.podcast.itunes_author(snippet['channelTitle']) fe.podcast.itunes_author(snippet['channelTitle']) fe.pubdate(snippet['publishedAt']) fe.link(href=f'http://www.youtube.com/watch?v={current_video}', title=snippet['title']) fe.podcast.itunes_summary(snippet['description']) fe.description(snippet['description']) await sleep(0) feed = { 'feed': fg.rss_str(), 'expire': datetime.now() + timedelta(hours=calls) } for _name in channel_name: channel_feed[_name] = feed return raw(feed['feed'], content_type='application/rss+xml')
async def playlist(request, playlist_id, return_type='video'): log.info(f'Playlist: {playlist_id}') playlist_name = f'{playlist_id}/{return_type}' if playlist_name in playlist_feed and playlist_feed[playlist_name][ 'expire'] > datetime.now(): return raw(playlist_feed[playlist_name]['feed'], content_type='application/rss+xml') calls = 0 payload = {'part': 'snippet', 'id': playlist_id, 'key': KEY} log.debug('Downloaded Playlist Information') response = json.loads(await get( 'https://www.googleapis.com/youtube/v3/playlists', params=payload)) calls += 1 fg = FeedGenerator() fg.load_extension('podcast') fg.generator('PodTube', __version__, 'https://github.com/aquacash5/PodTube') snippet = response['items'][0]['snippet'] icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) fg.title(snippet['title']) fg.id(f'http://{request.headers["host"]}{request.url}') fg.description(snippet['description'] or ' ') fg.author(name=snippet['channelTitle']) fg.image(snippet['thumbnails'][icon]['url']) fg.link(href=f'https://www.youtube.com/playlist?list={playlist_id}') fg.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fg.podcast.itunes_summary(snippet['description']) fg.podcast.itunes_category('Technology', 'Podcasting') fg.updated(f'{str(datetime.utcnow())}Z') response = {'nextPageToken': ''} while 'nextPageToken' in response.keys(): payload = { 'part': 'snippet', 'maxResults': 50, 'playlistId': playlist_id, 'key': KEY, 'pageToken': response['nextPageToken'] } response = json.loads(await get( 'https://www.googleapis.com/youtube/v3/playlistItems', params=payload)) calls += 1 for item in response['items']: snippet = item['snippet'] current_video = snippet['resourceId']['videoId'] if 'Private' in snippet['title']: continue log.debug(f'PlaylistVideo: {current_video} {snippet["title"]}') fe = fg.add_entry() fe.title(snippet['title']) fe.id(current_video) icon = max(snippet['thumbnails'], key=lambda x: snippet['thumbnails'][x]['width']) fe.podcast.itunes_image(snippet['thumbnails'][icon]['url']) fe.updated(snippet['publishedAt']) if return_type == 'audio': fe.enclosure( url= f'http://{request.headers["host"]}/audio/{current_video}', type="audio/mpeg") else: fe.enclosure( url= f'http://{request.headers["host"]}/video/{current_video}', type="video/mp4") fe.author(name=snippet['channelTitle']) fe.podcast.itunes_author(snippet['channelTitle']) fe.podcast.itunes_author(snippet['channelTitle']) fe.pubdate(snippet['publishedAt']) fe.link(href='http://www.youtube.com/watch?v=' + current_video, title=snippet['title']) fe.podcast.itunes_summary(snippet['description']) fe.description(snippet['description']) await sleep(0) feed = { 'feed': fg.rss_str(), 'expire': datetime.now() + timedelta(hours=calls) } playlist_feed[playlist_name] = feed return raw(feed['feed'], content_type='application/rss+xml')
def from_activities(activities, actor=None, title=None, feed_url=None, home_page_url=None, hfeed=None): """Converts ActivityStreams activities to an RSS 2.0 feed. Args: activities: sequence of ActivityStreams activity dicts actor: ActivityStreams actor dict, the author of the feed title: string, the feed title feed_url: string, the URL for this RSS feed home_page_url: string, the home page URL hfeed: dict, parsed mf2 h-feed, if available Returns: unicode string with RSS 2.0 XML """ try: iter(activities) except TypeError: raise TypeError('activities must be iterable') if isinstance(activities, (dict, basestring)): raise TypeError('activities may not be a dict or string') fg = FeedGenerator() fg.id(feed_url) assert feed_url fg.link(href=feed_url, rel='self') if home_page_url: fg.link(href=home_page_url, rel='alternate') # TODO: parse language from lang attribute: # https://github.com/microformats/mf2py/issues/150 fg.language('en') fg.generator('granary', uri='https://granary.io/') hfeed = hfeed or {} actor = actor or {} image = util.get_url(hfeed, 'image') or util.get_url(actor, 'image') if image: fg.image(image) props = hfeed.get('properties') or {} content = microformats2.get_text(util.get_first(props, 'content', '')) summary = util.get_first(props, 'summary', '') desc = content or summary or '-' fg.description(desc) # required fg.title(title or util.ellipsize(desc)) # required latest = None enclosures = False for activity in activities: obj = activity.get('object') or activity if obj.get('objectType') == 'person': continue item = fg.add_entry() url = obj.get('url') item.id(obj.get('id') or url) item.link(href=url) item.guid(url, permalink=True) item.title(obj.get('title') or obj.get('displayName') or '-') # required content = microformats2.render_content( obj, include_location=True, render_attachments=False) or obj.get('summary') if content: item.content(content, type='CDATA') item.category( [{'term': t['displayName']} for t in obj.get('tags', []) if t.get('displayName') and t.get('verb') not in ('like', 'react', 'share')]) author = obj.get('author', {}) item.author({ 'name': author.get('displayName') or author.get('username'), 'uri': author.get('url'), }) published = obj.get('published') or obj.get('updated') if published: try: dt = mf2util.parse_datetime(published) if not isinstance(dt, datetime): dt = datetime.combine(dt, time.min) if not dt.tzinfo: dt = dt.replace(tzinfo=util.UTC) item.published(dt) if not latest or dt > latest: latest = dt except ValueError: # bad datetime string pass for att in obj.get('attachments', []): stream = util.get_first(att, 'stream') or att if not stream: continue url = stream.get('url') or '' mime = mimetypes.guess_type(url)[0] or '' if (att.get('objectType') in ENCLOSURE_TYPES or mime and mime.split('/')[0] in ENCLOSURE_TYPES): enclosures = True item.enclosure(url=url, type=mime, length='REMOVEME') # TODO: length (bytes) item.load_extension('podcast') duration = stream.get('duration') if duration: item.podcast.itunes_duration(duration) if enclosures: fg.load_extension('podcast') fg.podcast.itunes_author(actor.get('displayName') or actor.get('username')) if summary: fg.podcast.itunes_summary(summary) fg.podcast.itunes_explicit('no') fg.podcast.itunes_block(False) if latest: fg.lastBuildDate(latest) return fg.rss_str(pretty=True).decode('utf-8').replace(' length="REMOVEME"', '')
def from_activities(activities, actor=None, title=None, feed_url=None, home_page_url=None, hfeed=None): """Converts ActivityStreams activities to an RSS 2.0 feed. Args: activities: sequence of ActivityStreams activity dicts actor: ActivityStreams actor dict, the author of the feed title: string, the feed title feed_url: string, the URL for this RSS feed home_page_url: string, the home page URL hfeed: dict, parsed mf2 h-feed, if available Returns: unicode string with RSS 2.0 XML """ try: iter(activities) except TypeError: raise TypeError('activities must be iterable') if isinstance(activities, (dict, str)): raise TypeError('activities may not be a dict or string') fg = FeedGenerator() fg.id(feed_url) assert feed_url fg.link(href=feed_url, rel='self') if home_page_url: fg.link(href=home_page_url, rel='alternate') # TODO: parse language from lang attribute: # https://github.com/microformats/mf2py/issues/150 fg.language('en') fg.generator('granary', uri='https://granary.io/') hfeed = hfeed or {} actor = actor or {} image = (util.get_url(hfeed.get('properties', {}), 'photo') or util.get_url(actor, 'image')) if image: fg.image(image) props = hfeed.get('properties') or {} content = microformats2.get_text(util.get_first(props, 'content', '')) summary = util.get_first(props, 'summary', '') desc = content or summary or '-' fg.description(desc) # required fg.title(title or util.ellipsize(desc)) # required latest = None feed_has_enclosure = False for activity in activities: obj = activity.get('object') or activity if obj.get('objectType') == 'person': continue item = fg.add_entry() url = obj.get('url') id = obj.get('id') or url item.id(id) item.link(href=url) item.guid(url, permalink=True) # title (required) title = (obj.get('title') or obj.get('displayName') or util.ellipsize(obj.get('content', '-'))) # strip HTML tags title = util.parse_html(title).get_text('').strip() item.title(title) content = microformats2.render_content(obj, include_location=True, render_attachments=True, render_image=True) if not content: content = obj.get('summary') if content: item.content(content, type='CDATA') categories = [ { 'term': t['displayName'] } for t in obj.get('tags', []) if t.get('displayName') and t.get('verb') not in ('like', 'react', 'share') and t.get('objectType') not in ('article', 'person', 'mention') ] item.category(categories) author = obj.get('author', {}) author = { 'name': author.get('displayName') or author.get('username'), 'uri': author.get('url'), 'email': author.get('email') or '-', } item.author(author) published = obj.get('published') or obj.get('updated') if published and isinstance(published, str): try: dt = mf2util.parse_datetime(published) if not isinstance(dt, datetime): dt = datetime.combine(dt, time.min) if not dt.tzinfo: dt = dt.replace(tzinfo=util.UTC) item.published(dt) if not latest or dt > latest: latest = dt except ValueError: # bad datetime string pass item_has_enclosure = False for att in obj.get('attachments', []): stream = util.get_first(att, 'stream') or att if not stream: continue url = stream.get('url') or '' mime = mimetypes.guess_type(url)[0] or '' if (att.get('objectType') in ENCLOSURE_TYPES or mime and mime.split('/')[0] in ENCLOSURE_TYPES): if item_has_enclosure: logging.info( 'Warning: item %s already has an RSS enclosure, skipping additional enclosure %s', id, url) continue item_has_enclosure = feed_has_enclosure = True item.enclosure(url=url, type=mime, length=str(stream.get('size', ''))) item.load_extension('podcast') duration = stream.get('duration') if duration: item.podcast.itunes_duration(duration) if feed_has_enclosure: fg.load_extension('podcast') fg.podcast.itunes_author( actor.get('displayName') or actor.get('username')) if summary: fg.podcast.itunes_summary(summary) fg.podcast.itunes_explicit('no') fg.podcast.itunes_block(False) name = author.get('name') if name: fg.podcast.itunes_author(name) if image: fg.podcast.itunes_image(image) fg.podcast.itunes_category(categories) if latest: fg.lastBuildDate(latest) return fg.rss_str(pretty=True).decode('utf-8')
podcast_cover = 'https://conteudo.depois.cafe/depois-do-cafe-exclusivo.png' fg = FeedGenerator() fg.load_extension('podcast') fg.id('https://episodios.depois.cafe') fg.title('Depois do Café Exclusivo') fg.subtitle(podcast_description) fg.author({'name': podcast_author, 'email': podcast_email}) fg.link(href='https://episodios.depois.cafe', rel='alternate') fg.link(href='https://episodios.depois.cafe', rel='self', type='application/rss+xml') fg.logo(podcast_cover) fg.language('pt-BR') fg.generator('https://github.com/lkiesow/python-feedgen') fg.copyright(podcast_author) fg.podcast.itunes_author(podcast_author) fg.podcast.itunes_category('Technology') fg.podcast.itunes_subtitle(podcast_description) fg.podcast.itunes_summary(podcast_description) fg.podcast.itunes_owner(name=podcast_author, email=podcast_email) fg.podcast.itunes_explicit('no') fg.podcast.itunes_image(podcast_cover) for ep_extra in extras: entry = fg.add_entry() entry.title(ep_extra['title']) entry.id(ep_extra['mp3_link']) entry.link(href=ep_extra['mp3_link'], rel='alternate')
def setUp(self): fg = FeedGenerator() self.nsAtom = "http://www.w3.org/2005/Atom" self.nsRss = "http://purl.org/rss/1.0/modules/content/" self.feedId = 'http://lernfunk.de/media/654321' self.title = 'Some Testfeed' self.authorName = 'John Doe' self.authorMail = '*****@*****.**' self.author = {'name': self.authorName, 'email': self.authorMail} self.linkHref = 'http://example.com' self.linkRel = 'alternate' self.logo = 'http://ex.com/logo.jpg' self.subtitle = 'This is a cool feed!' self.link2Href = 'http://larskiesow.de/test.atom' self.link2Rel = 'self' self.language = 'en' self.categoryTerm = 'This category term' self.categoryScheme = 'This category scheme' self.categoryLabel = 'This category label' self.cloudDomain = 'example.com' self.cloudPort = '4711' self.cloudPath = '/ws/example' self.cloudRegisterProcedure = 'registerProcedure' self.cloudProtocol = 'SOAP 1.1' self.icon = "http://example.com/icon.png" self.contributor = { 'name': "Contributor Name", 'uri': "Contributor Uri", 'email': 'Contributor email' } self.copyright = "The copyright notice" self.docs = 'http://www.rssboard.org/rss-specification' self.managingEditor = '*****@*****.**' self.rating = '(PICS-1.1 "http://www.classify.org/safesurf/" ' + \ '1 r (SS~~000 1))' self.skipDays = 'Tuesday' self.skipHours = 23 self.textInputTitle = "Text input title" self.textInputDescription = "Text input description" self.textInputName = "Text input name" self.textInputLink = "Text input link" self.ttl = 900 self.webMaster = '*****@*****.**' fg.id(self.feedId) fg.title(self.title) fg.author(self.author) fg.link(href=self.linkHref, rel=self.linkRel) fg.logo(self.logo) fg.subtitle(self.subtitle) fg.link(href=self.link2Href, rel=self.link2Rel) fg.language(self.language) fg.cloud(domain=self.cloudDomain, port=self.cloudPort, path=self.cloudPath, registerProcedure=self.cloudRegisterProcedure, protocol=self.cloudProtocol) fg.icon(self.icon) fg.category(term=self.categoryTerm, scheme=self.categoryScheme, label=self.categoryLabel) fg.contributor(self.contributor) fg.copyright(self.copyright) fg.docs(docs=self.docs) fg.managingEditor(self.managingEditor) fg.rating(self.rating) fg.skipDays(self.skipDays) fg.skipHours(self.skipHours) fg.textInput(title=self.textInputTitle, description=self.textInputDescription, name=self.textInputName, link=self.textInputLink) fg.ttl(self.ttl) fg.webMaster(self.webMaster) fg.updated('2017-02-05 13:26:58+01:00') fg.pubDate('2017-02-05 13:26:58+01:00') fg.generator('python-feedgen', 'x', uri='http://github.com/lkie...') fg.image(url=self.logo, title=self.title, link=self.link2Href, width='123', height='123', description='Example Inage') self.fg = fg