class YoutubeFeed:
ydl_opts = {
'format': 'bestaudio/best',
'outtmpl': '%(id)s.%(ext)s',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}]
}
def __init__(self, name):
self.name = name
self.ydl = youtube_dl.YoutubeDL(self.ydl_opts)
self.fg = FeedGenerator()
self.fg.title(name)
self.fg.author({"name": "Youtube Audio Feed", "email": ""})
self.fg.link(href="http://www.foo.bar.baz.com", rel="alternate")
self.fg.description("Personalized Youtube audio feed")
self.fg.generator("")
self.fg.docs("")
def add_video(self, url):
info = self.ydl.extract_info(url, download=True)
entry = self.fg.add_entry()
entry.id(info['id'])
entry.title(info['title'])
entry.description(info['description'])
entry.enclosure(info['id'] + ".mp3", str(info['duration']), 'audio/mpeg')
def save(self):
self.fg.rss_file(name + '.xml')
def make_feedgenerator(conf):
feedgen = FeedGenerator()
feedgen.title('Lojban twitter feed in {lang}'.format(lang=conf['long']))
feedgen.description('Twitter Atom feed in {lang} about the constructed language Lojban'.format(lang=conf['long']))
feedgen.language(conf['short'])
feedgen.link(href='{}.atom.xml'.format(conf['short']))
feedgen.id('{}.atom.xml'.format(conf['short']))
feedgen.generator(generator='bano', version='0.0.0', uri='https://github.com/kyrias/bano')
return feedgen
async def create_rss(channel_alias: str, request: Request):
"""
Get posts from the channel and return rss-feed
"""
global channel_hash, client
channel_alias = channel_alias.lstrip('@')
private_channel = channel_alias[:8] == 'joinchat'
if private_channel:
private_hash = channel_alias[8:]
channel_alias = 't.me/joinchat/' + private_hash
try:
await client.start()
if channel_alias not in channel_hash:
if private_channel:
await client(ImportChatInviteRequest(private_hash))
channel = await client.get_entity(channel_alias)
ch_full = await client(GetFullChannelRequest(channel=channel))
username = channel.username or channel.id
channel_hash[channel_alias] = {
'username': username,
'title': channel.title,
'id': channel.id,
'about': ch_full.full_chat.about or str(username),
}
logging.info(f"Adding to the hash '{channel_alias}'")
with open('hash.pickle', 'wb') as f:
pickle.dump(channel_hash, f)
ch = channel_hash[channel_alias]
messages = [
m async for m in client.iter_messages(
ch['username'], limit=int(config['RSS']['RECORDS']))
]
except Exception as e:
warn = f"{str(e)}, request: '{channel_alias}'"
logging.warning(warn)
return warn
fg = FeedGenerator()
fg.title(f"{ch['title']} (@{ch['username']}, id:{ch['id']})")
fg.subtitle(ch['about'])
link = channel_alias if private_channel else f"t.me/s/{ch['username']}"
fg.link(href=f'https://{link}', rel='alternate')
fg.generator(config['RSS']['GENERATOR'])
fg.language(config['RSS']['LANGUAGE'])
for m in messages:
if not (config['RSS'].getboolean('SKIP_EMPTY') and not m.text):
fe = fg.add_entry(order='append')
link = 'https://t.me/' + ('c/' if private_channel else '')
fe.guid(guid=f"{link}{ch['username']}/{m.id}", permalink=True)
fe.content(markdown(m.text))
fe.published(m.date)
logging.debug(f"Successfully requested '{ch['username']}'")
return Response(content=fg.rss_str(), media_type='application/xml')
def _get_feed(query: Optional[ClassicAPIQuery] = None) -> FeedGenerator:
fg = FeedGenerator()
fg.generator("")
fg.register_extension("opensearch", OpenSearchExtension)
fg.register_extension("arxiv",
ArXivExtension,
ArXivEntryExtension,
rss=False)
if query:
if query.phrase is not None:
query_string = phrase_to_query_string(query.phrase)
else:
query_string = ""
if query.id_list:
id_list = ",".join(query.id_list)
else:
id_list = ""
fg.title(f"arXiv Query: {query.to_query_string()}")
# From perl documentation of the old site:
# search_id is calculated by taking SHA-1 digest of the query
# string. Digest is in bytes form and it's 20 bytes long. Then it's
# base64 encoded, but perls version returns only 27 characters -
# it omits the `=` sign at the end.
search_id = base64.b64encode(
hashlib.sha1(query.to_query_string().encode(
"utf-8")).digest()).decode("utf-8")[:-1]
fg.id(
url_for("classic_api.query").replace("/query",
f"/{search_id}"))
fg.link({
"href":
url_for(
"classic_api.query",
search_query=query_string,
start=query.page_start,
max_results=query.size,
id_list=id_list,
),
"type":
"application/atom+xml",
})
else:
# TODO: Discuss better defaults
fg.title("arXiv Search Results")
fg.id("https://arxiv.org/")
fg.updated(to_utc(datetime.utcnow()))
return fg
def __init__(
self,
id_: str,
links: List[dict],
title: str,
updated: Optional[str],
):
fg = FeedGenerator()
fg.id(id_)
fg.link(links)
fg.title(title)
fg.updated(updated)
fg.generator("")
self._feed_generator = fg
def build_feed(uri, path, title, description, articles):
uri = uri.rstrip("/")
blog_uri = f"{uri}/{path.split('/')[1]}"
feed = FeedGenerator()
feed.generator("Python Feedgen")
feed.title(title)
feed.description(description)
feed.link(href=f"{uri}{path}", rel="self")
for article in articles:
feed.add_entry(build_entry(article, blog_uri), order="append")
return feed
def notices_feed(feed_type):
if feed_type not in ["atom", "rss"]:
flask.abort(404)
url_root = flask.request.url_root
base_url = flask.request.base_url
feed = FeedGenerator()
feed.generator("Feedgen")
feed.id(url_root)
feed.copyright(
f"{datetime.now().year} Canonical Ltd. "
"Ubuntu and Canonical are registered trademarks of Canonical Ltd."
)
feed.title("Ubuntu security notices")
feed.description("Recent content on Ubuntu security notices")
feed.link(href=base_url, rel="self")
def feed_entry(notice, url_root):
_id = notice.id
title = f"{_id}: {notice.title}"
description = notice.details
published = notice.published
notice_path = flask.url_for(".notice", notice_id=notice.id).lstrip("/")
link = f"{url_root}{notice_path}"
entry = FeedEntry()
entry.id(link)
entry.title(title)
entry.description(description)
entry.link(href=link)
entry.published(f"{published} UTC")
entry.author({"name": "Ubuntu Security Team"})
return entry
notices = (
db_session.query(Notice)
.order_by(desc(Notice.published))
.limit(10)
.all()
)
for notice in notices:
feed.add_entry(feed_entry(notice, url_root), order="append")
payload = feed.atom_str() if feed_type == "atom" else feed.rss_str()
return flask.Response(payload, mimetype="text/xml")
def get(self, request, user_pk, feed_token):
user = get_object_or_404(User,
pk=user_pk,
feed_token=feed_token,
is_active=True)
with user.localized():
gen = FeedGenerator()
gen.generator(generator="MatShare",
version=__version__,
uri=settings.MS_URL)
gen.author(name="MatShare", email=settings.MS_CONTACT_EMAIL)
gen.link(href=settings.MS_ROOT_URL + reverse("user_dashboard"),
rel="alternate")
self.populate_feed(user, gen)
return HttpResponse(gen.atom_str(),
content_type="application/atom+xml")
def _build_feed(
self, blog_url, feed_url, feed_title, feed_description, articles
):
"""
Build the content for the feed
:blog_url: string blog url
:feed_url: string url
:feed_title: string title
:feed_description: string description
:param articles: Articles to create feed from
"""
feed = FeedGenerator()
feed.generator("Python Feedgen")
feed.title(feed_title)
feed.description(feed_description)
feed.link(href=feed_url, rel="self")
for article in articles:
title = article["title"]["rendered"]
slug = article["slug"]
author = article["_embedded"]["author"][0]
description = article["excerpt"]["rendered"]
content = article["content"]["rendered"]
published = f'{article["date_gmt"]} GMT'
updated = f'{article["modified_gmt"]} GMT'
link = f"{blog_url}/{slug}"
categories = []
if "wp:term" in article["_embedded"]:
for category in article["_embedded"]["wp:term"][1]:
categories.append(
dict(term=category["slug"], label=category["name"])
)
entry = FeedEntry()
entry.title(title)
entry.description(description)
entry.content(content)
entry.author(name=author["name"], email=author["name"])
entry.link(href=link)
entry.category(categories)
entry.published(published)
entry.updated(updated)
feed.add_entry(entry, order="append")
return feed
async def read_item(channel_alias: str, request: Request):
try:
channel = await client.get_entity(channel_alias)
except Exception as e:
logger.error('channel ERROR: {}'.format(e))
channel = None
return channel
else:
logger.info('Запросили канал {}'.format(channel.username))
fg = FeedGenerator()
fg.title('Recent posts from @' + channel_alias)
fg.author({'name': 'Nik Parotikov', 'email': '*****@*****.**'})
fg.link(href=str(request.url), rel='alternate')
fg.subtitle(channel.title + ' ' + 'https://t.me/' + channel.username +
' (id ' + str(channel.id) + ')')
fg.generator(
'http://tg2rss.prosto-tak.ru/ - Convert any Telegram channel to RSS feed'
)
fg.link(href=str(request.url), rel='self')
fg.language('ru')
# return channel
# messages = await client.get_messages(channel, 20)
# logger.info('сообщения из канала {}'.format(messages))
async for message in client.iter_messages(channel, 20):
# if hasattr(message.photo, 'file_reference'):
# logger.info('Сообщение {}'.format(message.photo.sizes[-1].location))
fe = fg.add_entry()
fe.guid(guid='https://t.me/' + channel_alias + '/' + str(message.id),
permalink=True)
# fe.link(href='https://t.me/' + channel_alias + '/' + str(message.id))
# fe.title(message.text)
# fe.description(message.text)
fe.content(markdown2.markdown(message.text))
# fe.enclosure()
fe.published(message.date)
# rssfeed = fg.rss_str(pretty=True)
rssfeed = fg.rss_str()
# return rssfeed.decode()
return Response(content=rssfeed, media_type="application/xml")
def rss():
fg = FeedGenerator()
fg.title('CloudWalk DevSecOps test')
fg.description('A RSS Feed for HTTP and TCP service')
fg.docs('')
fg.generator('')
fg.link(href='http://example.com')
with open('log.txt') as f:
for line in f.readlines():
info = line.replace('\n', '').split(';')
fe = fg.add_entry()
fe.title(f"{info[1]}")
fe.pubDate(f"{info[0]} GTM-3")
fe.description(f"server: {info[2]} port:{info[3]}")
response = make_response(fg.rss_str())
response.headers.set('Content-type', 'application/rss+xml')
return response
def user_rss(service, id):
cursor = get_cursor()
query = "SELECT * FROM posts WHERE \"user\" = %s AND service = %s "
params = (id, service)
query += "ORDER BY added desc "
query += "LIMIT 10"
cursor.execute(query, params)
results = cursor.fetchall()
cursor3 = get_cursor()
query3 = "SELECT * FROM lookup WHERE id = %s AND service = %s"
params3 = (id, service)
cursor3.execute(query3, params3)
results3 = cursor.fetchall()
name = results3[0]['name'] if len(results3) > 0 else ''
fg = FeedGenerator()
fg.title(name)
fg.description('Feed for posts from ' + name + '.')
fg.id(f'http://{request.headers.get("host")}/{service}/user/{id}')
fg.link(href=f'http://{request.headers.get("host")}/{service}/user/{id}')
fg.generator(generator='Kemono')
fg.ttl(ttl=40)
for post in results:
fe = fg.add_entry()
fe.title(post['title'])
fe.id(
f'http://{request.headers.get("host")}/{service}/user/{id}/post/{post["id"]}'
)
fe.link(
href=
f'http://{request.headers.get("host")}/{service}/user/{id}/post/{post["id"]}'
)
fe.content(content=post["content"])
fe.pubDate(pytz.utc.localize(post["added"]))
response = make_response(fg.atom_str(pretty=True), 200)
response.headers['Content-Type'] = 'application/rss+xml'
return response
def gen_feed(title, base_url, feed_url, num_cutoff, entries):
fg = FeedGenerator()
if feed_url:
fg.id(feed_url)
fg.link(href=feed_url, rel="self")
else:
fg.id(base_url)
fg.title(title)
fg.generator(generator="dir2feed", uri="https://github.com/pR0Ps/dir2feed")
all_entries = sorted(entries, key=attrgetter("date", "title"))
for e in all_entries[-max(0, num_cutoff or 0):]:
fe = fg.add_entry()
fe.id(e.url)
fe.title(e.title)
fe.link(rel="alternate", href=e.url)
fe.updated(e.date)
for l in e.links():
fe.link(**l)
fe.summary(summary=e.summary(), type="html")
return fg
def build_feed(self, commits: List[Commit]):
log.info("build feed page %d" % len(commits))
feed = FeedGenerator()
feed.id("")
feed.title("AWS API Changes")
feed.author({
"name": "AWSPIChanges",
"email": "https://github.com/awslabs/aws-sdk-api-changes",
})
feed.link(href=self.site_url, rel="alternate")
feed.link(href="%s/feed/" % self.site_url, rel="self")
feed.description("AWS API ChangeLog")
feed.language("en-US")
feed.generator("artisan-sdk-gitops")
feed.image(
url=
"https://a0.awsstatic.com/main/images/logos/aws_logo_smile_179x109.png"
) # noqa
for c in commits:
for s in c.service_changes:
fe = feed.add_entry(order="append")
fe.title("{} - {}{}methods".format(
s.title,
s.count_new and "%d new " % s.count_new or "",
s.count_updated and "%d updated " % s.count_updated or "",
))
fe.id("{}-{}".format(c.id, s.name))
fe.description(s.change_log)
fe.link({
"href":
self.link("archive/changes/%s-%s.html" %
(c.id[:6], s.name))
})
fe.published(c.created)
self.render_page(
"feed/feed.rss",
force=True,
content=feed.rss_str(pretty=True).decode("utf8"),
)
def display_feed(feed_name):
if feed_name not in config['feeds']:
abort(404)
f = Feed(config['feeds'][feed_name])
f.load()
f.parse()
fg = FeedGenerator()
fg.generator(**_generator)
fg.id(request.base_url)
fg.link(
href=request.base_url,
rel='self',
)
fg.title(f.properties.get('title', feed_name))
fg.author(name=f.properties.get('author', ''))
fg.updated(
timezone.localize(
_round_date(max([e['updated'] for e in f.entries]),
config.get('date_rounding', None))))
for entry in f.entries:
fe = fg.add_entry()
fe.id(entry['url'])
fe.title(entry['title'])
fe.link(href=entry['url'])
fe.updated(
timezone.localize(
_round_date(entry['updated'],
config.get('date_rounding', None))))
fe.content(entry['content'])
atomfeed = fg.atom_str()
resp = make_response(atomfeed)
resp.headers['content-type'] = 'application/xml'
return resp
def render(self, data, accepted_media_type=None, renderer_context=None):
"""
Renders *data* into serialized XML.
"""
if data is None:
return ''
fg = FeedGenerator()
feed_url = settings.API_URL + reverse('api:veroeffentlichung-list')
feed_title = 'OffeneGesetze.de'
feed_description = 'Feed für Veröffentlichungen des Bundesgesetzblatts'
fg.id(feed_url)
fg.title(feed_title)
fg.subtitle(feed_description)
fg.link(href=feed_url, rel='alternate')
fg.logo('https://offenegesetze.de/apple-touch-icon.png')
fg.link(href=feed_url + '?format=rss', rel='self')
fg.language('de')
fg.generator('')
if not isinstance(data, list):
data = [data]
results = reversed(data[0].get('results', []))
for item in results:
fe = fg.add_entry()
fe.id('%s/%s' % (settings.SITE_URL, item['id']))
fe.pubDate(item['date'])
fe.title(item['title'])
fe.link({'href': item['url']})
if 'content' in item:
fe.description(item['content'] if isinstance(
item['content'], str) else ''.join(item['content']))
return fg.rss_str(pretty=True)
def to_atom(self, deployment_uri, use_summary=False):
fg = FeedGenerator()
fg.load_extension('podcast')
fg.id(deployment_uri)
fg.title(self.title)
fg.author({"name": "Atomizer/1.0"})
fg.generator("Atomizer")
fg.link(href=self.uri, rel='alternate', type="text/html")
fg.link(href=deployment_uri, rel='self')
fg.description(self.title)
for entry in self.entries:
feed_item = fg.add_entry(order='append')
feed_item.id(entry.link)
feed_item.title(entry.title)
feed_item.updated(self.ensure_tz_utc(entry.date))
author = {"name": entry.author}
if entry.author_uri:
author['uri'] = entry.author_uri
feed_item.author(author)
feed_item.published(self.ensure_tz_utc(entry.date))
feed_item.link(link={
"href": entry.link,
"rel": "alternate",
"type": "text/html"
})
if entry.enclosures:
for enclosure in entry.enclosures:
if enclosure.get("href"):
feed_item.enclosure(url=enclosure.get("href"),
length=enclosure.get("length", 0),
type=enclosure.get("type", ""))
if use_summary and entry.summary:
feed_item.summary(entry.summary_html)
feed_item.content(content=entry.content_html, type="html")
return fg.atom_str(pretty=True)
class SteamCommentsRss:
def __init__(self, app_id):
self.app_id = app_id
self.steam_comments = CommentsFeed(app_id)
self.feed = FeedGenerator()
self.feed.id(
"https://steamcommunity.com/games/{app_id}/allnews/".format(
app_id=self.app_id))
self.feed.title("PULSAR: Lost Colony Developer Comments on Steam")
self.feed.link(href="https://pulsar.wiki/leafytracker/")
self.feed.description(
"Comments by leafygamesdev on PULSAR news articles.")
self.feed.language("en")
self.feed.generator("https://pulsar.wiki/leafytracker/"
) # TODO: Automate project name and version
def append_comments(self, news_ids, user_ids):
news_ids = set(int(x) for x in news_ids)
user_ids = set(int(x) for x in user_ids)
for nid in news_ids:
for comment in self.steam_comments.get(nid, user_ids):
entry = self.feed.add_entry()
entry.id(str(comment.cid))
entry.link({"href": comment.url})
entry.title("{} commented on {}".format(
comment.author.name, comment.title))
entry.author({"name": comment.author.name})
entry.published(comment.datetime)
entry.content(comment.body)
def to_atom(self, output_path, pretty=True):
return self.feed.atom_file(output_path, pretty=pretty)
def to_rss(self, output_path, pretty=True):
return self.feed.rss_file(output_path, pretty=pretty)
class Feed:
def __init__(self,
url: str,
name: str,
email: str,
title: str = None,
generator: str = None,
generator_version: str = None,
logo: str = None,
icon: str = None,
description: str = None,
language: str = None) -> None:
self.name = name
self.email = email
self.fg = FeedGenerator()
self.fg.id(url + "feed.atom")
self.fg.link(href=url + "feed.xml", rel="self")
self.fg.link(href=url, rel="alternate")
self.fg.author(name=name, email=email)
self.fg.contributor(name=name, email=email)
self.fg.managingEditor(email)
self.fg.webMaster(email)
self.fg.title(title)
self.fg.generator(generator=generator, version=generator_version)
self.fg.logo(logo)
self.fg.icon(icon)
self.fg.description(description)
self.fg.language(language)
def add(self, article: Article) -> None:
feed_entry = self.fg.add_entry()
feed_entry.id(article.url)
feed_entry.title(article.title)
feed_entry.link(href=article.url)
feed_entry.guid(guid=article.url, permalink=True)
feed_entry.author(name=self.name, email=self.email)
feed_entry.summary(article.description or article.snippet)
feed_entry.content(content=article.content, type="CDATA")
feed_entry.published(article.date)
if article.date:
feed_entry.published(article.date)
feed_entry.updated(article.date)
else:
epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
feed_entry.published(epoch)
feed_entry.updated(epoch)
def add_from_blog(self, url: str) -> None:
blog = Blog(url)
if not self.fg.title():
self.fg.title(blog.title)
for article in blog.articles:
self.add(article)
def atom(self) -> bytes:
return self.fg.atom_str(pretty=True)
def rss(self) -> bytes:
return self.fg.rss_str(pretty=True)
def atom_file(self, filename: str = "feed.atom") -> None:
self.fg.atom_file(filename, pretty=True)
def rss_file(self, filename: str = "feed.xml") -> None:
self.fg.rss_file(filename, pretty=True)
def get(self, mc, db, pkey):
def check_encoding(string):
data = string
if string is not unicode:
data = unicode(string)
return ud.normalize('NFKD', data).encode('ascii', 'xmlcharrefreplace')
try:
# host URL
urlparts = request.urlparts
host_url = '%s://%s/feeds/%s' % (urlparts.scheme, urlparts.netloc, pkey)
# get feed data
cfg = self._app.config
obj = FeedService.get_feed_activities(db, mc, cfg, pkey)
activities = obj['activities']
user_id = obj['user_id']
# main element
channel = FeedGenerator()
channel.title('Plus Channel feed')
channel.description('Google+ List of Activities for %s' % obj['name'])
channel.generator('Plus Channel %s' % cfg.get('main.version'))
channel.id('https://plus.google.com/' + user_id)
channel.link(href=host_url, rel='self')
channel.docs('')
if 'photo_url' in obj and obj['photo_url'] is not None:
channel.image(url=obj['photo_url'],
title='Plus Channel feed',
link='https://plus.google.com/' + user_id,
width=str(cfg.get('feed.photo_size.database')),
height=str(cfg.get('feed.photo_size.database')))
# additional namespaces
channel.register_extension('media', MediaExtension, MediaEntryExtension)
channel.register_extension('geo', GeoExtension, GeoEntryExtension)
# compose items
h = HTMLParser.HTMLParser()
for activity in activities:
title = activity['title']
content = activity['content']
url = activity['url']
# check content
if content is None or content == title:
content = ''
# check title
if title is None:
title = 'notitle'
# reformat strings
title = h.unescape(title)
title = re.sub('<[^>]*>', '', title)
title = escape(title)
content = h.unescape(content)
content = re.sub('<[^>]*>', '', content)
content = escape(content)
# log activity
logging.debug('--- activity ---')
logging.debug(title)
logging.debug(content)
logging.debug(url)
logging.debug('----------------')
# create item
item = channel.add_entry()
item.title(check_encoding(title))
item.pubdate(activity['datePublished'])
# process content
c_content = check_encoding(content)
item.description(c_content)
item.content(content=c_content, type='CDATA')
# # check image presence
if 'imageUrl' in activity and activity['imageUrl'] != '':
item.media.media_thumbnail_url(activity['imageUrl'])
# check size
if 'imageWidth' in activity and 'imageHeight' in activity:
item.media.media_thumbnail_width(activity['imageWidth'])
item.media.media_thumbnail_height(activity['imageHeight'])
# check coordinates
if activity['hasCoordinates']:
item.geo.geo_lat(activity['latitude'])
item.geo.geo_long(activity['longitude'])
# check link
if url is None or url == '':
url = activity['url']
item.link(href=escape(url), rel='alternate')
item.guid(escape(activity['id']))
# return created feed
response.set_header('content-type', 'application/rss+xml; charset=utf-8')
out = channel.rss_str(pretty=True)
del channel, activities, user_id, obj
return out
except FeedService.FeedNotFoundException:
abort(404)
except FeedService.UserIdNotFoundException:
abort(410)
def get(self, channel):
channel = channel.split('/')
if len(channel) < 2:
channel.append('video')
channel_name = ['/'.join(channel)]
self.set_header('Content-type', 'application/rss+xml')
if channel_name[0] in channel_feed and channel_feed[
channel_name[0]]['expire'] > datetime.datetime.now():
self.write(channel_feed[channel_name[0]]['feed'])
self.finish()
return
fg = None
video = None
calls = 0
response = {'nextPageToken': ''}
while 'nextPageToken' in response.keys():
next_page = response['nextPageToken']
payload = {
'part': 'snippet,contentDetails',
'maxResults': 50,
'channelId': channel[0],
'key': key,
'pageToken': next_page
}
request = requests.get(
'https://www.googleapis.com/youtube/v3/activities',
params=payload)
calls += 1
if request.status_code != 200:
payload = {
'part': 'snippet',
'maxResults': 1,
'forUsername': channel[0],
'key': key
}
request = requests.get(
'https://www.googleapis.com/youtube/v3/channels',
params=payload)
response = request.json()
channel[0] = response['items'][0]['id']
channel_name.append('/'.join(channel))
payload = {
'part': 'snippet,contentDetails',
'maxResults': 50,
'channelId': channel[0],
'key': key,
'pageToken': next_page
}
request = requests.get(
'https://www.googleapis.com/youtube/v3/activities',
params=payload)
calls += 2
response = request.json()
if request.status_code == 200:
logging.debug('Downloaded Channel Information')
else:
logging.error('Error Downloading Channel: %s', request.reason)
self.send_error(reason='Error Downloading Channel')
return
if not fg:
fg = FeedGenerator()
fg.load_extension('podcast')
fg.generator('PodTube (python-feedgen)', __version__,
'https://github.com/aquacash5/PodTube')
for item in response['items']:
if item['snippet']['type'] != 'upload':
continue
elif 'Private' in item['snippet']['title']:
continue
else:
snippet = item['snippet']
break
logging.info('Channel: %s (%s)', channel[0],
snippet['channelTitle'])
icon = max(snippet['thumbnails'],
key=lambda x: snippet['thumbnails'][x]['width'])
fg.title(snippet['channelTitle'])
fg.id('http://' + self.request.host + self.request.uri)
fg.description(snippet['description'] or ' ')
fg.author(name='Podtube',
email='*****@*****.**',
uri='https://github.com/aquacash5/PodTube')
fg.podcast.itunes_author(snippet['channelTitle'])
fg.image(snippet['thumbnails'][icon]['url'])
fg.link(href=f'http://youtube.com/channel/{channel}',
rel='self')
fg.language('en-US')
fg.podcast.itunes_image(snippet['thumbnails'][icon]['url'])
fg.podcast.itunes_explicit('no')
fg.podcast.itunes_owner(name='Podtube',
email='*****@*****.**')
fg.podcast.itunes_summary(snippet['description'])
fg.podcast.itunes_category(cat='Technology')
fg.updated(str(datetime.datetime.utcnow()) + 'Z')
for item in response['items']:
snippet = item['snippet']
if snippet['type'] != 'upload':
continue
if 'private' in snippet['title'].lower():
continue
current_video = item['contentDetails']['upload']['videoId']
logging.debug('ChannelVideo: %s (%s)', current_video,
snippet['title'])
fe = fg.add_entry()
fe.title(snippet['title'])
fe.id(current_video)
icon = max(snippet['thumbnails'],
key=lambda x: snippet['thumbnails'][x]['width'])
fe.podcast.itunes_image(snippet['thumbnails'][icon]['url'])
fe.updated(snippet['publishedAt'])
if channel[1] == 'video':
fe.enclosure(
url=f'http://{self.request.host}/video/{current_video}',
type="video/mp4")
elif channel[1] == 'audio':
fe.enclosure(
url=f'http://{self.request.host}/audio/{current_video}',
type="audio/mpeg")
fe.author(name=snippet['channelTitle'])
fe.podcast.itunes_author(snippet['channelTitle'])
fe.pubDate(snippet['publishedAt'])
fe.link(href=f'http://www.youtube.com/watch?v={current_video}',
title=snippet['title'])
fe.podcast.itunes_summary(snippet['description'])
fe.description(snippet['description'])
if not video or video['expire'] < fe.pubDate():
video = {'video': fe.id(), 'expire': fe.pubDate()}
feed = {
'feed': fg.rss_str(),
'expire': datetime.datetime.now() + datetime.timedelta(hours=calls)
}
for chan in channel_name:
channel_feed[chan] = feed
self.write(feed['feed'])
self.finish()
video = video['video']
mp3_file = 'audio/{}.mp3'.format(video)
if channel[1] == 'audio' and not os.path.exists(
mp3_file) and video not in conversion_queue.keys():
conversion_queue[video] = {
'status': False,
'added': datetime.datetime.now()
}
def get(self, playlist):
playlist = playlist.split('/')
if len(playlist) < 2:
playlist.append('video')
playlist_name = '/'.join(playlist)
self.set_header('Content-type', 'application/rss+xml')
if playlist_name in playlist_feed and playlist_feed[playlist_name][
'expire'] > datetime.datetime.now():
self.write(playlist_feed[playlist_name]['feed'])
self.finish()
return
calls = 0
payload = {'part': 'snippet', 'id': playlist[0], 'key': key}
request = requests.get(
'https://www.googleapis.com/youtube/v3/playlists', params=payload)
calls += 1
response = request.json()
if request.status_code == 200:
logging.debug('Downloaded Playlist Information')
else:
logging.error('Error Downloading Playlist: %s', request.reason)
self.send_error(reason='Error Downloading Playlist')
return
fg = FeedGenerator()
fg.load_extension('podcast')
fg.generator('PodTube (python-feedgen)', __version__,
'https://github.com/aquacash5/PodTube')
snippet = response['items'][0]['snippet']
icon = max(snippet['thumbnails'],
key=lambda x: snippet['thumbnails'][x]['width'])
logging.info('Playlist: %s (%s)', playlist[0], snippet['title'])
fg.title(snippet['title'])
fg.id('http://' + self.request.host + self.request.uri)
fg.description(snippet['description'] or ' ')
fg.author(name='Podtube',
email='*****@*****.**',
uri='https://github.com/aquacash5/PodTube')
fg.podcast.itunes_author(snippet['channelTitle'])
fg.image(snippet['thumbnails'][icon]['url'])
fg.link(href=f'http://youtube.com/playlist/?list={playlist}',
rel='self')
fg.language('en-US')
fg.podcast.itunes_image(snippet['thumbnails'][icon]['url'])
fg.podcast.itunes_explicit('no')
fg.podcast.itunes_owner(name='Podtube',
email='*****@*****.**')
fg.podcast.itunes_summary(snippet['description'])
fg.podcast.itunes_category(cat='Technology')
fg.updated(str(datetime.datetime.utcnow()) + 'Z')
video = None
response = {'nextPageToken': ''}
while 'nextPageToken' in response.keys():
payload = {
'part': 'snippet',
'maxResults': 50,
'playlistId': playlist[0],
'key': key,
'pageToken': response['nextPageToken']
}
request = requests.get(
'https://www.googleapis.com/youtube/v3/playlistItems',
params=payload)
calls += 1
response = request.json()
if request.status_code == 200:
logging.debug('Downloaded Playlist Information')
else:
logging.error('Error Downloading Playlist: %s', request.reason)
self.send_error(reason='Error Downloading Playlist Items')
return
for item in response['items']:
snippet = item['snippet']
current_video = snippet['resourceId']['videoId']
if 'Private' in snippet['title']:
continue
logging.debug('PlaylistVideo: %s (%s)', current_video,
snippet['title'])
fe = fg.add_entry()
fe.title(snippet['title'])
fe.id(current_video)
icon = max(snippet['thumbnails'],
key=lambda x: snippet['thumbnails'][x]['width'])
fe.podcast.itunes_image(snippet['thumbnails'][icon]['url'])
fe.updated(snippet['publishedAt'])
if playlist[1] == 'video':
fe.enclosure(
url=f'http://{self.request.host}/video/{current_video}',
type="video/mp4")
elif playlist[1] == 'audio':
fe.enclosure(
url=f'http://{self.request.host}/audio/{current_video}',
type="audio/mpeg")
fe.author(name=snippet['channelTitle'])
fe.podcast.itunes_author(snippet['channelTitle'])
fe.pubDate(snippet['publishedAt'])
fe.link(href=f'http://www.youtube.com/watch?v={current_video}',
title=snippet['title'])
fe.podcast.itunes_summary(snippet['description'])
fe.description(snippet['description'])
if not video or video['expire'] < fe.pubDate():
video = {'video': fe.id(), 'expire': fe.pubDate()}
feed = {
'feed': fg.rss_str(),
'expire': datetime.datetime.now() + datetime.timedelta(hours=calls)
}
playlist_feed[playlist_name] = feed
self.write(feed['feed'])
self.finish()
video = video['video']
mp3_file = 'audio/{}.mp3'.format(video)
if playlist[1] == 'audio' and not os.path.exists(
mp3_file) and video not in conversion_queue.keys():
conversion_queue[video] = {
'status': False,
'added': datetime.datetime.now()
}
def render_atom(datasette, request, sql, columns, rows, database, table,
query_name, view_name, data):
from datasette.views.base import DatasetteError
if not REQUIRED_COLUMNS.issubset(columns):
raise DatasetteError(
"SQL query must return columns {}".format(
", ".join(REQUIRED_COLUMNS)),
status=400,
)
fg = FeedGenerator()
fg.generator(
generator="Datasette",
version=__version__,
uri="https://github.com/simonw/datasette",
)
fg.id(request.url)
fg.link(href=request.url, rel="self")
fg.updated(max(row["atom_updated"] for row in rows))
title = request.args.get("_feed_title", sql)
if table:
title += "/" + table
if data.get("human_description_en"):
title += ": " + data["human_description_en"]
# If this is a canned query the configured title for that over-rides all others
if query_name:
try:
title = datasette.metadata(
database=database)["queries"][query_name]["title"]
except (KeyError, TypeError):
pass
fg.title(title)
clean_function = clean
if query_name:
# Check allow_unsafe_html_in_canned_queries
plugin_config = datasette.plugin_config("datasette-atom")
if plugin_config:
allow_unsafe_html_in_canned_queries = plugin_config.get(
"allow_unsafe_html_in_canned_queries")
if allow_unsafe_html_in_canned_queries is True:
clean_function = lambda s: s
elif isinstance(allow_unsafe_html_in_canned_queries, dict):
allowlist = allow_unsafe_html_in_canned_queries.get(
database) or []
if query_name in allowlist:
clean_function = lambda s: s
# And the rows
for row in reversed(rows):
entry = fg.add_entry()
entry.id(str(row["atom_id"]))
if "atom_content_html" in columns:
entry.content(clean_function(row["atom_content_html"]),
type="html")
elif "atom_content" in columns:
entry.content(row["atom_content"], type="text")
entry.updated(row["atom_updated"])
entry.title(str(row["atom_title"]))
# atom_link is optional
if "atom_link" in columns:
entry.link(href=row["atom_link"])
if "atom_author_name" in columns and row["atom_author_name"]:
author = {
"name": row["atom_author_name"],
}
for key in ("uri", "email"):
colname = "atom_author_{}".format(key)
if colname in columns and row[colname]:
author[key] = row[colname]
entry.author(author)
return Response(
fg.atom_str(pretty=True),
content_type="application/xml; charset=utf-8",
status=200,
)
def generate_atom_feeds(app):
"""
Generate archive pages for all posts, categories, tags, authors, and
drafts.
"""
if not ablog.builder_support(app):
return
blog = Blog(app)
url = blog.blog_baseurl
if not url:
return
feeds = [
(
blog.posts,
blog.blog_path,
os.path.join(app.builder.outdir, blog.blog_path, feed_root + ".xml"),
blog.blog_title,
os_path_join(url, blog.blog_path, feed_root + ".xml"),
feed_templates,
)
for feed_root, feed_templates in blog.blog_feed_templates.items()
]
if blog.blog_feed_archives:
for header, catalog in [
(_("Posts by"), blog.author),
(_("Posts from"), blog.location),
(_("Posts in"), blog.language),
(_("Posts in"), blog.category),
(_("Posted in"), blog.archive),
(_("Posts tagged"), blog.tags),
]:
for coll in catalog:
# skip collections containing only drafts
if not len(coll):
continue
folder = os.path.join(app.builder.outdir, coll.path)
if not os.path.isdir(folder):
os.makedirs(folder)
for feed_root, feed_templates in blog.blog_feed_templates.items():
feeds.append(
(
coll,
coll.path,
os.path.join(folder, feed_root + ".xml"),
blog.blog_title + " - " + header + " " + text_type(coll),
os_path_join(url, coll.path, feed_root + ".xml"),
feed_templates,
)
)
# Config options
feed_length = blog.blog_feed_length
feed_fulltext = blog.blog_feed_fulltext
for feed_posts, pagename, feed_path, feed_title, feed_url, feed_templates in feeds:
feed = FeedGenerator()
feed.id(blog.blog_baseurl)
feed.title(feed_title)
feed.link(href=url)
feed.subtitle(blog.blog_feed_subtitle)
feed.link(href=feed_url, rel="self")
feed.language(app.config.language)
feed.generator("ABlog", ablog.__version__, "https://ablog.readthedocs.org/")
for i, post in enumerate(feed_posts):
if feed_length and i == feed_length:
break
post_url = os_path_join(url, app.builder.get_target_uri(post.docname))
if post.section:
post_url += "#" + post.section
if blog.blog_feed_titles:
content = None
else:
content = post.to_html(pagename, fulltext=feed_fulltext)
feed_entry = feed.add_entry()
feed_entry.id(post_url)
feed_entry.link(href=post_url)
feed_entry.author({"name": author.name for author in post.author})
feed_entry.pubDate(post.date.astimezone())
feed_entry.updated(post.update.astimezone())
for tag in post.tags:
feed_entry.category(
dict(
term=tag.name.strip().replace(" ", ""),
label=tag.label,
)
)
# Entry values that support templates
title = post.title
summary = "".join(paragraph.astext() for paragraph in post.excerpt)
template_values = {}
for element in ("title", "summary", "content"):
if element in feed_templates:
template_values[element] = jinja2.Template(feed_templates[element]).render(**locals())
feed_entry.title(template_values.get("title", title))
summary = template_values.get("summary", summary)
if summary:
feed_entry.summary(summary)
content = template_values.get("content", content)
if content:
feed_entry.content(content=content, type="html")
parent_dir = os.path.dirname(feed_path)
if not os.path.isdir(parent_dir):
os.makedirs(parent_dir)
with open(feed_path, "w", encoding="utf-8") as out:
feed_str = feed.atom_str(pretty=True)
out.write(feed_str.decode())
if 0:
# this is to make the function a generator
# and make work for Sphinx 'html-collect-pages'
yield
def write_rss(self, audio=False):
"""Write podcast feeds to files."""
print("playlist self.info", flush=True)
pp.pprint(self.info)
prefix = "audio-" if audio else ""
feed_url = self.controller.base_url + self.folder + '/' + prefix + 'podcast.xml'
feedgen = FeedGenerator()
feedgen.load_extension('podcast')
feedgen.generator('Adafruit-Podcast')
feedgen.id(feed_url)
feedgen.title(self.info['title'])
feedgen.subtitle(self.info['itunesSubtitle'])
feedgen.author({'name': self.info['author']})
for category in self.info['categories']:
feedgen.category(term=category)
feedgen.webMaster(self.info['webMaster'])
feedgen.managingEditor(self.info['managingEditor'])
feedgen.link(href=feed_url, rel='self')
# Link to a chosen URL as an alternate, if set.
if 'htmlUrl' in self.info:
feedgen.link(href=self.info['htmlUrl'], rel='alternate')
else:
# Otherwise link to the original YouTube playlist as an alternate:
if isinstance(self.url, list):
for url in self.url:
feedgen.link(href=url, rel='alternate')
else:
feedgen.link(href=self.url, rel='alternate')
feedgen.language('en')
# feedgen.logo('http://ex.com/logo.jpg')
# pylint: disable=no-member
feedgen.podcast.itunes_category(self.info['itunesCategory']['text'])
feedgen.podcast.itunes_subtitle(self.info['itunesSubtitle'])
feedgen.podcast.itunes_summary(self.info['description'])
feedgen.podcast.itunes_owner(email=self.info['itunesOwner']['email'],
name=self.info['itunesOwner']['name'])
feedgen.podcast.itunes_author(self.info['itunesOwner']['name'])
feedgen.podcast.itunes_image(self.controller.base_url + self.folder +
'/image.jpg')
feedgen.podcast.itunes_explicit('clean')
for vid in self.videos:
print("vid:\n", flush=True)
pp.pprint(vid)
print("\n", flush=True)
vid_filename = vid['_filename'].split('.')[0] + (".mp3" if audio
else ".mp4")
vid_url = self.video_url(vid_filename)
# Size of enclosed file in bytes:
vid_size = os.path.getsize(vid_filename)
# Date of upload (from the youtube-dl JSON data)
eastern = pytz.timezone('US/Eastern')
vid_date = eastern.localize(
datetime.datetime.strptime(vid['upload_date'], '%Y%m%d'))
entry = feedgen.add_entry()
entry.id(vid_url)
entry.title(vid['fulltitle'])
entry.published(vid_date)
for category in vid['categories']:
entry.category(term=category)
entry.description(vid['description'])
entry.enclosure(vid_url, str(vid_size),
('audio/mp3' if audio else 'video/mp4'))
entry.podcast.itunes_image(self.controller.base_url + self.folder +
'/image.jpg')
entry.podcast.itunes_author(self.info['author'])
entry.podcast.itunes_summary(vid['description'])
entry.podcast.itunes_duration(vid['duration'])
feedgen.rss_str(pretty=True)
# Ensure output folder for this podcast exists:
os.makedirs(os.path.join(self.controller.output_dir, self.folder),
exist_ok=True)
# Generate RSS file in output folder:
feedgen.rss_file(
os.path.join(self.controller.output_dir, self.folder,
prefix + 'podcast.xml'))
def generate_atom_feeds(app):
"""
Generate archive pages for all posts, categories, tags, authors, and
drafts.
"""
if not ablog.builder_support(app):
return
blog = Blog(app)
url = blog.blog_baseurl
if not url:
return
feed_path = os.path.join(app.builder.outdir, blog.blog_path, "atom.xml")
feeds = [(
blog.posts,
blog.blog_path,
feed_path,
blog.blog_title,
os_path_join(url, blog.blog_path, "atom.xml"),
)]
if blog.blog_feed_archives:
for header, catalog in [
(_("Posts by"), blog.author),
(_("Posts from"), blog.location),
(_("Posts in"), blog.language),
(_("Posts in"), blog.category),
(_("Posted in"), blog.archive),
(_("Posts tagged"), blog.tags),
]:
for coll in catalog:
# skip collections containing only drafts
if not len(coll):
continue
folder = os.path.join(app.builder.outdir, coll.path)
if not os.path.isdir(folder):
os.makedirs(folder)
feeds.append((
coll,
coll.path,
os.path.join(folder, "atom.xml"),
blog.blog_title + " - " + header + " " + text_type(coll),
os_path_join(url, coll.path, "atom.xml"),
))
# Config options
feed_length = blog.blog_feed_length
feed_fulltext = blog.blog_feed_fulltext
for feed_posts, pagename, feed_path, feed_title, feed_url in feeds:
feed = FeedGenerator()
feed.id("http://lernfunk.de/media/654321")
feed.title(feed_title)
feed.link(href=url)
feed.subtitle(blog.blog_feed_subtitle)
feed.link(href=feed_url)
feed.language("en")
feed.generator("ABlog", ablog.__version__,
"https://ablog.readthedocs.org")
for i, post in enumerate(feed_posts):
if feed_length and i == feed_length:
break
post_url = os_path_join(url,
app.builder.get_target_uri(post.docname))
if post.section:
post_url += "#" + post.section
if blog.blog_feed_titles:
content = None
else:
content = post.to_html(pagename, fulltext=feed_fulltext)
feed_entry = feed.add_entry()
feed_entry.id(post_url)
feed_entry.title(post.title)
feed_entry.link(href=post_url)
feed_entry.author({"name": author.name for author in post.author})
feed_entry.pubDate(post.date.astimezone())
feed_entry.updated(post.update.astimezone())
feed_entry.content(content=content, type="html")
parent_dir = os.path.dirname(feed_path)
if not os.path.isdir(parent_dir):
os.makedirs(parent_dir)
with open(feed_path, "w", encoding="utf-8") as out:
feed_str = feed.atom_str(pretty=True)
out.write(feed_str.decode())
if 0:
# this is to make the function a generator
# and make work for Sphinx 'html-collect-pages'
yield
def setUp(self):
fg = FeedGenerator()
self.nsAtom = "http://www.w3.org/2005/Atom"
self.nsRss = "http://purl.org/rss/1.0/modules/content/"
self.feedId = 'http://lernfunk.de/media/654321'
self.title = 'Some Testfeed'
self.authorName = 'John Doe'
self.authorMail = '*****@*****.**'
self.author = {'name': self.authorName, 'email': self.authorMail}
self.linkHref = 'http://example.com'
self.linkRel = 'alternate'
self.logo = 'http://ex.com/logo.jpg'
self.subtitle = 'This is a cool feed!'
self.link2Href = 'http://larskiesow.de/test.atom'
self.link2Rel = 'self'
self.language = 'en'
self.categoryTerm = 'This category term'
self.categoryScheme = 'This category scheme'
self.categoryLabel = 'This category label'
self.cloudDomain = 'example.com'
self.cloudPort = '4711'
self.cloudPath = '/ws/example'
self.cloudRegisterProcedure = 'registerProcedure'
self.cloudProtocol = 'SOAP 1.1'
self.icon = "http://example.com/icon.png"
self.contributor = {'name': "Contributor Name",
'uri': "Contributor Uri",
'email': 'Contributor email'}
self.copyright = "The copyright notice"
self.docs = 'http://www.rssboard.org/rss-specification'
self.managingEditor = '*****@*****.**'
self.rating = '(PICS-1.1 "http://www.classify.org/safesurf/" ' + \
'1 r (SS~~000 1))'
self.skipDays = 'Tuesday'
self.skipHours = 23
self.textInputTitle = "Text input title"
self.textInputDescription = "Text input description"
self.textInputName = "Text input name"
self.textInputLink = "Text input link"
self.ttl = 900
self.webMaster = '*****@*****.**'
fg.id(self.feedId)
fg.title(self.title)
fg.author(self.author)
fg.link(href=self.linkHref, rel=self.linkRel)
fg.logo(self.logo)
fg.subtitle(self.subtitle)
fg.link(href=self.link2Href, rel=self.link2Rel)
fg.language(self.language)
fg.cloud(domain=self.cloudDomain, port=self.cloudPort,
path=self.cloudPath,
registerProcedure=self.cloudRegisterProcedure,
protocol=self.cloudProtocol)
fg.icon(self.icon)
fg.category(term=self.categoryTerm, scheme=self.categoryScheme,
label=self.categoryLabel)
fg.contributor(self.contributor)
fg.copyright(self.copyright)
fg.docs(docs=self.docs)
fg.managingEditor(self.managingEditor)
fg.rating(self.rating)
fg.skipDays(self.skipDays)
fg.skipHours(self.skipHours)
fg.textInput(title=self.textInputTitle,
description=self.textInputDescription,
name=self.textInputName, link=self.textInputLink)
fg.ttl(self.ttl)
fg.webMaster(self.webMaster)
fg.updated('2017-02-05 13:26:58+01:00')
fg.pubDate('2017-02-05 13:26:58+01:00')
fg.generator('python-feedgen', 'x', uri='http://github.com/lkie...')
fg.image(url=self.logo,
title=self.title,
link=self.link2Href,
width='123',
height='123',
description='Example Inage')
self.fg = fg
async def channel(request, channel_id, return_type='video'):
log.info(f'Channel: {channel_id}')
channel_name = [f'{channel_id}/{return_type}']
if channel_name[0] in channel_feed and channel_feed[
channel_name[0]]['expire'] > datetime.now():
return raw(channel_feed[channel_name[0]]['feed'],
content_type='application/rss+xml')
fg = None
calls = 0
response = {'nextPageToken': ''}
while 'nextPageToken' in response:
next_page = response['nextPageToken']
payload = {
'part': 'snippet,contentDetails',
'maxResults': 50,
'channelId': channel_id,
'key': KEY,
'pageToken': next_page
}
response = json.loads(
await get('https://www.googleapis.com/youtube/v3/activities',
params=payload))
calls += 1
if 'error' in response:
payload = {
'part': 'snippet',
'maxResults': 1,
'forUsername': channel_id,
'key': KEY
}
response = json.loads(await get(
'https://www.googleapis.com/youtube/v3/channels',
params=payload))
channel_id = response['items'][0]['id']
channel_name.append(f'{channel_id}/{return_type}')
payload = {
'part': 'snippet,contentDetails',
'maxResults': 50,
'channelId': channel_id,
'key': KEY,
'pageToken': next_page
}
response = json.loads(await get(
'https://www.googleapis.com/youtube/v3/activities',
params=payload))
calls += 2
if not fg:
fg = FeedGenerator()
fg.load_extension('podcast')
fg.generator('PodTube', __version__,
'https://github.com/aquacash5/PodTube')
snippet = response['items'][0]['snippet']
if 'Private' in snippet['title']:
continue
icon = max(snippet['thumbnails'],
key=lambda x: snippet['thumbnails'][x]['width'])
fg.title(snippet['title'])
fg.id(f'http://{request.headers["host"]}{request.url}')
fg.description(snippet['description'] or ' ')
fg.author(name=snippet['channelTitle'])
fg.image(snippet['thumbnails'][icon]['url'])
fg.link(href=f'https://www.youtube.com/playlist?list={channel_id}')
fg.podcast.itunes_image(snippet['thumbnails'][icon]['url'])
fg.podcast.itunes_summary(snippet['description'])
fg.podcast.itunes_category('Technology', 'Podcasting')
fg.updated(f'{str(datetime.utcnow())}Z')
for item in response['items']:
snippet = item['snippet']
if snippet['type'] != 'upload':
continue
current_video = item['contentDetails']['upload']['videoId']
log.debug(f'ChannelVideo: {current_video} {snippet["title"]}')
fe = fg.add_entry()
fe.title(snippet['title'])
fe.id(current_video)
icon = max(snippet['thumbnails'],
key=lambda x: snippet['thumbnails'][x]['width'])
fe.podcast.itunes_image(snippet['thumbnails'][icon]['url'])
fe.updated(snippet['publishedAt'])
if return_type == 'audio':
fe.enclosure(
url=
f'http://{request.headers["host"]}/audio/{current_video}',
type="audio/mpeg")
else:
fe.enclosure(
url=
f'http://{request.headers["host"]}/video/{current_video}',
type="video/mp4")
fe.author(name=snippet['channelTitle'])
fe.podcast.itunes_author(snippet['channelTitle'])
fe.podcast.itunes_author(snippet['channelTitle'])
fe.pubdate(snippet['publishedAt'])
fe.link(href=f'http://www.youtube.com/watch?v={current_video}',
title=snippet['title'])
fe.podcast.itunes_summary(snippet['description'])
fe.description(snippet['description'])
await sleep(0)
feed = {
'feed': fg.rss_str(),
'expire': datetime.now() + timedelta(hours=calls)
}
for _name in channel_name:
channel_feed[_name] = feed
return raw(feed['feed'], content_type='application/rss+xml')
async def playlist(request, playlist_id, return_type='video'):
log.info(f'Playlist: {playlist_id}')
playlist_name = f'{playlist_id}/{return_type}'
if playlist_name in playlist_feed and playlist_feed[playlist_name][
'expire'] > datetime.now():
return raw(playlist_feed[playlist_name]['feed'],
content_type='application/rss+xml')
calls = 0
payload = {'part': 'snippet', 'id': playlist_id, 'key': KEY}
log.debug('Downloaded Playlist Information')
response = json.loads(await get(
'https://www.googleapis.com/youtube/v3/playlists', params=payload))
calls += 1
fg = FeedGenerator()
fg.load_extension('podcast')
fg.generator('PodTube', __version__,
'https://github.com/aquacash5/PodTube')
snippet = response['items'][0]['snippet']
icon = max(snippet['thumbnails'],
key=lambda x: snippet['thumbnails'][x]['width'])
fg.title(snippet['title'])
fg.id(f'http://{request.headers["host"]}{request.url}')
fg.description(snippet['description'] or ' ')
fg.author(name=snippet['channelTitle'])
fg.image(snippet['thumbnails'][icon]['url'])
fg.link(href=f'https://www.youtube.com/playlist?list={playlist_id}')
fg.podcast.itunes_image(snippet['thumbnails'][icon]['url'])
fg.podcast.itunes_summary(snippet['description'])
fg.podcast.itunes_category('Technology', 'Podcasting')
fg.updated(f'{str(datetime.utcnow())}Z')
response = {'nextPageToken': ''}
while 'nextPageToken' in response.keys():
payload = {
'part': 'snippet',
'maxResults': 50,
'playlistId': playlist_id,
'key': KEY,
'pageToken': response['nextPageToken']
}
response = json.loads(await get(
'https://www.googleapis.com/youtube/v3/playlistItems',
params=payload))
calls += 1
for item in response['items']:
snippet = item['snippet']
current_video = snippet['resourceId']['videoId']
if 'Private' in snippet['title']:
continue
log.debug(f'PlaylistVideo: {current_video} {snippet["title"]}')
fe = fg.add_entry()
fe.title(snippet['title'])
fe.id(current_video)
icon = max(snippet['thumbnails'],
key=lambda x: snippet['thumbnails'][x]['width'])
fe.podcast.itunes_image(snippet['thumbnails'][icon]['url'])
fe.updated(snippet['publishedAt'])
if return_type == 'audio':
fe.enclosure(
url=
f'http://{request.headers["host"]}/audio/{current_video}',
type="audio/mpeg")
else:
fe.enclosure(
url=
f'http://{request.headers["host"]}/video/{current_video}',
type="video/mp4")
fe.author(name=snippet['channelTitle'])
fe.podcast.itunes_author(snippet['channelTitle'])
fe.podcast.itunes_author(snippet['channelTitle'])
fe.pubdate(snippet['publishedAt'])
fe.link(href='http://www.youtube.com/watch?v=' + current_video,
title=snippet['title'])
fe.podcast.itunes_summary(snippet['description'])
fe.description(snippet['description'])
await sleep(0)
feed = {
'feed': fg.rss_str(),
'expire': datetime.now() + timedelta(hours=calls)
}
playlist_feed[playlist_name] = feed
return raw(feed['feed'], content_type='application/rss+xml')
def from_activities(activities, actor=None, title=None, feed_url=None,
home_page_url=None, hfeed=None):
"""Converts ActivityStreams activities to an RSS 2.0 feed.
Args:
activities: sequence of ActivityStreams activity dicts
actor: ActivityStreams actor dict, the author of the feed
title: string, the feed title
feed_url: string, the URL for this RSS feed
home_page_url: string, the home page URL
hfeed: dict, parsed mf2 h-feed, if available
Returns:
unicode string with RSS 2.0 XML
"""
try:
iter(activities)
except TypeError:
raise TypeError('activities must be iterable')
if isinstance(activities, (dict, basestring)):
raise TypeError('activities may not be a dict or string')
fg = FeedGenerator()
fg.id(feed_url)
assert feed_url
fg.link(href=feed_url, rel='self')
if home_page_url:
fg.link(href=home_page_url, rel='alternate')
# TODO: parse language from lang attribute:
# https://github.com/microformats/mf2py/issues/150
fg.language('en')
fg.generator('granary', uri='https://granary.io/')
hfeed = hfeed or {}
actor = actor or {}
image = util.get_url(hfeed, 'image') or util.get_url(actor, 'image')
if image:
fg.image(image)
props = hfeed.get('properties') or {}
content = microformats2.get_text(util.get_first(props, 'content', ''))
summary = util.get_first(props, 'summary', '')
desc = content or summary or '-'
fg.description(desc) # required
fg.title(title or util.ellipsize(desc)) # required
latest = None
enclosures = False
for activity in activities:
obj = activity.get('object') or activity
if obj.get('objectType') == 'person':
continue
item = fg.add_entry()
url = obj.get('url')
item.id(obj.get('id') or url)
item.link(href=url)
item.guid(url, permalink=True)
item.title(obj.get('title') or obj.get('displayName') or '-') # required
content = microformats2.render_content(
obj, include_location=True, render_attachments=False) or obj.get('summary')
if content:
item.content(content, type='CDATA')
item.category(
[{'term': t['displayName']} for t in obj.get('tags', [])
if t.get('displayName') and t.get('verb') not in ('like', 'react', 'share')])
author = obj.get('author', {})
item.author({
'name': author.get('displayName') or author.get('username'),
'uri': author.get('url'),
})
published = obj.get('published') or obj.get('updated')
if published:
try:
dt = mf2util.parse_datetime(published)
if not isinstance(dt, datetime):
dt = datetime.combine(dt, time.min)
if not dt.tzinfo:
dt = dt.replace(tzinfo=util.UTC)
item.published(dt)
if not latest or dt > latest:
latest = dt
except ValueError: # bad datetime string
pass
for att in obj.get('attachments', []):
stream = util.get_first(att, 'stream') or att
if not stream:
continue
url = stream.get('url') or ''
mime = mimetypes.guess_type(url)[0] or ''
if (att.get('objectType') in ENCLOSURE_TYPES or
mime and mime.split('/')[0] in ENCLOSURE_TYPES):
enclosures = True
item.enclosure(url=url, type=mime, length='REMOVEME') # TODO: length (bytes)
item.load_extension('podcast')
duration = stream.get('duration')
if duration:
item.podcast.itunes_duration(duration)
if enclosures:
fg.load_extension('podcast')
fg.podcast.itunes_author(actor.get('displayName') or actor.get('username'))
if summary:
fg.podcast.itunes_summary(summary)
fg.podcast.itunes_explicit('no')
fg.podcast.itunes_block(False)
if latest:
fg.lastBuildDate(latest)
return fg.rss_str(pretty=True).decode('utf-8').replace(' length="REMOVEME"', '')
def from_activities(activities,
actor=None,
title=None,
feed_url=None,
home_page_url=None,
hfeed=None):
"""Converts ActivityStreams activities to an RSS 2.0 feed.
Args:
activities: sequence of ActivityStreams activity dicts
actor: ActivityStreams actor dict, the author of the feed
title: string, the feed title
feed_url: string, the URL for this RSS feed
home_page_url: string, the home page URL
hfeed: dict, parsed mf2 h-feed, if available
Returns:
unicode string with RSS 2.0 XML
"""
try:
iter(activities)
except TypeError:
raise TypeError('activities must be iterable')
if isinstance(activities, (dict, str)):
raise TypeError('activities may not be a dict or string')
fg = FeedGenerator()
fg.id(feed_url)
assert feed_url
fg.link(href=feed_url, rel='self')
if home_page_url:
fg.link(href=home_page_url, rel='alternate')
# TODO: parse language from lang attribute:
# https://github.com/microformats/mf2py/issues/150
fg.language('en')
fg.generator('granary', uri='https://granary.io/')
hfeed = hfeed or {}
actor = actor or {}
image = (util.get_url(hfeed.get('properties', {}), 'photo')
or util.get_url(actor, 'image'))
if image:
fg.image(image)
props = hfeed.get('properties') or {}
content = microformats2.get_text(util.get_first(props, 'content', ''))
summary = util.get_first(props, 'summary', '')
desc = content or summary or '-'
fg.description(desc) # required
fg.title(title or util.ellipsize(desc)) # required
latest = None
feed_has_enclosure = False
for activity in activities:
obj = activity.get('object') or activity
if obj.get('objectType') == 'person':
continue
item = fg.add_entry()
url = obj.get('url')
id = obj.get('id') or url
item.id(id)
item.link(href=url)
item.guid(url, permalink=True)
# title (required)
title = (obj.get('title') or obj.get('displayName')
or util.ellipsize(obj.get('content', '-')))
# strip HTML tags
title = util.parse_html(title).get_text('').strip()
item.title(title)
content = microformats2.render_content(obj,
include_location=True,
render_attachments=True,
render_image=True)
if not content:
content = obj.get('summary')
if content:
item.content(content, type='CDATA')
categories = [
{
'term': t['displayName']
} for t in obj.get('tags', [])
if t.get('displayName') and t.get('verb') not in ('like', 'react',
'share')
and t.get('objectType') not in ('article', 'person', 'mention')
]
item.category(categories)
author = obj.get('author', {})
author = {
'name': author.get('displayName') or author.get('username'),
'uri': author.get('url'),
'email': author.get('email') or '-',
}
item.author(author)
published = obj.get('published') or obj.get('updated')
if published and isinstance(published, str):
try:
dt = mf2util.parse_datetime(published)
if not isinstance(dt, datetime):
dt = datetime.combine(dt, time.min)
if not dt.tzinfo:
dt = dt.replace(tzinfo=util.UTC)
item.published(dt)
if not latest or dt > latest:
latest = dt
except ValueError: # bad datetime string
pass
item_has_enclosure = False
for att in obj.get('attachments', []):
stream = util.get_first(att, 'stream') or att
if not stream:
continue
url = stream.get('url') or ''
mime = mimetypes.guess_type(url)[0] or ''
if (att.get('objectType') in ENCLOSURE_TYPES
or mime and mime.split('/')[0] in ENCLOSURE_TYPES):
if item_has_enclosure:
logging.info(
'Warning: item %s already has an RSS enclosure, skipping additional enclosure %s',
id, url)
continue
item_has_enclosure = feed_has_enclosure = True
item.enclosure(url=url,
type=mime,
length=str(stream.get('size', '')))
item.load_extension('podcast')
duration = stream.get('duration')
if duration:
item.podcast.itunes_duration(duration)
if feed_has_enclosure:
fg.load_extension('podcast')
fg.podcast.itunes_author(
actor.get('displayName') or actor.get('username'))
if summary:
fg.podcast.itunes_summary(summary)
fg.podcast.itunes_explicit('no')
fg.podcast.itunes_block(False)
name = author.get('name')
if name:
fg.podcast.itunes_author(name)
if image:
fg.podcast.itunes_image(image)
fg.podcast.itunes_category(categories)
if latest:
fg.lastBuildDate(latest)
return fg.rss_str(pretty=True).decode('utf-8')
podcast_cover = 'https://conteudo.depois.cafe/depois-do-cafe-exclusivo.png'
fg = FeedGenerator()
fg.load_extension('podcast')
fg.id('https://episodios.depois.cafe')
fg.title('Depois do Café Exclusivo')
fg.subtitle(podcast_description)
fg.author({'name': podcast_author, 'email': podcast_email})
fg.link(href='https://episodios.depois.cafe', rel='alternate')
fg.link(href='https://episodios.depois.cafe',
rel='self',
type='application/rss+xml')
fg.logo(podcast_cover)
fg.language('pt-BR')
fg.generator('https://github.com/lkiesow/python-feedgen')
fg.copyright(podcast_author)
fg.podcast.itunes_author(podcast_author)
fg.podcast.itunes_category('Technology')
fg.podcast.itunes_subtitle(podcast_description)
fg.podcast.itunes_summary(podcast_description)
fg.podcast.itunes_owner(name=podcast_author, email=podcast_email)
fg.podcast.itunes_explicit('no')
fg.podcast.itunes_image(podcast_cover)
for ep_extra in extras:
entry = fg.add_entry()
entry.title(ep_extra['title'])
entry.id(ep_extra['mp3_link'])
entry.link(href=ep_extra['mp3_link'], rel='alternate')
def setUp(self):
fg = FeedGenerator()
self.nsAtom = "http://www.w3.org/2005/Atom"
self.nsRss = "http://purl.org/rss/1.0/modules/content/"
self.feedId = 'http://lernfunk.de/media/654321'
self.title = 'Some Testfeed'
self.authorName = 'John Doe'
self.authorMail = '*****@*****.**'
self.author = {'name': self.authorName, 'email': self.authorMail}
self.linkHref = 'http://example.com'
self.linkRel = 'alternate'
self.logo = 'http://ex.com/logo.jpg'
self.subtitle = 'This is a cool feed!'
self.link2Href = 'http://larskiesow.de/test.atom'
self.link2Rel = 'self'
self.language = 'en'
self.categoryTerm = 'This category term'
self.categoryScheme = 'This category scheme'
self.categoryLabel = 'This category label'
self.cloudDomain = 'example.com'
self.cloudPort = '4711'
self.cloudPath = '/ws/example'
self.cloudRegisterProcedure = 'registerProcedure'
self.cloudProtocol = 'SOAP 1.1'
self.icon = "http://example.com/icon.png"
self.contributor = {
'name': "Contributor Name",
'uri': "Contributor Uri",
'email': 'Contributor email'
}
self.copyright = "The copyright notice"
self.docs = 'http://www.rssboard.org/rss-specification'
self.managingEditor = '*****@*****.**'
self.rating = '(PICS-1.1 "http://www.classify.org/safesurf/" ' + \
'1 r (SS~~000 1))'
self.skipDays = 'Tuesday'
self.skipHours = 23
self.textInputTitle = "Text input title"
self.textInputDescription = "Text input description"
self.textInputName = "Text input name"
self.textInputLink = "Text input link"
self.ttl = 900
self.webMaster = '*****@*****.**'
fg.id(self.feedId)
fg.title(self.title)
fg.author(self.author)
fg.link(href=self.linkHref, rel=self.linkRel)
fg.logo(self.logo)
fg.subtitle(self.subtitle)
fg.link(href=self.link2Href, rel=self.link2Rel)
fg.language(self.language)
fg.cloud(domain=self.cloudDomain,
port=self.cloudPort,
path=self.cloudPath,
registerProcedure=self.cloudRegisterProcedure,
protocol=self.cloudProtocol)
fg.icon(self.icon)
fg.category(term=self.categoryTerm,
scheme=self.categoryScheme,
label=self.categoryLabel)
fg.contributor(self.contributor)
fg.copyright(self.copyright)
fg.docs(docs=self.docs)
fg.managingEditor(self.managingEditor)
fg.rating(self.rating)
fg.skipDays(self.skipDays)
fg.skipHours(self.skipHours)
fg.textInput(title=self.textInputTitle,
description=self.textInputDescription,
name=self.textInputName,
link=self.textInputLink)
fg.ttl(self.ttl)
fg.webMaster(self.webMaster)
fg.updated('2017-02-05 13:26:58+01:00')
fg.pubDate('2017-02-05 13:26:58+01:00')
fg.generator('python-feedgen', 'x', uri='http://github.com/lkie...')
fg.image(url=self.logo,
title=self.title,
link=self.link2Href,
width='123',
height='123',
description='Example Inage')
self.fg = fg