def find_urls(page):
r4 = re.search('url_encoded_fmt_stream_map=([^&]+)', page)
if r4 is not None:
fmt_url_map = urllib.parse.unquote(r4.group(1))
for fmt_url_encoded in fmt_url_map.split(','):
video_info = parse_qs(fmt_url_encoded)
yield int(video_info['itag'][0]), video_info['url'][0]
else:
error_info = parse_qs(page)
if 'reason' in error_info:
error_message = util.remove_html_tags(
error_info['reason'][0])
elif 'player_response' in error_info:
player_response = json.loads(
error_info['player_response'][0])
if 'reason' in player_response['playabilityStatus']:
error_message = util.remove_html_tags(
player_response['playabilityStatus']['reason'])
elif 'live_playback' in error_info:
error_message = 'live stream'
elif 'post_live_playback' in error_info:
error_message = 'post live stream'
else:
error_message = ''
else:
error_message = ''
raise YouTubeError('Cannot download video: %s' % error_message)
def from_feedparser_entry( entry, channel):
episode=podcastItem( channel)
episode.title=entry.get( 'title', util.get_first_line( util.remove_html_tags( entry.get( 'summary', ''))))
episode.link=entry.get( 'link', '')
episode.description=util.remove_html_tags( entry.get( 'summary', entry.get( 'link', entry.get( 'title', ''))))
episode.guid=entry.get( 'id', '')
if entry.get( 'updated_parsed', None):
episode.pubDate=util.updated_parsed_to_rfc2822( entry.updated_parsed)
if episode.title == '':
log( 'Warning: Episode has no title, adding anyways.. (Feed Is Buggy!)', sender=episode)
enclosure=None
if hasattr(entry, 'enclosures') and len(entry.enclosures) > 0:
enclosure=entry.enclosures[0]
if len(entry.enclosures) > 1:
for e in entry.enclosures:
if hasattr( e, 'href') and hasattr( e, 'length') and hasattr( e, 'type') and (e.type.startswith('audio/') or e.type.startswith('video/')):
if util.normalize_feed_url(e.href) is not None:
log( 'Selected enclosure: %s', e.href, sender=episode)
enclosure=e
break
episode.url=util.normalize_feed_url( enclosure.get( 'href', ''))
elif hasattr(entry, 'link'):
extension=util.file_extension_from_url(entry.link)
file_type=util.file_type_by_extension(extension)
if file_type is not None:
log('Adding episode with link to file type "%s".', file_type, sender=episode)
episode.url=entry.link
if not episode.url:
raise ValueError( 'Episode has an invalid URL')
if not episode.pubDate:
metainfo=episode.get_metainfo()
if 'pubdate' in metainfo:
episode.pubDate=metainfo['pubdate']
if hasattr( enclosure, 'length'):
try:
episode.length=int(enclosure.length)
except:
episode.length=-1
# For episodes with a small length amount, try to find it via HTTP HEAD
if episode.length <= 100:
metainfo=episode.get_metainfo()
if 'length' in metainfo:
episode.length=metainfo['length']
if hasattr( enclosure, 'type'):
episode.mimetype=enclosure.type
if episode.title == '':
( filename, extension )=os.path.splitext( os.path.basename( episode.url))
episode.title=filename
return episode
def find_urls(page):
# streamingData is preferable to url_encoded_fmt_stream_map
# streamingData.formats are the same as url_encoded_fmt_stream_map
# streamingData.adaptiveFormats are audio-only and video-only formats
x = parse_qs(page)
error_message = None
if 'reason' in x:
error_message = util.remove_html_tags(x['reason'][0])
elif 'player_response' in x:
player_response = json.loads(x['player_response'][0])
playabilityStatus = player_response['playabilityStatus']
if 'reason' in playabilityStatus:
error_message = util.remove_html_tags(
playabilityStatus['reason'])
elif 'liveStreamability' in playabilityStatus \
and not playabilityStatus['liveStreamability'].get('liveStreamabilityRenderer', {}).get('displayEndscreen', False):
# playabilityStatus.liveStreamability -- video is or was a live stream
# playabilityStatus.liveStreamability.liveStreamabilityRenderer.displayEndscreen -- video has ended if present
error_message = 'live stream'
elif 'streamingData' in player_response:
# DRM videos store url inside a cipher key - not supported
if 'formats' in player_response['streamingData']:
for f in player_response['streamingData']['formats']:
if 'url' in f:
yield int(f['itag']), [
f['url'],
f.get('approxDurationMs')
]
if 'adaptiveFormats' in player_response['streamingData']:
for f in player_response['streamingData'][
'adaptiveFormats']:
if 'url' in f:
yield int(f['itag']), [
f['url'],
f.get('approxDurationMs')
]
return
if error_message is not None:
raise YouTubeError('Cannot download video: %s' % error_message)
r4 = re.search('url_encoded_fmt_stream_map=([^&]+)', page)
if r4 is not None:
fmt_url_map = urllib.parse.unquote(r4.group(1))
for fmt_url_encoded in fmt_url_map.split(','):
video_info = parse_qs(fmt_url_encoded)
yield int(
video_info['itag'][0]), [video_info['url'][0], None]
def safe_first_line(txt):
txt = safe_str(txt)
lines = util.remove_html_tags(txt).strip().splitlines()
if not lines or lines[0] == '':
return ''
else:
return lines[0]
def _format_description(self, channel, total, deleted, new, downloaded,
unplayed):
title_markup = html.escape(channel.title)
if channel._update_error is not None:
description_markup = html.escape(
_('ERROR: %s') % channel._update_error)
elif not channel.pause_subscription:
description_markup = html.escape(
util.get_first_line(util.remove_html_tags(channel.description))
or ' ')
else:
description_markup = html.escape(_('Subscription paused'))
d = []
if new:
d.append('<span weight="bold">')
d.append(title_markup)
if new:
d.append('</span>')
if channel._update_error is not None:
return ''.join(
d +
['\n', '<span weight="bold">', description_markup, '</span>'])
elif description_markup.strip():
return ''.join(d +
['\n', '<small>', description_markup, '</small>'])
else:
return ''.join(d)
def on_display_text(self):
# Now do the stuff that takes a bit longer...
heading = self.episode.title
subheading = 'from %s' % (self.episode.channel.title)
description = self.episode.description
if self.have_webkit:
global SHOWNOTES_HTML_TEMPLATE
import webkit
args = (
saxutils.escape(heading),
saxutils.escape(subheading),
self.episode.description,
)
url = os.path.dirname(self.episode.channel.url)
self.htmlview.load_html_string(SHOWNOTES_HTML_TEMPLATE % args, url)
else:
self.b.create_tag('heading', scale=pango.SCALE_LARGE, weight=pango.WEIGHT_BOLD)
self.b.create_tag('subheading', scale=pango.SCALE_SMALL)
self.b.insert_with_tags_by_name(self.b.get_end_iter(), heading, 'heading')
self.b.insert_at_cursor('\n')
self.b.insert_with_tags_by_name(self.b.get_end_iter(), subheading, 'subheading')
self.b.insert_at_cursor('\n\n')
self.b.insert(self.b.get_end_iter(), util.remove_html_tags(description))
self.b.place_cursor(self.b.get_start_iter())
def safe_first_line(txt):
txt = safe_str(txt)
lines = util.remove_html_tags(txt).strip().splitlines()
if not lines or lines[0] == '':
return ''
else:
return lines[0]
def treeview_episodes_query_tooltip(self, treeview, x, y, keyboard_tooltip, tooltip):
# With get_bin_window, we get the window that contains the rows without
# the header. The Y coordinate of this window will be the height of the
# treeview header. This is the amount we have to subtract from the
# event's Y coordinate to get the coordinate to pass to get_path_at_pos
(x_bin, y_bin) = treeview.get_bin_window().get_position()
y -= x_bin
y -= y_bin
(path, column, rx, ry) = treeview.get_path_at_pos(x, y) or (None,) * 4
if not self.episode_list_can_tooltip or column != treeview.get_columns()[1]:
self.last_tooltip_episode = None
return False
if path is not None:
model = treeview.get_model()
iter = model.get_iter(path)
index = model.get_value(iter, self.COLUMN_INDEX)
description = model.get_value(iter, self.COLUMN_TOOLTIP)
if self.last_tooltip_episode is not None and self.last_tooltip_episode != index:
self.last_tooltip_episode = None
return False
self.last_tooltip_episode = index
description = util.remove_html_tags(description)
if description is not None:
if len(description) > 400:
description = description[:398] + "[...]"
tooltip.set_text(description)
return True
else:
return False
self.last_tooltip_episode = None
return False
def on_display_text(self):
# Now do the stuff that takes a bit longer...
heading = self.episode.title
subheading = _('from %s') % (self.episode.channel.title)
description = self.episode.description
if self.have_webkit:
global SHOWNOTES_HTML_TEMPLATE
# Get the description - if it looks like plaintext, replace the
# newline characters with line breaks for the HTML view
description = self.episode.description
if '<' not in description:
description = description.replace('\n', '<br>')
args = (
saxutils.escape(heading),
saxutils.escape(subheading),
description,
)
url = os.path.dirname(self.episode.channel.url)
self.htmlview.load_html_string(SHOWNOTES_HTML_TEMPLATE % args, url)
else:
self.b.create_tag('heading', scale=pango.SCALE_LARGE, weight=pango.WEIGHT_BOLD)
self.b.create_tag('subheading', scale=pango.SCALE_SMALL)
self.b.insert_with_tags_by_name(self.b.get_end_iter(), heading, 'heading')
self.b.insert_at_cursor('\n')
self.b.insert_with_tags_by_name(self.b.get_end_iter(), subheading, 'subheading')
self.b.insert_at_cursor('\n\n')
self.b.insert(self.b.get_end_iter(), util.remove_html_tags(description))
self.b.place_cursor(self.b.get_start_iter())
def update(self, heading, subheading, episode):
self.text_buffer.set_text('')
self.text_buffer.insert_with_tags_by_name(self.text_buffer.get_end_iter(), heading, 'heading')
self.text_buffer.insert_at_cursor('\n')
self.text_buffer.insert_with_tags_by_name(self.text_buffer.get_end_iter(), subheading, 'subheading')
self.text_buffer.insert_at_cursor('\n\n')
self.text_buffer.insert(self.text_buffer.get_end_iter(), util.remove_html_tags(episode.description))
self.text_buffer.place_cursor(self.text_buffer.get_start_iter())
def update(self, heading, subheading, episode):
self.text_buffer.set_text('')
self.text_buffer.insert_with_tags_by_name(self.text_buffer.get_end_iter(), heading, 'heading')
self.text_buffer.insert_at_cursor('\n')
self.text_buffer.insert_with_tags_by_name(self.text_buffer.get_end_iter(), subheading, 'subheading')
self.text_buffer.insert_at_cursor('\n\n')
self.text_buffer.insert(self.text_buffer.get_end_iter(), util.remove_html_tags(episode.description))
self.text_buffer.place_cursor(self.text_buffer.get_start_iter())
def from_podcastparser_entry(cls, entry, channel):
episode = cls(channel)
episode.guid = entry['guid']
episode.title = entry['title']
episode.link = entry['link']
episode.description = entry['description']
if entry.get('description_html'):
episode.description_html = entry['description_html']
# TODO: This really should be handled in podcastparser and not here.
elif util.is_html(entry['description']):
episode.description_html = entry['description']
episode.description = util.remove_html_tags(entry['description'])
episode.total_time = entry['total_time']
episode.published = entry['published']
episode.payment_url = entry['payment_url']
audio_available = any(enclosure['mime_type'].startswith('audio/') for enclosure in entry['enclosures'])
video_available = any(enclosure['mime_type'].startswith('video/') for enclosure in entry['enclosures'])
for enclosure in entry['enclosures']:
episode.mime_type = enclosure['mime_type']
# Skip images in feeds if audio or video is available (bug 979)
# This must (and does) also look in Media RSS enclosures (bug 1430)
if episode.mime_type.startswith('image/') and (audio_available or video_available):
continue
# If we have audio or video available later on, skip
# 'application/octet-stream' data types (fixes Linux Outlaws)
if episode.mime_type == 'application/octet-stream' and (audio_available or video_available):
continue
episode.url = util.normalize_feed_url(enclosure['url'])
if not episode.url:
continue
episode.file_size = enclosure['file_size']
return episode
# Brute-force detection of the episode link
episode.url = util.normalize_feed_url(entry['link'])
if not episode.url:
return None
if any(mod.is_video_link(episode.url) for mod in (youtube, vimeo, escapist_videos)):
return episode
# Check if we can resolve this link to a audio/video file
filename, extension = util.filename_from_url(episode.url)
file_type = util.file_type_by_extension(extension)
# The link points to a audio or video file - use it!
if file_type is not None:
return episode
return None
def find_urls(page):
r4 = re.search('url_encoded_fmt_stream_map=([^&]+)', page)
if r4 is not None:
fmt_url_map = urllib.unquote(r4.group(1))
for fmt_url_encoded in fmt_url_map.split(','):
video_info = parse_qs(fmt_url_encoded)
yield int(video_info['itag'][0]), video_info['url'][0]
else:
error_info = parse_qs(page)
error_message = util.remove_html_tags(error_info['reason'][0])
raise YouTubeError('Cannot download video: %s' % error_message)
def find_urls(page):
r4 = re.search('url_encoded_fmt_stream_map=([^&]+)', page)
if r4 is not None:
fmt_url_map = urllib.parse.unquote(r4.group(1))
for fmt_url_encoded in fmt_url_map.split(','):
video_info = parse_qs(fmt_url_encoded)
yield int(video_info['itag'][0]), video_info['url'][0]
else:
error_info = parse_qs(page)
error_message = util.remove_html_tags(error_info['reason'][0])
raise YouTubeError('Cannot download video: %s' % error_message)
def on_display_text(self):
heading = self.episode.title
subheading = "; ".join(
(self.episode.cute_pubdate(), self.episode.get_filesize_string(), self.episode.channel.title)
)
description = self.episode.description
self.text_buffer.insert_with_tags_by_name(self.text_buffer.get_end_iter(), heading, "heading")
self.text_buffer.insert_at_cursor("\n")
self.text_buffer.insert_with_tags_by_name(self.text_buffer.get_end_iter(), subheading, "subheading")
self.text_buffer.insert_at_cursor("\n\n")
self.text_buffer.insert(self.text_buffer.get_end_iter(), util.remove_html_tags(description))
self.text_buffer.place_cursor(self.text_buffer.get_start_iter())
def one_line_description(self):
MAX_LINE_LENGTH = 120
desc = util.remove_html_tags(self.description or '')
desc = re.sub('\s+', ' ', desc).strip()
if not desc:
return _('No description available')
else:
# Decode the description to avoid gPodder bug 1277
if isinstance(desc, str):
desc = desc.decode('utf-8', 'ignore')
if len(desc) > MAX_LINE_LENGTH:
return desc[:MAX_LINE_LENGTH] + '...'
else:
return desc
def one_line_description(self):
MAX_LINE_LENGTH = 120
desc = util.remove_html_tags(self.description or "")
desc = re.sub("\s+", " ", desc).strip()
if not desc:
return _("No description available")
else:
# Decode the description to avoid gPodder bug 1277
desc = util.convert_bytes(desc).strip()
if len(desc) > MAX_LINE_LENGTH:
return desc[:MAX_LINE_LENGTH] + "..."
else:
return desc
def on_display_text(self):
heading = self.episode.title
subheading = _('from %s') % (self.episode.channel.title)
description = self.episode.description
self.text_buffer.insert_with_tags_by_name(\
self.text_buffer.get_end_iter(), heading, 'heading')
self.text_buffer.insert_at_cursor('\n')
self.text_buffer.insert_with_tags_by_name(\
self.text_buffer.get_end_iter(), subheading, 'subheading')
self.text_buffer.insert_at_cursor('\n\n')
self.text_buffer.insert(self.text_buffer.get_end_iter(), \
util.remove_html_tags(description))
self.text_buffer.place_cursor(self.text_buffer.get_start_iter())
def one_line_description(self):
MAX_LINE_LENGTH = 120
desc = util.remove_html_tags(self.description or '')
desc = re.sub('\s+', ' ', desc).strip()
if not desc:
return _('No description available')
else:
# Decode the description to avoid gPodder bug 1277
desc = util.convert_bytes(desc).strip()
if len(desc) > MAX_LINE_LENGTH:
return desc[:MAX_LINE_LENGTH] + '...'
else:
return desc
def one_line_description(self):
MAX_LINE_LENGTH = 120
desc = util.remove_html_tags(self.description or '')
desc = re.sub('\s+', ' ', desc).strip()
if not desc:
return _('No description available')
else:
# Decode the description to avoid gPodder bug 1277
desc = util.convert_bytes(desc).strip()
if len(desc) > MAX_LINE_LENGTH:
return desc[:MAX_LINE_LENGTH] + '...'
else:
return desc
def get_by_url(cls, url, force_update=False, offline=False, default_title=None, old_channel=None):
if isinstance( url, unicode):
url=url.encode('utf-8')
(updated, c)=cls.fc.fetch( url, force_update, offline)
# If we have an old instance of this channel, and
# feedcache says the feed hasn't changed, return old
if not updated and old_channel:
log('using old channel for %s', url)
return old_channel
channel=podcastChannel( url)
channel.parse_error=c.get('bozo_exception', None)
channel.load_settings()
if hasattr(c.feed, 'title'):
channel.title=c.feed.title
elif default_title is not None:
channel.title=default_title
else:
channel.title=url
if hasattr( c.feed, 'link'):
channel.link=c.feed.link
if hasattr( c.feed, 'subtitle'):
channel.description=util.remove_html_tags(c.feed.subtitle)
if hasattr(c.feed, 'updated_parsed') and c.feed.updated_parsed is not None:
channel.pubDate=util.updated_parsed_to_rfc2822(c.feed.updated_parsed)
if hasattr( c.feed, 'image'):
if c.feed.image.href:
channel.image=c.feed.image.href
# We can limit the maximum number of entries that gPodder will parse
# via the "max_episodes_per_feed" configuration option.
if len(c.entries) > gl.config.max_episodes_per_feed:
log('Limiting number of episodes for %s to %d', channel.title, gl.config.max_episodes_per_feed)
for entry in c.entries[:min(gl.config.max_episodes_per_feed, len(c.entries))]:
episode=None
try:
episode=podcastItem.from_feedparser_entry( entry, channel)
except:
log( 'Cannot instantiate episode: %s. Skipping.', entry.get( 'id', '(no id available)'), sender=channel, traceback=True)
if episode:
channel.append( episode)
channel.sort( reverse=True)
return channel
def get_new_episodes(self, channel, existing_guids):
# entries are already sorted by decreasing date
# trim guids to max episodes
entries = [
e for i, e in enumerate(self._ie_result['entries'])
if not self._max_episodes or i < self._max_episodes
]
all_seen_guids = set(e['guid'] for e in entries)
# only fetch new ones from youtube since they are so slow to get
new_entries = [e for e in entries if e['guid'] not in existing_guids]
logger.debug('%i/%i new entries', len(new_entries),
len(all_seen_guids))
self._ie_result['entries'] = new_entries
self._downloader.refresh_entries(self._ie_result)
# episodes from entries
episodes = []
for en in self._ie_result['entries']:
guid = video_guid(en['id'])
description = remove_html_tags(
en.get('description') or _('No description available'))
html_description = nice_html_description(en.get('thumbnail'),
description)
if en.get('ext'):
mime_type = mimetype_from_extension('.{}'.format(en['ext']))
else:
mime_type = 'application/octet-stream'
if en.get('filesize'):
filesize = int(en['filesize'] or 0)
else:
filesize = sum(
int(f.get('filesize') or 0)
for f in en.get('requested_formats', []))
ep = {
'title': en.get('title', guid),
'link': en.get('webpage_url'),
'description': description,
'description_html': html_description,
'url': en.get('webpage_url'),
'file_size': filesize,
'mime_type': mime_type,
'guid': guid,
'published': youtube_parsedate(en.get('upload_date', None)),
'total_time': int(en.get('duration') or 0),
}
episode = channel.episode_factory(ep)
episode.save()
episodes.append(episode)
return episodes, all_seen_guids
def on_display_text(self):
heading = self.episode.title
subheading = '; '.join((self.episode.cute_pubdate(), \
self.episode.get_filesize_string(), \
self.episode.channel.title))
description = self.episode.description
self.text_buffer.insert_with_tags_by_name(\
self.text_buffer.get_end_iter(), heading, 'heading')
self.text_buffer.insert_at_cursor('\n')
self.text_buffer.insert_with_tags_by_name(\
self.text_buffer.get_end_iter(), subheading, 'subheading')
self.text_buffer.insert_at_cursor('\n\n')
self.text_buffer.insert(self.text_buffer.get_end_iter(), \
util.remove_html_tags(description))
self.text_buffer.place_cursor(self.text_buffer.get_start_iter())
def __init__( self, url="", title="", link="", description=""):
self.url=url
self.title=title
self.link=link
self.description=util.remove_html_tags( description)
self.image=None
self.pubDate=''
self.parse_error=None
# should this channel be synced to devices? (ex: iPod)
self.sync_to_devices=True
# to which playlist should be synced
self.device_playlist_name='gPodder'
# if set, this overrides the channel-provided title
self.override_title=''
self.username=''
self.password=''
self.save_dir_size=0
self.__tree_model=None
def treeview_episodes_query_tooltip(self, treeview, x, y, keyboard_tooltip,
tooltip):
# With get_bin_window, we get the window that contains the rows without
# the header. The Y coordinate of this window will be the height of the
# treeview header. This is the amount we have to subtract from the
# event's Y coordinate to get the coordinate to pass to get_path_at_pos
(x_bin, y_bin) = treeview.get_bin_window().get_position()
y -= x_bin
y -= y_bin
(path, column, rx, ry) = treeview.get_path_at_pos(x, y) or (None, ) * 4
if not self.episode_list_can_tooltip or column != treeview.get_columns(
)[1]:
self.last_tooltip_episode = None
return False
if path is not None:
model = treeview.get_model()
iter = model.get_iter(path)
index = model.get_value(iter, self.COLUMN_INDEX)
description = model.get_value(iter, self.COLUMN_TOOLTIP)
if self.last_tooltip_episode is not None and self.last_tooltip_episode != index:
self.last_tooltip_episode = None
return False
self.last_tooltip_episode = index
description = util.remove_html_tags(description)
# Bug 1825: make sure description is a unicode string,
# so it may be cut correctly on UTF-8 char boundaries
description = util.convert_bytes(description)
if description is not None:
if len(description) > 400:
description = description[:398] + '[...]'
tooltip.set_text(description)
return True
else:
return False
self.last_tooltip_episode = None
return False
def on_display_text(self):
# Now do the stuff that takes a bit longer...
heading = self.episode.title
subheading = _('from %s') % (self.episode.channel.title)
description = self.episode.description
if self.have_webkit:
global SHOWNOTES_HTML_TEMPLATE
# Get the description - if it looks like plaintext, replace the
# newline characters with line breaks for the HTML view
description = self.episode.description
if '<' not in description:
description = description.replace('\n', '<br>')
args = (
saxutils.escape(heading),
saxutils.escape(subheading),
self.episode.get_play_info_string(),
description,
)
url = os.path.dirname(self.episode.channel.url)
self.htmlview.load_html_string(SHOWNOTES_HTML_TEMPLATE % args, url)
else:
self.b.create_tag('heading',
scale=pango.SCALE_LARGE,
weight=pango.WEIGHT_BOLD)
self.b.create_tag('subheading', scale=pango.SCALE_SMALL)
self.b.insert_with_tags_by_name(self.b.get_end_iter(), heading,
'heading')
self.b.insert_at_cursor('\n')
self.b.insert_with_tags_by_name(self.b.get_end_iter(), subheading,
'subheading')
self.b.insert_at_cursor('\n\n')
self.b.insert(self.b.get_end_iter(),
util.remove_html_tags(description))
self.b.place_cursor(self.b.get_start_iter())
def on_display_text(self):
# Now do the stuff that takes a bit longer...
heading = self.episode.title
subheading = 'from %s' % (self.episode.channel.title)
description = self.episode.description
if self.have_gtkhtml2:
import gtkhtml2
self.d.connect('link-clicked', lambda doc, url: util.open_website(url))
def request_url(document, url, stream):
def opendata(url, stream):
fp = util.urlopen(url)
data = fp.read(1024*10)
while data != '':
stream.write(data)
data = fp.read(1024*10)
stream.close()
threading.Thread(target=opendata, args=[url, stream]).start()
self.d.connect('request-url', request_url)
self.d.clear()
self.d.open_stream('text/html')
self.d.write_stream('<html><head><meta http-equiv="content-type" content="text/html; charset=utf-8"/></head><body>')
self.d.write_stream('<span style="font-size: big; font-weight: bold;">%s</span><br><span style="font-size: small;">%s</span><hr style="border: 1px #eeeeee solid;"><p>' % (saxutils.escape(heading), saxutils.escape(subheading)))
self.d.write_stream(self.episode.description)
self.d.write_stream('</p></body></html>')
self.d.close_stream()
else:
self.b.create_tag('heading', scale=pango.SCALE_LARGE, weight=pango.WEIGHT_BOLD)
self.b.create_tag('subheading', scale=pango.SCALE_SMALL)
self.b.insert_with_tags_by_name(self.b.get_end_iter(), heading, 'heading')
self.b.insert_at_cursor('\n')
self.b.insert_with_tags_by_name(self.b.get_end_iter(), subheading, 'subheading')
self.b.insert_at_cursor('\n\n')
self.b.insert(self.b.get_end_iter(), util.remove_html_tags(description))
self.b.place_cursor(self.b.get_start_iter())
def add_track(self, episode, reporthook=None):
self.notify('status', _('Adding %s') % episode.title)
tracklist = gpod.sw_get_playlist_tracks(self.podcasts_playlist)
podcasturls = [track.podcasturl for track in tracklist]
if episode.url in podcasturls:
# Mark as played on iPod if played locally (and set podcast flags)
self.set_podcast_flags(tracklist[podcasturls.index(episode.url)],
episode)
return True
original_filename = episode.local_filename(create=False)
# The file has to exist, if we ought to transfer it, and therefore,
# local_filename(create=False) must never return None as filename
assert original_filename is not None
local_filename = original_filename
if util.calculate_size(original_filename) > self.get_free_space():
logger.error('Not enough space on %s, sync aborted...',
self.mountpoint)
d = {'episode': episode.title, 'mountpoint': self.mountpoint}
message = _(
'Error copying %(episode)s: Not enough free space on %(mountpoint)s'
)
self.errors.append(message % d)
self.cancelled = True
return False
local_filename = episode.local_filename(create=False)
(fn, extension) = os.path.splitext(local_filename)
if extension.lower().endswith('ogg'):
logger.error('Cannot copy .ogg files to iPod.')
return False
track = gpod.itdb_track_new()
# Add release time to track if episode.published has a valid value
if episode.published > 0:
try:
# libgpod>= 0.5.x uses a new timestamp format
track.time_released = gpod.itdb_time_host_to_mac(
int(episode.published))
except:
# old (pre-0.5.x) libgpod versions expect mactime, so
# we're going to manually build a good mactime timestamp here :)
#
# + 2082844800 for unixtime => mactime (1970 => 1904)
track.time_released = int(episode.published + 2082844800)
track.title = str(episode.title)
track.album = str(episode.channel.title)
track.artist = str(episode.channel.title)
track.description = str(util.remove_html_tags(episode.description))
track.podcasturl = str(episode.url)
track.podcastrss = str(episode.channel.url)
track.tracklen = get_track_length(local_filename)
track.size = os.path.getsize(local_filename)
if episode.file_type() == 'audio':
track.filetype = 'mp3'
track.mediatype = 0x00000004
elif episode.file_type() == 'video':
track.filetype = 'm4v'
track.mediatype = 0x00000006
self.set_podcast_flags(track, episode)
gpod.itdb_track_add(self.itdb, track, -1)
gpod.itdb_playlist_add_track(self.master_playlist, track, -1)
gpod.itdb_playlist_add_track(self.podcasts_playlist, track, -1)
copied = gpod.itdb_cp_track_to_ipod(track, str(local_filename), None)
reporthook(episode.file_size, 1, episode.file_size)
# If the file has been converted, delete the temporary file here
if local_filename != original_filename:
util.delete_file(local_filename)
return True
def one_line_description( self):
lines = util.remove_html_tags(self.description).strip().splitlines()
if not lines or lines[0] == '':
return _('No description available')
else:
return ' '.join(lines)
def get_tracks(self, feed):
"""Get a generator of tracks from a SC user
The generator will give you a dictionary for every
track it can find for its user."""
global CONSUMER_KEY
try:
json_url = ('https://api.soundcloud.com/users/%(user)s/%(feed)s.'
'json?consumer_key=%'
'(consumer_key)s&limit=200' % {
"user": self.get_user_id(),
"feed": feed,
"consumer_key": CONSUMER_KEY
})
logger.debug("loading %s", json_url)
json_tracks = util.urlopen(json_url).json()
tracks = [
track for track in json_tracks
if track['streamable'] or track['downloadable']
]
total_count = len(json_tracks)
if len(tracks) == 0 and total_count > 0:
logger.warning("Download of all %i %s of user %s is disabled" %
(total_count, feed, self.username))
else:
logger.info("%i/%i downloadable tracks for user %s %s feed" %
(len(tracks), total_count, self.username, feed))
for track in tracks:
# Prefer stream URL (MP3), fallback to download URL
base_url = track.get(
'stream_url'
) if track['streamable'] else track['download_url']
url = base_url + '?consumer_key=' + CONSUMER_KEY
if url not in self.cache:
try:
self.cache[url] = get_metadata(url)
except:
continue
filesize, filetype, filename = self.cache[url]
yield {
'title':
track.get('title', track.get('permalink'))
or _('Unknown track'),
'link':
track.get('permalink_url')
or 'https://soundcloud.com/' + self.username,
'description':
util.remove_html_tags(track.get('description') or ''),
'description_html':
'',
'url':
url,
'file_size':
int(filesize),
'mime_type':
filetype,
'guid':
str(track.get('permalink', track.get('id'))),
'published':
soundcloud_parsedate(track.get('created_at', None)),
}
finally:
self.commit_cache()
def new(self):
self.show_on_cover_load = True
self.gPodderChannel.set_title(self.channel.title)
self.entryTitle.set_text(self.channel.title)
self.labelURL.set_text(self.channel.url)
self.cbSkipFeedUpdate.set_active(self.channel.pause_subscription)
self.cbEnableDeviceSync.set_active(self.channel.sync_to_mp3_player)
self.section_list = Gtk.ListStore(str)
active_index = 0
for index, section in enumerate(sorted(self.sections)):
self.section_list.append([section])
if section == self.channel.section:
active_index = index
self.combo_section.set_model(self.section_list)
cell_renderer = Gtk.CellRendererText()
self.combo_section.pack_start(cell_renderer, True)
self.combo_section.add_attribute(cell_renderer, 'text', 0)
self.combo_section.set_active(active_index)
self.strategy_list = Gtk.ListStore(str, int)
active_index = 0
for index, (checked, strategy_id, strategy) in \
enumerate(self.channel.get_download_strategies()):
self.strategy_list.append([strategy, strategy_id])
if checked:
active_index = index
self.combo_strategy.set_model(self.strategy_list)
cell_renderer = Gtk.CellRendererText()
self.combo_strategy.pack_start(cell_renderer, True)
self.combo_strategy.add_attribute(cell_renderer, 'text', 0)
self.combo_strategy.set_active(active_index)
self.LabelDownloadTo.set_text(self.channel.save_dir)
self.LabelWebsite.set_text(self.channel.link)
if self.channel.auth_username:
self.FeedUsername.set_text(self.channel.auth_username)
if self.channel.auth_password:
self.FeedPassword.set_text(self.channel.auth_password)
self.cover_downloader.register('cover-available', self.cover_download_finished)
self.cover_downloader.request_cover(self.channel)
# Hide the website button if we don't have a valid URL
if not self.channel.link:
self.btn_website.hide()
b = Gtk.TextBuffer()
if self.channel._update_error:
err = '\n\nERROR: {}'.format(self.channel._update_error)
else:
err = ''
b.set_text(util.remove_html_tags(self.channel.description) + err)
self.channel_description.set_buffer(b)
# Add Drag and Drop Support
flags = Gtk.DestDefaults.ALL
targets = [Gtk.TargetEntry.new('text/uri-list', 0, 2), Gtk.TargetEntry.new('text/plain', 0, 4)]
actions = Gdk.DragAction.DEFAULT | Gdk.DragAction.COPY
self.imgCover.drag_dest_set(flags, targets, actions)
self.imgCover.connect('drag_data_received', self.drag_data_received)
border = 6
size = self.MAX_SIZE + border * 2
self.imgCover.set_size_request(size, size)
self.imgCoverEventBox.connect('button-press-event',
self.on_cover_popup_menu)
gpodder.user_extensions.on_ui_object_available('channel-gtk', self)
result = gpodder.user_extensions.on_channel_settings(self.channel)
if result:
for label, callback in result:
self.notebookChannelEditor.append_page(callback(self.channel), Gtk.Label(label))
def add_track(self, episode,reporthook=None):
self.notify('status', _('Adding %s') % episode.title)
tracklist = gpod.sw_get_playlist_tracks(self.podcasts_playlist)
podcasturls=[track.podcasturl for track in tracklist]
if episode.url in podcasturls:
# Mark as played on iPod if played locally (and set podcast flags)
self.set_podcast_flags(tracklist[podcasturls.index(episode.url)], episode)
return True
original_filename = episode.local_filename(create=False)
# The file has to exist, if we ought to transfer it, and therefore,
# local_filename(create=False) must never return None as filename
assert original_filename is not None
local_filename = original_filename
if util.calculate_size(original_filename) > self.get_free_space():
logger.error('Not enough space on %s, sync aborted...', self.mountpoint)
d = {'episode': episode.title, 'mountpoint': self.mountpoint}
message =_('Error copying %(episode)s: Not enough free space on %(mountpoint)s')
self.errors.append(message % d)
self.cancelled = True
return False
local_filename = episode.local_filename(create=False)
(fn, extension) = os.path.splitext(local_filename)
if extension.lower().endswith('ogg'):
logger.error('Cannot copy .ogg files to iPod.')
return False
track = gpod.itdb_track_new()
# Add release time to track if episode.published has a valid value
if episode.published > 0:
try:
# libgpod>= 0.5.x uses a new timestamp format
track.time_released = gpod.itdb_time_host_to_mac(int(episode.published))
except:
# old (pre-0.5.x) libgpod versions expect mactime, so
# we're going to manually build a good mactime timestamp here :)
#
# + 2082844800 for unixtime => mactime (1970 => 1904)
track.time_released = int(episode.published + 2082844800)
track.title = str(episode.title)
track.album = str(episode.channel.title)
track.artist = str(episode.channel.title)
track.description = str(util.remove_html_tags(episode.description))
track.podcasturl = str(episode.url)
track.podcastrss = str(episode.channel.url)
track.tracklen = get_track_length(local_filename)
track.size = os.path.getsize(local_filename)
if episode.file_type() == 'audio':
track.filetype = 'mp3'
track.mediatype = 0x00000004
elif episode.file_type() == 'video':
track.filetype = 'm4v'
track.mediatype = 0x00000006
self.set_podcast_flags(track, episode)
gpod.itdb_track_add(self.itdb, track, -1)
gpod.itdb_playlist_add_track(self.master_playlist, track, -1)
gpod.itdb_playlist_add_track(self.podcasts_playlist, track, -1)
copied = gpod.itdb_cp_track_to_ipod(track, str(local_filename), None)
reporthook(episode.file_size, 1, episode.file_size)
# If the file has been converted, delete the temporary file here
if local_filename != original_filename:
util.delete_file(local_filename)
return True
def find_urls(page):
# streamingData is preferable to url_encoded_fmt_stream_map
# streamingData.formats are the same as url_encoded_fmt_stream_map
# streamingData.adaptiveFormats are audio-only and video-only formats
x = parse_qs(page)
error_message = None
if 'reason' in x:
error_message = util.remove_html_tags(x['reason'][0])
elif 'player_response' in x:
player_response = json.loads(x['player_response'][0])
playabilityStatus = player_response['playabilityStatus']
if 'reason' in playabilityStatus:
error_message = util.remove_html_tags(
playabilityStatus['reason'])
elif 'liveStreamability' in playabilityStatus \
and not playabilityStatus['liveStreamability'].get('liveStreamabilityRenderer', {}).get('displayEndscreen', False):
# playabilityStatus.liveStreamability -- video is or was a live stream
# playabilityStatus.liveStreamability.liveStreamabilityRenderer.displayEndscreen -- video has ended if present
if allow_partial and 'streamingData' in player_response and 'hlsManifestUrl' in player_response[
'streamingData']:
manifest = None
url = player_response['streamingData'][
'hlsManifestUrl']
while manifest is None:
req = util.http_request(url, method='GET')
if 'location' in req.msg:
url = req.msg['location']
else:
manifest = req.read()
manifest = manifest.decode().splitlines()
urls = [line for line in manifest if line[0] != '#']
itag_re = re.compile(r'/itag/([0-9]+)/')
for url in urls:
itag = itag_re.search(url).group(1)
yield int(itag), [url, None]
return
error_message = 'live stream'
elif 'streamingData' in player_response:
# DRM videos store url inside a cipher key - not supported
if 'formats' in player_response['streamingData']:
for f in player_response['streamingData']['formats']:
if 'url' in f:
yield int(f['itag']), [
f['url'],
f.get('approxDurationMs')
]
if 'adaptiveFormats' in player_response['streamingData']:
for f in player_response['streamingData'][
'adaptiveFormats']:
if 'url' in f:
yield int(f['itag']), [
f['url'],
f.get('approxDurationMs')
]
return
if error_message is not None:
raise YouTubeError('Cannot download video: %s' % error_message)
r4 = re.search(r'url_encoded_fmt_stream_map=([^&]+)', page)
if r4 is not None:
fmt_url_map = urllib.parse.unquote(r4.group(1))
for fmt_url_encoded in fmt_url_map.split(','):
video_info = parse_qs(fmt_url_encoded)
yield int(
video_info['itag'][0]), [video_info['url'][0], None]
def new(self):
self.show_on_cover_load = True
self.gPodderChannel.set_transient_for(self.parent_widget)
self.title_label.set_text(self.channel.title)
self.labelURL.set_text(self.channel.url)
self.skip_feed_update_switch.set_active(
self.channel.pause_subscription)
self.enable_device_sync_switch.set_active(
self.channel.sync_to_mp3_player)
self.section_list = Gtk.ListStore(str)
active_index = 0
for index, section in enumerate(sorted(self.sections)):
self.section_list.append([section])
if section == self.channel.section:
active_index = index
self.combo_section.set_model(self.section_list)
cell_renderer = Gtk.CellRendererText()
self.combo_section.pack_start(cell_renderer, True)
self.combo_section.add_attribute(cell_renderer, 'text', 0)
self.combo_section.set_active(active_index)
self.strategy_list = Gtk.ListStore(str, int)
active_index = 0
for index, (checked, strategy_id, strategy) in \
enumerate(self.channel.get_download_strategies()):
self.strategy_list.append([strategy, strategy_id])
if checked:
active_index = index
self.combo_strategy.set_model(self.strategy_list)
cell_renderer = Gtk.CellRendererText()
self.combo_strategy.pack_start(cell_renderer, True)
self.combo_strategy.add_attribute(cell_renderer, 'text', 0)
self.combo_strategy.set_active(active_index)
self.LabelDownloadTo.set_text(self.channel.save_dir)
self.website_label.set_markup('<a href="{}">{}</a>'.format(
self.channel.link, self.channel.link) if self.channel.link else '')
self.website_label.connect('activate-link',
lambda label, url: util.open_website(url))
if self.channel.auth_username:
self.FeedUsername.set_text(self.channel.auth_username)
if self.channel.auth_password:
self.FeedPassword.set_text(self.channel.auth_password)
self.cover_downloader.register('cover-available',
self.cover_download_finished)
self.cover_downloader.request_cover(self.channel)
if self.channel._update_error:
err = '\n\n' + (_('ERROR: %s') % self.channel._update_error)
else:
err = ''
self.channel_description.set_text(
util.remove_html_tags(self.channel.description) + err)
# Add Drag and Drop Support
flags = Gtk.DestDefaults.ALL
targets = [
Gtk.TargetEntry.new('text/uri-list', 0, 2),
Gtk.TargetEntry.new('text/plain', 0, 4)
]
actions = Gdk.DragAction.DEFAULT | Gdk.DragAction.COPY
self.imgCover.drag_dest_set(flags, targets, actions)
self.imgCover.connect('drag_data_received', self.drag_data_received)
border = 6
size = self.MAX_SIZE + border * 2
self.imgCover.set_size_request(size, size)
self.imgCoverEventBox.connect('button-press-event',
self.on_cover_popup_menu)
# Title save button state
self.title_save_button_saves = True
self._config.connect_gtk_window(self.gPodderChannel, 'channel_editor',
True)
gpodder.user_extensions.on_ui_object_available('channel-gtk', self)
result = gpodder.user_extensions.on_channel_settings(self.channel)
if result:
for label, callback in result:
sw = Gtk.ScrolledWindow()
sw.add(callback(self.channel))
sw.show_all()
self.notebookChannelEditor.append_page(sw, Gtk.Label(label))
def find_urls(old_page, new_page):
# streamingData is preferable to url_encoded_fmt_stream_map
# streamingData.formats are the same as url_encoded_fmt_stream_map
# streamingData.adaptiveFormats are audio-only and video-only formats
x = parse_qs(old_page) if old_page else json.loads(new_page)
player_response = json.loads(
x['player_response']
[0]) if old_page and 'player_response' in x else x
error_message = None
if 'reason' in x:
# TODO: unknown if this is valid for new_page
error_message = util.remove_html_tags(x['reason'][0])
elif 'playabilityStatus' in player_response:
playabilityStatus = player_response['playabilityStatus']
if 'reason' in playabilityStatus:
error_message = util.remove_html_tags(
playabilityStatus['reason'])
elif 'liveStreamability' in playabilityStatus \
and not playabilityStatus['liveStreamability'].get('liveStreamabilityRenderer', {}).get('displayEndscreen', False):
# playabilityStatus.liveStreamability -- video is or was a live stream
# playabilityStatus.liveStreamability.liveStreamabilityRenderer.displayEndscreen -- video has ended if present
if allow_partial and 'streamingData' in player_response and 'hlsManifestUrl' in player_response[
'streamingData']:
r = util.urlopen(
player_response['streamingData']['hlsManifestUrl'])
if not r.ok:
raise YouTubeError('HLS Manifest: %d %s' %
(r.status_code, r.reason))
manifest = r.text.splitlines()
urls = [line for line in manifest if line[0] != '#']
itag_re = re.compile(r'/itag/([0-9]+)/')
for url in urls:
itag = itag_re.search(url).group(1)
yield int(itag), [url, None]
return
error_message = 'live stream'
elif 'streamingData' in player_response:
if 'formats' in player_response['streamingData']:
for f in player_response['streamingData']['formats']:
if 'url' in f: # DRM videos store url inside a signatureCipher key
yield int(f['itag']), [
f['url'],
f.get('approxDurationMs')
]
if 'adaptiveFormats' in player_response['streamingData']:
for f in player_response['streamingData'][
'adaptiveFormats']:
if 'url' in f: # DRM videos store url inside a signatureCipher key
yield int(f['itag']), [
f['url'],
f.get('approxDurationMs')
]
return
if error_message is not None:
raise YouTubeError(
('Cannot stream video: %s' if allow_partial else
'Cannot download video: %s') % error_message)
if old_page:
r4 = re.search(r'url_encoded_fmt_stream_map=([^&]+)', old_page)
if r4 is not None:
fmt_url_map = urllib.parse.unquote(r4.group(1))
for fmt_url_encoded in fmt_url_map.split(','):
video_info = parse_qs(fmt_url_encoded)
yield int(video_info['itag'][0]), [
video_info['url'][0], None
]
