def add_track(self, episode):
self.notify('status', _('Adding %s...') % episode.title)
filename = str(self.convert_track(episode))
log("sending " + filename + " (" + episode.title + ").", sender=self)
try:
# verify free space
needed = util.calculate_size(filename)
free = self.get_free_space()
if needed > free:
log('Not enough space on device %s: %s available, but need at least %s', self.get_name(), util.format_filesize(free), util.format_filesize(needed), sender=self)
self.cancelled = True
return False
# fill metadata
metadata = pymtp.LIBMTP_Track()
metadata.title = str(episode.title)
metadata.artist = str(episode.channel.title)
metadata.album = str(episode.channel.title)
metadata.genre = "podcast"
metadata.date = self.__date_to_mtp(episode.pubDate)
metadata.duration = get_track_length(str(filename))
# send the file
self.__MTPDevice.send_track_from_file(filename,
util.sanitize_filename(metadata.title)+episode.extension(),
metadata, 0, callback=self.__callback)
except:
log('unable to add episode %s', episode.title, sender=self, traceback=True)
return False
return True
def get_all_tracks(self):
tracks = []
for track in gpod.sw_get_playlist_tracks(self.podcasts_playlist):
filename = gpod.itdb_filename_on_ipod(track)
if filename is None:
# This can happen if the episode is deleted on the device
logger.info('Episode has no file: %s', track.title)
self.remove_track_gpod(track)
continue
length = util.calculate_size(filename)
timestamp = util.file_modification_timestamp(filename)
modified = util.format_date(timestamp)
try:
released = gpod.itdb_time_mac_to_host(track.time_released)
released = util.format_date(released)
except ValueError, ve:
# timestamp out of range for platform time_t (bug 418)
logger.info('Cannot convert track time: %s', ve)
released = 0
t = SyncTrack(track.title, length, modified,
modified_sort=timestamp,
libgpodtrack=track,
playcount=track.playcount,
released=released,
podcast=track.artist)
tracks.append(t)
def run(self):
# Speed calculation (re-)starts here
self.__start_time = 0
self.__start_blocks = 0
# If the download has already been cancelled/paused, skip it
with self:
if self.status in (SyncTask.CANCELLING, SyncTask.CANCELLED):
self.progress = 0.0
self.speed = 0.0
self.status = SyncTask.CANCELLED
return False
if self.status == SyncTask.PAUSING:
self.status = SyncTask.PAUSED
return False
# We only start this download if its status is downloading
if self.status != SyncTask.DOWNLOADING:
return False
# We are synching this file right now
self._notification_shown = False
sync_result = SyncTask.DOWNLOADING
try:
logger.info('Starting SyncTask')
self.device.add_track(self, reporthook=self.status_updated)
except SyncCancelledException as e:
sync_result = SyncTask.CANCELLED
except Exception as e:
sync_result = SyncTask.FAILED
logger.error('Sync failed: %s', str(e), exc_info=True)
self.error_message = _('Error: %s') % (str(e), )
with self:
if sync_result == SyncTask.DOWNLOADING:
# Everything went well - we're done
self.status = SyncTask.DONE
if self.total_size <= 0:
self.total_size = util.calculate_size(self.filename)
logger.info('Total size updated to %d', self.total_size)
self.progress = 1.0
gpodder.user_extensions.on_episode_synced(
self.device, self.__episode)
return True
self.speed = 0.0
if sync_result == SyncTask.FAILED:
self.status = SyncTask.FAILED
# cancelled/paused -- update state to mark it as safe to manipulate this task again
elif self.status == SyncTask.PAUSING:
self.status = SyncTask.PAUSED
elif self.status == SyncTask.CANCELLING:
self.status = SyncTask.CANCELLED
# We finished, but not successfully (at least not really)
return False
def get_all_tracks(self):
tracks = []
if self._config.one_folder_per_podcast:
files = glob.glob(os.path.join(self.destination, '*', '*'))
else:
files = glob.glob(os.path.join(self.destination, '*'))
for filename in files:
(title, extension) = os.path.splitext(os.path.basename(filename))
length = util.calculate_size(filename)
timestamp = util.file_modification_timestamp(filename)
modified = util.format_date(timestamp)
if self._config.one_folder_per_podcast:
podcast_name = os.path.basename(os.path.dirname(filename))
else:
podcast_name = None
t = SyncTrack(title, length, modified,
modified_sort=timestamp,
filename=filename,
podcast=podcast_name)
tracks.append(t)
return tracks
def get_all_tracks(self):
tracks = []
for track in gpod.sw_get_playlist_tracks(self.podcasts_playlist):
filename = gpod.itdb_filename_on_ipod(track)
if filename is None:
# This can happen if the episode is deleted on the device
logger.info('Episode has no file: %s', track.title)
self.remove_track_gpod(track)
continue
length = util.calculate_size(filename)
timestamp = util.file_modification_timestamp(filename)
modified = util.format_date(timestamp)
try:
released = gpod.itdb_time_mac_to_host(track.time_released)
released = util.format_date(released)
except ValueError, ve:
# timestamp out of range for platform time_t (bug 418)
logger.info('Cannot convert track time: %s', ve)
released = 0
t = SyncTrack(track.title,
length,
modified,
modified_sort=timestamp,
libgpodtrack=track,
playcount=track.playcount,
released=released,
podcast=track.artist)
tracks.append(t)
def get_all_tracks(self):
tracks = []
if self._config.one_folder_per_podcast:
files = glob.glob(os.path.join(self.destination, '*', '*'))
else:
files = glob.glob(os.path.join(self.destination, '*'))
for filename in files:
(title, extension) = os.path.splitext(os.path.basename(filename))
length = util.calculate_size(filename)
timestamp = util.file_modification_timestamp(filename)
modified = util.format_date(timestamp)
if self._config.one_folder_per_podcast:
podcast_name = os.path.basename(os.path.dirname(filename))
else:
podcast_name = None
t = SyncTrack(title,
length,
modified,
modified_sort=timestamp,
filename=filename,
podcast=podcast_name)
tracks.append(t)
return tracks
def add_track(self, episode):
self.notify('status', _('Adding %s...') % episode.title)
filename = str(self.convert_track(episode))
logger.info("sending %s (%s).", filename, episode.title)
try:
# verify free space
needed = util.calculate_size(filename)
free = self.get_free_space()
if needed > free:
logger.error('Not enough space on device %s: %s available, but '
'need at least %s',
self.get_name(),
util.format_filesize(free),
util.format_filesize(needed))
self.cancelled = True
return False
# fill metadata
metadata = pymtp.LIBMTP_Track()
metadata.title = str(episode.title)
metadata.artist = str(episode.channel.title)
metadata.album = str(episode.channel.title)
metadata.genre = "podcast"
metadata.date = self.__date_to_mtp(episode.published)
metadata.duration = get_track_length(str(filename))
folder_name = ''
if episode.mimetype.startswith('audio/') and self._config.mtp_audio_folder:
folder_name = self._config.mtp_audio_folder
if episode.mimetype.startswith('video/') and self._config.mtp_video_folder:
folder_name = self._config.mtp_video_folder
if episode.mimetype.startswith('image/') and self._config.mtp_image_folder:
folder_name = self._config.mtp_image_folder
if folder_name != '' and self._config.mtp_podcast_folders:
folder_name += os.path.sep + str(episode.channel.title)
# log('Target MTP folder: %s' % folder_name)
if folder_name == '':
folder_id = 0
else:
folder_id = self.__MTPDevice.mkdir(folder_name)
# send the file
to_file = util.sanitize_filename(metadata.title) + episode.extension()
self.__MTPDevice.send_track_from_file(filename, to_file,
metadata, folder_id, callback=self.__callback)
if gpodder.user_hooks is not None:
gpodder.user_hooks.on_file_copied_to_mtp(self, filename, to_file)
except:
logger.error('unable to add episode %s', episode.title)
return False
return True
def add_track(self, episode):
self.notify('status', _('Adding %s...') % episode.title)
filename = str(self.convert_track(episode))
log("sending %s (%s).", filename, episode.title, sender=self)
try:
# verify free space
needed = util.calculate_size(filename)
free = self.get_free_space()
if needed > free:
log('Not enough space on device %s: %s available, but need at least %s', self.get_name(), util.format_filesize(free), util.format_filesize(needed), sender=self)
self.cancelled = True
return False
# fill metadata
metadata = pymtp.LIBMTP_Track()
metadata.title = str(episode.title)
metadata.artist = str(episode.channel.title)
metadata.album = str(episode.channel.title)
metadata.genre = "podcast"
metadata.date = self.__date_to_mtp(episode.pubDate)
metadata.duration = get_track_length(str(filename))
folder_name = ''
if episode.mimetype.startswith('audio/') and self._config.mtp_audio_folder:
folder_name = self._config.mtp_audio_folder
if episode.mimetype.startswith('video/') and self._config.mtp_video_folder:
folder_name = self._config.mtp_video_folder
if episode.mimetype.startswith('image/') and self._config.mtp_image_folder:
folder_name = self._config.mtp_image_folder
if folder_name != '' and self._config.mtp_podcast_folders:
folder_name += os.path.sep + str(episode.channel.title)
# log('Target MTP folder: %s' % folder_name)
if folder_name == '':
folder_id = 0
else:
folder_id = self.__MTPDevice.mkdir(folder_name)
# send the file
to_file = util.sanitize_filename(metadata.title) + episode.extension()
self.__MTPDevice.send_track_from_file(filename, to_file,
metadata, folder_id, callback=self.__callback)
if gpodder.user_hooks is not None:
gpodder.user_hooks.on_file_copied_to_mtp(self, filename, to_file)
except:
log('unable to add episode %s', episode.title, sender=self, traceback=True)
return False
return True
def get_all_tracks(self):
tracks = []
for track in gpod.sw_get_playlist_tracks(self.podcasts_playlist):
filename = gpod.itdb_filename_on_ipod(track)
length = util.calculate_size(filename)
timestamp = util.file_modification_timestamp(filename)
modified = util.format_date(timestamp)
released = gpod.itdb_time_mac_to_host(track.time_released)
released = util.format_date(released)
t = SyncTrack(track.title, length, modified, modified_sort=timestamp, libgpodtrack=track, playcount=track.playcount, released=released, podcast=track.artist)
tracks.append(t)
return tracks
def add_track(self, task, reporthook=None):
episode = task.episode
self.notify('status', _('Adding %s') % episode.title)
# get the folder on the device
folder = self.get_episode_folder_on_device(episode)
filename = episode.local_filename(create=False)
# The file has to exist, if we ought to transfer it, and therefore,
# local_filename(create=False) must never return None as filename
assert filename is not None
from_file = filename
# verify free space
needed = util.calculate_size(from_file)
free = self.get_free_space()
if free == -1:
logger.warn('Cannot determine free disk space on device')
elif needed > free:
d = {'path': self.destination, 'free': util.format_filesize(free), 'need': util.format_filesize(needed)}
message = _('Not enough space in %(path)s: %(free)s available, but need at least %(need)s')
raise SyncFailedException(message % d)
# get the filename that will be used on the device
to_file = self.get_episode_file_on_device(episode)
to_file = folder.get_child(to_file)
util.make_directory(folder)
if not to_file.query_exists():
logger.info('Copying %s => %s',
os.path.basename(from_file),
to_file.get_uri())
from_file = Gio.File.new_for_path(from_file)
try:
def hookconvert(current_bytes, total_bytes, user_data):
return reporthook(current_bytes, 1, total_bytes)
from_file.copy(to_file, Gio.FileCopyFlags.OVERWRITE, task.cancellable, hookconvert, None)
except GLib.Error as err:
if err.matches(Gio.io_error_quark(), Gio.IOErrorEnum.CANCELLED):
raise SyncCancelledException()
logger.error('Error copying %s to %s: %s', from_file.get_uri(), to_file.get_uri(), err.message)
d = {'from_file': from_file.get_uri(), 'to_file': to_file.get_uri(), 'message': err.message}
self.errors.append(_('Error copying %(from_file)s to %(to_file)s: %(message)s') % d)
return False
return True
def add_track(self, episode, reporthook=None):
self.notify('status',
_('Adding %s') % episode.title.decode('utf-8', 'ignore'))
# get the folder on the device
folder = self.get_episode_folder_on_device(episode)
filename = episode.local_filename(create=False)
# The file has to exist, if we ought to transfer it, and therefore,
# local_filename(create=False) must never return None as filename
assert filename is not None
from_file = util.sanitize_encoding(filename)
# verify free space
needed = util.calculate_size(from_file)
free = self.get_free_space()
if free == -1:
logger.warn('Cannot determine free disk space on device')
elif needed > free:
d = {
'path': self.destination,
'free': util.format_filesize(free),
'need': util.format_filesize(needed)
}
message = _(
'Not enough space in %(path)s: %(free)s available, but need at least %(need)s'
)
raise SyncFailedException(message % d)
# get the filename that will be used on the device
to_file = self.get_episode_file_on_device(episode)
to_file = util.sanitize_encoding(os.path.join(folder, to_file))
if not os.path.exists(folder):
try:
os.makedirs(folder)
except:
logger.error('Cannot create folder on MP3 player: %s', folder)
return False
if not os.path.exists(to_file):
logger.info('Copying %s => %s', os.path.basename(from_file),
to_file.decode(util.encoding))
self.copy_file_progress(from_file, to_file, reporthook)
return True
def add_track(self, task, reporthook=None):
episode = task.episode
self.notify('status', _('Adding %s') % episode.title)
tracklist = self.ipod.get_podcast_tracks()
episode_urls = [track.podcast_url for track in tracklist]
if episode.url in episode_urls:
# Mark as played on iPod if played locally (and set podcast flags)
self.update_from_episode(
tracklist[episode_urls.index(episode.url)], episode)
return True
local_filename = episode.local_filename(create=False)
# The file has to exist, if we ought to transfer it, and therefore,
# local_filename(create=False) must never return None as filename
assert local_filename is not None
if util.calculate_size(local_filename) > self.get_free_space():
logger.error('Not enough space on %s, sync aborted...',
self.mountpoint)
d = {'episode': episode.title, 'mountpoint': self.mountpoint}
message = _(
'Error copying %(episode)s: Not enough free space on %(mountpoint)s'
)
self.errors.append(message % d)
self.cancelled = True
return False
(fn, extension) = os.path.splitext(local_filename)
if extension.lower().endswith('ogg'):
# XXX: Proper file extension/format support check for iPod
logger.error('Cannot copy .ogg files to iPod.')
return False
track = self.ipod.add_track(local_filename, episode.title,
episode.channel.title,
episode._text_description, episode.url,
episode.channel.url, episode.published,
get_track_length(local_filename),
episode.file_type() == 'audio')
self.update_from_episode(track, episode, initial=True)
reporthook(episode.file_size, 1, episode.file_size)
return True
def run(self):
# Speed calculation (re-)starts here
self.__start_time = 0
self.__start_blocks = 0
# If the download has already been cancelled, skip it
if self.status == SyncTask.CANCELLED:
util.delete_file(self.tempname)
self.progress = 0.0
self.speed = 0.0
return False
# We only start this download if its status is "downloading"
if self.status != SyncTask.DOWNLOADING:
return False
# We are synching this file right now
self.status = SyncTask.DOWNLOADING
self._notification_shown = False
try:
logger.info('Starting SyncTask')
self.device.add_track(self.episode, reporthook=self.status_updated)
except Exception as e:
self.status = SyncTask.FAILED
logger.error('Sync failed: %s', str(e), exc_info=True)
self.error_message = _('Error: %s') % (str(e), )
if self.status == SyncTask.DOWNLOADING:
# Everything went well - we're done
self.status = SyncTask.DONE
if self.total_size <= 0:
self.total_size = util.calculate_size(self.filename)
logger.info('Total size updated to %d', self.total_size)
self.progress = 1.0
gpodder.user_extensions.on_episode_synced(self.device,
self.__episode)
return True
self.speed = 0.0
# We finished, but not successfully (at least not really)
return False
def run(self):
# Speed calculation (re-)starts here
self.__start_time = 0
self.__start_blocks = 0
# If the download has already been cancelled, skip it
if self.status == SyncTask.CANCELLED:
util.delete_file(self.tempname)
self.progress = 0.0
self.speed = 0.0
return False
# We only start this download if its status is "downloading"
if self.status != SyncTask.DOWNLOADING:
return False
# We are synching this file right now
self.status = SyncTask.DOWNLOADING
self._notification_shown = False
try:
logger.info('Starting SyncTask')
self.device.add_track(self.episode, reporthook=self.status_updated)
except Exception as e:
self.status = SyncTask.FAILED
logger.error('Sync failed: %s', str(e), exc_info=True)
self.error_message = _('Error: %s') % (str(e),)
if self.status == SyncTask.DOWNLOADING:
# Everything went well - we're done
self.status = SyncTask.DONE
if self.total_size <= 0:
self.total_size = util.calculate_size(self.filename)
logger.info('Total size updated to %d', self.total_size)
self.progress = 1.0
gpodder.user_extensions.on_episode_synced(self.device, self.__episode)
return True
self.speed = 0.0
# We finished, but not successfully (at least not really)
return False
def add_track(self, episode,reporthook=None):
self.notify('status', _('Adding %s') % episode.title.decode('utf-8', 'ignore'))
# get the folder on the device
folder = self.get_episode_folder_on_device(episode)
filename = episode.local_filename(create=False)
# The file has to exist, if we ought to transfer it, and therefore,
# local_filename(create=False) must never return None as filename
assert filename is not None
from_file = util.sanitize_encoding(filename)
# verify free space
needed = util.calculate_size(from_file)
free = self.get_free_space()
if free == -1:
logger.warn('Cannot determine free disk space on device')
elif needed > free:
d = {'path': self.destination, 'free': util.format_filesize(free), 'need': util.format_filesize(needed)}
message =_('Not enough space in %(path)s: %(free)s available, but need at least %(need)s')
raise SyncFailedException(message % d)
# get the filename that will be used on the device
to_file = self.get_episode_file_on_device(episode)
to_file = util.sanitize_encoding(os.path.join(folder, to_file))
if not os.path.exists(folder):
try:
os.makedirs(folder)
except:
logger.error('Cannot create folder on MP3 player: %s', folder)
return False
if not os.path.exists(to_file):
logger.info('Copying %s => %s',
os.path.basename(from_file),
to_file.decode(util.encoding))
self.copy_file_progress(from_file, to_file, reporthook)
return True
def get_all_tracks(self):
tracks = []
for track in self.ipod.get_podcast_tracks():
filename = track.filename_on_ipod
if filename is None:
length = 0
modified = ''
else:
length = util.calculate_size(filename)
timestamp = util.file_modification_timestamp(filename)
modified = util.format_date(timestamp)
t = SyncTrack(track.episode_title,
length,
modified,
ipod_track=track,
playcount=track.playcount,
podcast=track.podcast_title)
tracks.append(t)
return tracks
def get_all_tracks(self):
tracks=[]
if gl.config.fssync_channel_subfolders:
files=glob.glob(os.path.join(self.destination, '*', '*'))
else:
files=glob.glob(os.path.join(self.destination, '*'))
for filename in files:
(title, extension)=os.path.splitext(os.path.basename(filename))
length=util.calculate_size(filename)
age_in_days=util.file_age_in_days(filename)
modified=util.file_age_to_string(age_in_days)
if gl.config.fssync_channel_subfolders:
podcast_name=os.path.basename(os.path.dirname(filename))
else:
podcast_name=None
t=SyncTrack(title, length, modified, filename=filename, podcast=podcast_name)
tracks.append(t)
return tracks
def add_track(self, episode, reporthook=None):
self.notify('status', _('Adding %s') % episode.title)
tracklist = gpod.sw_get_playlist_tracks(self.podcasts_playlist)
podcasturls = [track.podcasturl for track in tracklist]
if episode.url in podcasturls:
# Mark as played on iPod if played locally (and set podcast flags)
self.set_podcast_flags(tracklist[podcasturls.index(episode.url)],
episode)
return True
original_filename = episode.local_filename(create=False)
# The file has to exist, if we ought to transfer it, and therefore,
# local_filename(create=False) must never return None as filename
assert original_filename is not None
local_filename = original_filename
if util.calculate_size(original_filename) > self.get_free_space():
logger.error('Not enough space on %s, sync aborted...',
self.mountpoint)
d = {'episode': episode.title, 'mountpoint': self.mountpoint}
message = _(
'Error copying %(episode)s: Not enough free space on %(mountpoint)s'
)
self.errors.append(message % d)
self.cancelled = True
return False
local_filename = episode.local_filename(create=False)
(fn, extension) = os.path.splitext(local_filename)
if extension.lower().endswith('ogg'):
logger.error('Cannot copy .ogg files to iPod.')
return False
track = gpod.itdb_track_new()
# Add release time to track if episode.published has a valid value
if episode.published > 0:
try:
# libgpod>= 0.5.x uses a new timestamp format
track.time_released = gpod.itdb_time_host_to_mac(
int(episode.published))
except:
# old (pre-0.5.x) libgpod versions expect mactime, so
# we're going to manually build a good mactime timestamp here :)
#
# + 2082844800 for unixtime => mactime (1970 => 1904)
track.time_released = int(episode.published + 2082844800)
track.title = str(episode.title)
track.album = str(episode.channel.title)
track.artist = str(episode.channel.title)
track.description = str(util.remove_html_tags(episode.description))
track.podcasturl = str(episode.url)
track.podcastrss = str(episode.channel.url)
track.tracklen = get_track_length(local_filename)
track.size = os.path.getsize(local_filename)
if episode.file_type() == 'audio':
track.filetype = 'mp3'
track.mediatype = 0x00000004
elif episode.file_type() == 'video':
track.filetype = 'm4v'
track.mediatype = 0x00000006
self.set_podcast_flags(track, episode)
gpod.itdb_track_add(self.itdb, track, -1)
gpod.itdb_playlist_add_track(self.master_playlist, track, -1)
gpod.itdb_playlist_add_track(self.podcasts_playlist, track, -1)
copied = gpod.itdb_cp_track_to_ipod(track, str(local_filename), None)
reporthook(episode.file_size, 1, episode.file_size)
# If the file has been converted, delete the temporary file here
if local_filename != original_filename:
util.delete_file(local_filename)
return True
def update_save_dir_size(self):
self.save_dir_size = util.calculate_size(self._config.download_dir)
def run(self):
# Speed calculation (re-)starts here
self.__start_time = 0
self.__start_blocks = 0
# If the download has already been cancelled, skip it
if self.status == DownloadTask.CANCELLED:
util.delete_file(self.tempname)
self.progress = 0.0
self.speed = 0.0
self.recycle()
return False
# We only start this download if its status is "downloading"
if self.status != DownloadTask.DOWNLOADING:
return False
# We are downloading this file right now
self.status = DownloadTask.DOWNLOADING
self._notification_shown = False
# Restore a reference to this task in the episode
# when running a recycled task following a pause or failed
# see #649
if not self.episode.download_task:
self.episode.download_task = self
url = self.__episode.url
try:
if url == '':
raise DownloadNoURLException()
if self.downloader:
downloader = self.downloader.custom_downloader(
self._config, self.episode)
else:
downloader = registry.custom_downloader.resolve(
self._config, None, self.episode)
if downloader:
logger.info('Downloading %s with %s', url, downloader)
else:
downloader = DefaultDownloader.custom_downloader(
self._config, self.episode)
headers, real_url = downloader.retrieve_resume(
self.tempname, self.status_updated)
new_mimetype = headers.get('content-type',
self.__episode.mime_type)
old_mimetype = self.__episode.mime_type
_basename, ext = os.path.splitext(self.filename)
if new_mimetype != old_mimetype or util.wrong_extension(ext):
logger.info('Updating mime type: %s => %s', old_mimetype,
new_mimetype)
old_extension = self.__episode.extension()
self.__episode.mime_type = new_mimetype
# don't call local_filename because we'll get the old download name
new_extension = self.__episode.extension(
may_call_local_filename=False)
# If the desired filename extension changed due to the new
# mimetype, we force an update of the local filename to fix the
# extension.
if old_extension != new_extension or util.wrong_extension(ext):
self.filename = self.__episode.local_filename(
create=True, force_update=True)
# In some cases, the redirect of a URL causes the real filename to
# be revealed in the final URL (e.g. http://gpodder.org/bug/1423)
if real_url != url and not util.is_known_redirecter(real_url):
realname, realext = util.filename_from_url(real_url)
# Only update from redirect if the redirected-to filename has
# a proper extension (this is needed for e.g. YouTube)
if not util.wrong_extension(realext):
real_filename = ''.join((realname, realext))
self.filename = self.__episode.local_filename(
create=True, force_update=True, template=real_filename)
logger.info(
'Download was redirected (%s). New filename: %s',
real_url, os.path.basename(self.filename))
# Look at the Content-disposition header; use if if available
disposition_filename = util.get_header_param(
headers, 'filename', 'content-disposition')
# Some servers do send the content-disposition header, but provide
# an empty filename, resulting in an empty string here (bug 1440)
if disposition_filename is not None and disposition_filename != '':
# The server specifies a download filename - try to use it
# filename_from_url to remove query string; see #591
fn, ext = util.filename_from_url(disposition_filename)
logger.debug(
"converting disposition filename '%s' to local filename '%s%s'",
disposition_filename, fn, ext)
disposition_filename = fn + ext
self.filename = self.__episode.local_filename(
create=True,
force_update=True,
template=disposition_filename)
new_mimetype, encoding = mimetypes.guess_type(self.filename)
if new_mimetype is not None:
logger.info('Using content-disposition mimetype: %s',
new_mimetype)
self.__episode.mime_type = new_mimetype
# Re-evaluate filename and tempname to take care of podcast renames
# while downloads are running (which will change both file names)
self.filename = self.__episode.local_filename(create=False)
self.tempname = os.path.join(os.path.dirname(self.filename),
os.path.basename(self.tempname))
shutil.move(self.tempname, self.filename)
# Model- and database-related updates after a download has finished
self.__episode.on_downloaded(self.filename)
except DownloadCancelledException:
logger.info('Download has been cancelled/paused: %s', self)
if self.status == DownloadTask.CANCELLED:
util.delete_file(self.tempname)
self.progress = 0.0
self.speed = 0.0
except DownloadNoURLException:
self.status = DownloadTask.FAILED
self.error_message = _('Episode has no URL to download')
except urllib.error.ContentTooShortError as ctse:
self.status = DownloadTask.FAILED
self.error_message = _('Missing content from server')
except ConnectionError as ce:
# special case request exception
self.status = DownloadTask.FAILED
logger.error('Download failed: %s', str(ce), exc_info=True)
d = {'host': ce.args[0].pool.host, 'port': ce.args[0].pool.port}
self.error_message = _(
"Couldn't connect to server %(host)s:%(port)s" % d)
except RequestException as re:
# extract MaxRetryError to shorten the exception message
if isinstance(re.args[0], MaxRetryError):
re = re.args[0]
logger.error('%s while downloading "%s"',
str(re),
self.__episode.title,
exc_info=True)
self.status = DownloadTask.FAILED
d = {'error': str(re)}
self.error_message = _('Request Error: %(error)s') % d
except IOError as ioe:
logger.error('%s while downloading "%s": %s',
ioe.strerror,
self.__episode.title,
ioe.filename,
exc_info=True)
self.status = DownloadTask.FAILED
d = {'error': ioe.strerror, 'filename': ioe.filename}
self.error_message = _('I/O Error: %(error)s: %(filename)s') % d
except gPodderDownloadHTTPError as gdhe:
logger.error('HTTP %s while downloading "%s": %s',
gdhe.error_code,
self.__episode.title,
gdhe.error_message,
exc_info=True)
self.status = DownloadTask.FAILED
d = {'code': gdhe.error_code, 'message': gdhe.error_message}
self.error_message = _('HTTP Error %(code)s: %(message)s') % d
except Exception as e:
self.status = DownloadTask.FAILED
logger.error('Download failed: %s', str(e), exc_info=True)
self.error_message = _('Error: %s') % (str(e), )
if self.status == DownloadTask.DOWNLOADING:
# Everything went well - we're done
self.status = DownloadTask.DONE
if self.total_size <= 0:
self.total_size = util.calculate_size(self.filename)
logger.info('Total size updated to %d', self.total_size)
self.progress = 1.0
gpodder.user_extensions.on_episode_downloaded(self.__episode)
return True
self.speed = 0.0
# We finished, but not successfully (at least not really)
return False
def update_save_dir_size(self):
self.save_dir_size = util.calculate_size(self._config.download_dir)
d = {'error': ioe.strerror, 'filename': ioe.filename}
self.error_message = _('I/O Error: %(error)s: %(filename)s') % d
except gPodderDownloadHTTPError, gdhe:
logger.error('HTTP %s while downloading "%s": %s',
gdhe.error_code,
self.__episode.title,
gdhe.error_message,
exc_info=True)
self.status = DownloadTask.FAILED
d = {'code': gdhe.error_code, 'message': gdhe.error_message}
self.error_message = _('HTTP Error %(code)s: %(message)s') % d
except Exception, e:
self.status = DownloadTask.FAILED
logger.error('Download failed: %s', str(e), exc_info=True)
self.error_message = _('Error: %s') % (str(e), )
if self.status == DownloadTask.DOWNLOADING:
# Everything went well - we're done
self.status = DownloadTask.DONE
if self.total_size <= 0:
self.total_size = util.calculate_size(self.filename)
logger.info('Total size updated to %d', self.total_size)
self.progress = 1.0
gpodder.user_extensions.on_episode_downloaded(self.__episode)
return True
self.speed = 0.0
# We finished, but not successfully (at least not really)
return False
def file_size(episode):
filename = episode.local_filename(create=False)
if filename is None:
return 0
return util.calculate_size(str(filename))
def add_track(self, episode, reporthook=None):
self.notify('status', _('Adding %s') % episode.title)
# episode.description
# episode.description_html
# file = episode.local_filename(create=False)
# episode.link
# episode.title
# cover
#
#bb =io.BytesIO(episode.channel.cover)
folder = self.get_episode_folder_on_device(episode)
filename = episode.local_filename(create=False)
# The file has to exist, if we ought to transfer it, and therefore,
# local_filename(create=False) must never return None as filename
assert filename is not None
from_file = filename
# verify free space
needed = util.calculate_size(from_file)
free = self.get_free_space()
if free == -1:
logger.warn('Cannot determine free disk space on device')
elif needed > free:
d = {
'path': self.destination,
'free': util.format_filesize(free),
'need': util.format_filesize(needed)
}
message = _(
'Not enough space in %(path)s: %(free)s available, but need at least %(need)s'
)
raise SyncFailedException(message % d)
# get the filename that will be used on the device
to_file = self.get_episode_file_on_device(episode)
to_file_name = to_file
to_file = os.path.join(folder, to_file)
video_file = to_file.replace(".mp3", "").replace(".m4a", "") + ".avi"
cmd_file = to_file.replace(".mp3", "").replace(".m4a", "") + ".ps1"
image_file = to_file.replace(".mp3", "").replace(".m4a", "") + ".png"
jpg_file = episode.channel.cover_file + ".jpg"
if not os.path.exists(image_file):
img = Image.open(jpg_file)
draw = ImageDraw.Draw(img)
font = ImageFont.truetype(
"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 72)
draw.text((0, 0), episode.title, (255, 255, 255), font=font)
font = ImageFont.truetype(
"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 42)
yy = draw.multiline_text((0, 120),
episode.description, (255, 255, 255),
font=font)
img.save(image_file) # write the image
if not os.path.exists(folder):
try:
os.makedirs(folder)
except:
logger.error('Cannot create folder on MP3 player: %s', folder)
return False
if not os.path.exists(to_file):
logger.info('Copying %s => %s', os.path.basename(from_file),
to_file)
self.copy_file_progress(from_file, to_file, reporthook)
#ffmpeg -i "$1" -acodec libfaac -ac 2 -ab 128k -s 480x320
#-vcodec libx264 -vpre libx264-hq -vpre libx264-ipod640 -b 768k -bt 512k -aspect 3:2 -threads 0 -f mp4 $1.mp4
cmd = [
'ffmpeg', '-framerate', '2', '-loop', '1', '-i', image_file,
"-i", to_file, "-qmin", "3", "-qmax", "5", "-ac", "2", "-ab",
"128k", "-b:v", "512k", "-aspect", "3:2", "-threads", "4",
"-s", "480x320", "-ar", "16000", "-b:a", "44K", "-vbr",
"constrained", "-vf", "scale=iw/4:ih/4", "-profile:v",
"baseline", "-preset", "ultrafast", "-movflags", "+faststart",
"-crf", "16", "-c:v", "libx264", "-c:a", "libmp3lame",
"-reset_timestamps", "1", "-tune", "stillimage", "-pix_fmt",
"yuv420p", "-shortest", video_file
#"'{}'".format(video_file)
]
#ffmpeg -i test.mpg -c:v libx264 test.avi
cmd = subprocess.list2cmdline(cmd)
if not os.path.exists(cmd_file):
with open(cmd_file, "w") as fo:
fo.write(cmd.replace(folder + "/", ""))
if not os.path.exists(video_file):
check_call(shlex.split(cmd))
# now upload
#loop = asyncio.get_event_loop()
#loop.run_until_complete(run_bcupload(video_file, image_file, episode))
#loop.close()
return True
def run(self):
# Speed calculation (re-)starts here
self.__start_time = 0
self.__start_blocks = 0
# If the download has already been cancelled, skip it
if self.status == DownloadTask.CANCELLED:
util.delete_file(self.tempname)
self.progress = 0.0
self.speed = 0.0
return False
# We only start this download if its status is "downloading"
if self.status != DownloadTask.DOWNLOADING:
return False
# We are downloading this file right now
self.status = DownloadTask.DOWNLOADING
self._notification_shown = False
try:
# Resolve URL and start downloading the episode
fmt_ids = youtube.get_fmt_ids(self._config.youtube)
url = youtube.get_real_download_url(self.__episode.url, fmt_ids)
url = vimeo.get_real_download_url(url,
self._config.vimeo.fileformat)
url = escapist_videos.get_real_download_url(url)
url = url.strip()
# Properly escapes Unicode characters in the URL path section
# TODO: Explore if this should also handle the domain
# Based on: http://stackoverflow.com/a/18269491/1072626
# In response to issue: https://github.com/gpodder/gpodder/issues/232
def iri_to_url(url):
url = urllib.parse.urlsplit(url)
url = list(url)
# First unquote to avoid escaping quoted content
url[2] = urllib.parse.unquote(url[2])
url[2] = urllib.parse.quote(url[2])
url = urllib.parse.urlunsplit(url)
return url
url = iri_to_url(url)
downloader = DownloadURLOpener(self.__episode.channel)
# HTTP Status codes for which we retry the download
retry_codes = (408, 418, 504, 598, 599)
max_retries = max(0, self._config.auto.retries)
# Retry the download on timeout (bug 1013)
for retry in range(max_retries + 1):
if retry > 0:
logger.info('Retrying download of %s (%d)', url, retry)
time.sleep(1)
try:
headers, real_url = downloader.retrieve_resume(
url, self.tempname, reporthook=self.status_updated)
# If we arrive here, the download was successful
break
except urllib.error.ContentTooShortError as ctse:
if retry < max_retries:
logger.info('Content too short: %s - will retry.', url)
continue
raise
except socket.timeout as tmout:
if retry < max_retries:
logger.info('Socket timeout: %s - will retry.', url)
continue
raise
except gPodderDownloadHTTPError as http:
if retry < max_retries and http.error_code in retry_codes:
logger.info('HTTP error %d: %s - will retry.',
http.error_code, url)
continue
raise
new_mimetype = headers.get('content-type',
self.__episode.mime_type)
old_mimetype = self.__episode.mime_type
_basename, ext = os.path.splitext(self.filename)
if new_mimetype != old_mimetype or util.wrong_extension(ext):
logger.info('Updating mime type: %s => %s', old_mimetype,
new_mimetype)
old_extension = self.__episode.extension()
self.__episode.mime_type = new_mimetype
new_extension = self.__episode.extension()
# If the desired filename extension changed due to the new
# mimetype, we force an update of the local filename to fix the
# extension.
if old_extension != new_extension or util.wrong_extension(ext):
self.filename = self.__episode.local_filename(
create=True, force_update=True)
# In some cases, the redirect of a URL causes the real filename to
# be revealed in the final URL (e.g. http://gpodder.org/bug/1423)
if real_url != url and not util.is_known_redirecter(real_url):
realname, realext = util.filename_from_url(real_url)
# Only update from redirect if the redirected-to filename has
# a proper extension (this is needed for e.g. YouTube)
if not util.wrong_extension(realext):
real_filename = ''.join((realname, realext))
self.filename = self.__episode.local_filename(
create=True, force_update=True, template=real_filename)
logger.info(
'Download was redirected (%s). New filename: %s',
real_url, os.path.basename(self.filename))
# Look at the Content-disposition header; use if if available
disposition_filename = get_header_param(headers, 'filename',
'content-disposition')
if disposition_filename is not None:
try:
disposition_filename.decode('ascii')
except:
logger.warn(
'Content-disposition header contains non-ASCII characters - ignoring'
)
disposition_filename = None
# Some servers do send the content-disposition header, but provide
# an empty filename, resulting in an empty string here (bug 1440)
if disposition_filename is not None and disposition_filename != '':
# The server specifies a download filename - try to use it
disposition_filename = os.path.basename(disposition_filename)
self.filename = self.__episode.local_filename(create=True, \
force_update=True, template=disposition_filename)
new_mimetype, encoding = mimetypes.guess_type(self.filename)
if new_mimetype is not None:
logger.info('Using content-disposition mimetype: %s',
new_mimetype)
self.__episode.mime_type = new_mimetype
# Re-evaluate filename and tempname to take care of podcast renames
# while downloads are running (which will change both file names)
self.filename = self.__episode.local_filename(create=False)
self.tempname = os.path.join(os.path.dirname(self.filename),
os.path.basename(self.tempname))
shutil.move(self.tempname, self.filename)
# Model- and database-related updates after a download has finished
self.__episode.on_downloaded(self.filename)
except DownloadCancelledException:
logger.info('Download has been cancelled/paused: %s', self)
if self.status == DownloadTask.CANCELLED:
util.delete_file(self.tempname)
self.progress = 0.0
self.speed = 0.0
except urllib.error.ContentTooShortError as ctse:
self.status = DownloadTask.FAILED
self.error_message = _('Missing content from server')
except IOError as ioe:
logger.error('%s while downloading "%s": %s',
ioe.strerror,
self.__episode.title,
ioe.filename,
exc_info=True)
self.status = DownloadTask.FAILED
d = {'error': ioe.strerror, 'filename': ioe.filename}
self.error_message = _('I/O Error: %(error)s: %(filename)s') % d
except gPodderDownloadHTTPError as gdhe:
logger.error('HTTP %s while downloading "%s": %s',
gdhe.error_code,
self.__episode.title,
gdhe.error_message,
exc_info=True)
self.status = DownloadTask.FAILED
d = {'code': gdhe.error_code, 'message': gdhe.error_message}
self.error_message = _('HTTP Error %(code)s: %(message)s') % d
except Exception as e:
self.status = DownloadTask.FAILED
logger.error('Download failed: %s', str(e), exc_info=True)
self.error_message = _('Error: %s') % (str(e), )
if self.status == DownloadTask.DOWNLOADING:
# Everything went well - we're done
self.status = DownloadTask.DONE
if self.total_size <= 0:
self.total_size = util.calculate_size(self.filename)
logger.info('Total size updated to %d', self.total_size)
self.progress = 1.0
gpodder.user_extensions.on_episode_downloaded(self.__episode)
return True
self.speed = 0.0
# We finished, but not successfully (at least not really)
return False
def add_track(self, episode,reporthook=None):
self.notify('status', _('Adding %s') % episode.title)
tracklist = gpod.sw_get_playlist_tracks(self.podcasts_playlist)
podcasturls=[track.podcasturl for track in tracklist]
if episode.url in podcasturls:
# Mark as played on iPod if played locally (and set podcast flags)
self.set_podcast_flags(tracklist[podcasturls.index(episode.url)], episode)
return True
original_filename = episode.local_filename(create=False)
# The file has to exist, if we ought to transfer it, and therefore,
# local_filename(create=False) must never return None as filename
assert original_filename is not None
local_filename = original_filename
if util.calculate_size(original_filename) > self.get_free_space():
logger.error('Not enough space on %s, sync aborted...', self.mountpoint)
d = {'episode': episode.title, 'mountpoint': self.mountpoint}
message =_('Error copying %(episode)s: Not enough free space on %(mountpoint)s')
self.errors.append(message % d)
self.cancelled = True
return False
local_filename = episode.local_filename(create=False)
(fn, extension) = os.path.splitext(local_filename)
if extension.lower().endswith('ogg'):
logger.error('Cannot copy .ogg files to iPod.')
return False
track = gpod.itdb_track_new()
# Add release time to track if episode.published has a valid value
if episode.published > 0:
try:
# libgpod>= 0.5.x uses a new timestamp format
track.time_released = gpod.itdb_time_host_to_mac(int(episode.published))
except:
# old (pre-0.5.x) libgpod versions expect mactime, so
# we're going to manually build a good mactime timestamp here :)
#
# + 2082844800 for unixtime => mactime (1970 => 1904)
track.time_released = int(episode.published + 2082844800)
track.title = str(episode.title)
track.album = str(episode.channel.title)
track.artist = str(episode.channel.title)
track.description = str(util.remove_html_tags(episode.description))
track.podcasturl = str(episode.url)
track.podcastrss = str(episode.channel.url)
track.tracklen = get_track_length(local_filename)
track.size = os.path.getsize(local_filename)
if episode.file_type() == 'audio':
track.filetype = 'mp3'
track.mediatype = 0x00000004
elif episode.file_type() == 'video':
track.filetype = 'm4v'
track.mediatype = 0x00000006
self.set_podcast_flags(track, episode)
gpod.itdb_track_add(self.itdb, track, -1)
gpod.itdb_playlist_add_track(self.master_playlist, track, -1)
gpod.itdb_playlist_add_track(self.podcasts_playlist, track, -1)
copied = gpod.itdb_cp_track_to_ipod(track, str(local_filename), None)
reporthook(episode.file_size, 1, episode.file_size)
# If the file has been converted, delete the temporary file here
if local_filename != original_filename:
util.delete_file(local_filename)
return True
def add_track(self, episode):
self.notify('status', _('Adding %s') % episode.title)
for track in gpod.sw_get_playlist_tracks(self.podcasts_playlist):
if episode.url == track.podcasturl:
if track.playcount > 0:
gl.history_mark_played(track.podcasturl)
# Mark as played on iPod if played locally (and set podcast flags)
self.set_podcast_flags(track)
return True
original_filename=str(episode.local_filename())
local_filename=original_filename
# Reserve 10 MiB for iTunesDB writing (to be on the safe side)
RESERVED_FOR_ITDB=1024*1024*10
space_for_track=util.get_free_disk_space(self.mountpoint) - RESERVED_FOR_ITDB
needed=util.calculate_size(local_filename)
if needed > space_for_track:
log('Not enough space on %s: %s available, but need at least %s', self.mountpoint, util.format_filesize(space_for_track), util.format_filesize(needed), sender=self)
self.errors.append( _('Error copying %s: Not enough free disk space on %s') % (episode.title, self.mountpoint))
self.cancelled=True
return False
(fn, extension)=os.path.splitext(original_filename)
if libconverter.converters.has_converter(extension):
log('Converting: %s', original_filename, sender=self)
callback_status=lambda percentage: self.notify('sub-progress', int(percentage))
local_filename=libconverter.converters.convert(original_filename, callback=callback_status)
if not libtagupdate.update_metadata_on_file(local_filename, title=episode.title, artist=episode.channel.title):
log('Could not set metadata on converted file %s', local_filename, sender=self)
if local_filename is None:
log('Cannot convert %s', original_filename, sender=self)
return False
else:
local_filename=str(local_filename)
(fn, extension)=os.path.splitext(local_filename)
if extension.lower().endswith('ogg'):
log('Cannot copy .ogg files to iPod.', sender=self)
return False
track=gpod.itdb_track_new()
# Add release time to track if pubDate is parseable
ipod_date=email.Utils.parsedate(episode.pubDate)
if ipod_date is not None:
try:
# libgpod>= 0.5.x uses a new timestamp format
track.time_released=gpod.itdb_time_host_to_mac(int(time.mktime(ipod_date)))
except:
# old (pre-0.5.x) libgpod versions expect mactime, so
# we're going to manually build a good mactime timestamp here :)
#
# + 2082844800 for unixtime => mactime (1970 => 1904)
track.time_released=int(time.mktime(ipod_date) + 2082844800)
track.title=str(episode.title)
track.album=str(episode.channel.title)
track.artist=str(episode.channel.title)
track.description=str(episode.description)
track.podcasturl=str(episode.url)
track.podcastrss=str(episode.channel.url)
track.tracklen=get_track_length(local_filename)
track.size=os.path.getsize(local_filename)
if episode.file_type() == 'audio':
track.filetype='mp3'
track.mediatype=0x00000004
elif episode.file_type() == 'video':
track.filetype='m4v'
track.mediatype=0x00000006
self.set_podcast_flags(track)
self.set_cover_art(track, local_filename)
gpod.itdb_track_add(self.itdb, track, -1)
gpod.itdb_playlist_add_track(self.podcasts_playlist, track, -1)
gpod.itdb_cp_track_to_ipod( track, local_filename, None)
# If the file has been converted, delete the temporary file here
if local_filename != original_filename:
util.delete_file(local_filename)
return True
def file_size(episode):
filename = episode.local_filename(create=False)
if filename is None:
return 0
return util.calculate_size(str(filename))
def run(self):
# Speed calculation (re-)starts here
self.__start_time = 0
self.__start_blocks = 0
# If the download has already been cancelled, skip it
if self.status == DownloadTask.CANCELLED:
util.delete_file(self.tempname)
self.progress = 0.0
self.speed = 0.0
return False
# We only start this download if its status is "downloading"
if self.status != DownloadTask.DOWNLOADING:
return False
# We are downloading this file right now
self.status = DownloadTask.DOWNLOADING
self._notification_shown = False
# Restore a reference to this task in the episode
# when running a recycled task following a pause or failed
# see #649
if not self.episode.download_task:
self.episode.download_task = self
try:
custom_downloader = registry.custom_downloader.resolve(
self._config, None, self.episode)
url = self.__episode.url
if custom_downloader:
logger.info('Downloading %s with %s', url, custom_downloader)
headers, real_url = custom_downloader.retrieve_resume(
self.tempname, reporthook=self.status_updated)
else:
# Resolve URL and start downloading the episode
res = registry.download_url.resolve(self._config, None,
self.episode)
if res:
url = res
if url == self.__episode.url:
# don't modify custom urls (#635 - vimeo breaks if * is unescaped)
url = url.strip()
url = util.iri_to_url(url)
logger.info("Downloading %s", url)
downloader = DownloadURLOpener(self.__episode.channel)
# HTTP Status codes for which we retry the download
retry_codes = (408, 418, 504, 598, 599)
max_retries = max(0, self._config.auto.retries)
# Retry the download on timeout (bug 1013)
for retry in range(max_retries + 1):
if retry > 0:
logger.info('Retrying download of %s (%d)', url, retry)
time.sleep(1)
try:
headers, real_url = downloader.retrieve_resume(
url, self.tempname, reporthook=self.status_updated)
# If we arrive here, the download was successful
break
except urllib.error.ContentTooShortError as ctse:
if retry < max_retries:
logger.info('Content too short: %s - will retry.',
url)
continue
raise
except socket.timeout as tmout:
if retry < max_retries:
logger.info('Socket timeout: %s - will retry.',
url)
continue
raise
except gPodderDownloadHTTPError as http:
if retry < max_retries and http.error_code in retry_codes:
logger.info('HTTP error %d: %s - will retry.',
http.error_code, url)
continue
raise
new_mimetype = headers.get('content-type',
self.__episode.mime_type)
old_mimetype = self.__episode.mime_type
_basename, ext = os.path.splitext(self.filename)
if new_mimetype != old_mimetype or util.wrong_extension(ext):
logger.info('Updating mime type: %s => %s', old_mimetype,
new_mimetype)
old_extension = self.__episode.extension()
self.__episode.mime_type = new_mimetype
new_extension = self.__episode.extension()
# If the desired filename extension changed due to the new
# mimetype, we force an update of the local filename to fix the
# extension.
if old_extension != new_extension or util.wrong_extension(ext):
self.filename = self.__episode.local_filename(
create=True, force_update=True)
# In some cases, the redirect of a URL causes the real filename to
# be revealed in the final URL (e.g. http://gpodder.org/bug/1423)
if real_url != url and not util.is_known_redirecter(real_url):
realname, realext = util.filename_from_url(real_url)
# Only update from redirect if the redirected-to filename has
# a proper extension (this is needed for e.g. YouTube)
if not util.wrong_extension(realext):
real_filename = ''.join((realname, realext))
self.filename = self.__episode.local_filename(
create=True, force_update=True, template=real_filename)
logger.info(
'Download was redirected (%s). New filename: %s',
real_url, os.path.basename(self.filename))
# Look at the Content-disposition header; use if if available
disposition_filename = get_header_param(headers, 'filename',
'content-disposition')
# Some servers do send the content-disposition header, but provide
# an empty filename, resulting in an empty string here (bug 1440)
if disposition_filename is not None and disposition_filename != '':
# The server specifies a download filename - try to use it
# filename_from_url to remove query string; see #591
fn, ext = util.filename_from_url(disposition_filename)
logger.debug(
"converting disposition filename '%s' to local filename '%s%s'",
disposition_filename, fn, ext)
disposition_filename = fn + ext
self.filename = self.__episode.local_filename(
create=True,
force_update=True,
template=disposition_filename)
new_mimetype, encoding = mimetypes.guess_type(self.filename)
if new_mimetype is not None:
logger.info('Using content-disposition mimetype: %s',
new_mimetype)
self.__episode.mime_type = new_mimetype
# Re-evaluate filename and tempname to take care of podcast renames
# while downloads are running (which will change both file names)
self.filename = self.__episode.local_filename(create=False)
self.tempname = os.path.join(os.path.dirname(self.filename),
os.path.basename(self.tempname))
shutil.move(self.tempname, self.filename)
# Model- and database-related updates after a download has finished
self.__episode.on_downloaded(self.filename)
except DownloadCancelledException:
logger.info('Download has been cancelled/paused: %s', self)
if self.status == DownloadTask.CANCELLED:
util.delete_file(self.tempname)
self.progress = 0.0
self.speed = 0.0
except urllib.error.ContentTooShortError as ctse:
self.status = DownloadTask.FAILED
self.error_message = _('Missing content from server')
except IOError as ioe:
logger.error('%s while downloading "%s": %s',
ioe.strerror,
self.__episode.title,
ioe.filename,
exc_info=True)
self.status = DownloadTask.FAILED
d = {'error': ioe.strerror, 'filename': ioe.filename}
self.error_message = _('I/O Error: %(error)s: %(filename)s') % d
except gPodderDownloadHTTPError as gdhe:
logger.error('HTTP %s while downloading "%s": %s',
gdhe.error_code,
self.__episode.title,
gdhe.error_message,
exc_info=True)
self.status = DownloadTask.FAILED
d = {'code': gdhe.error_code, 'message': gdhe.error_message}
self.error_message = _('HTTP Error %(code)s: %(message)s') % d
except Exception as e:
self.status = DownloadTask.FAILED
logger.error('Download failed: %s', str(e), exc_info=True)
self.error_message = _('Error: %s') % (str(e), )
if self.status == DownloadTask.DOWNLOADING:
# Everything went well - we're done
self.status = DownloadTask.DONE
if self.total_size <= 0:
self.total_size = util.calculate_size(self.filename)
logger.info('Total size updated to %d', self.total_size)
self.progress = 1.0
gpodder.user_extensions.on_episode_downloaded(self.__episode)
return True
self.speed = 0.0
# We finished, but not successfully (at least not really)
return False
def run(self):
# Speed calculation (re-)starts here
self.__start_time = 0
self.__start_blocks = 0
# If the download has already been cancelled, skip it
if self.status == DownloadTask.CANCELLED:
util.delete_file(self.tempname)
self.progress = 0.0
self.speed = 0.0
return False
# We only start this download if its status is "queued"
if self.status != DownloadTask.QUEUED:
return False
# We are downloading this file right now
self.status = DownloadTask.DOWNLOADING
self._notification_shown = False
try:
# Resolve URL and start downloading the episode
url = registry.download_url.resolve(self.__episode, self.url, self._config)
downloader = DownloadURLOpener(self.__episode.podcast)
# HTTP Status codes for which we retry the download
retry_codes = (408, 418, 504, 598, 599)
max_retries = max(0, self._config.auto.retries)
# Retry the download on timeout (bug 1013)
for retry in range(max_retries + 1):
if retry > 0:
logger.info('Retrying download of %s (%d)', url, retry)
time.sleep(1)
try:
headers, real_url = downloader.retrieve_resume(url, self.tempname,
reporthook=self.status_updated)
# If we arrive here, the download was successful
break
except urllib.error.ContentTooShortError as ctse:
if retry < max_retries:
logger.info('Content too short: %s - will retry.', url)
continue
raise
except socket.timeout as tmout:
if retry < max_retries:
logger.info('Socket timeout: %s - will retry.', url)
continue
raise
except gPodderDownloadHTTPError as http:
if retry < max_retries and http.error_code in retry_codes:
logger.info('HTTP error %d: %s - will retry.', http.error_code, url)
continue
raise
new_mimetype = headers.get('content-type', self.__episode.mime_type)
old_mimetype = self.__episode.mime_type
_basename, ext = os.path.splitext(self.filename)
if new_mimetype != old_mimetype or util.wrong_extension(ext):
logger.info('Updating mime type: %s => %s', old_mimetype, new_mimetype)
old_extension = self.__episode.extension()
self.__episode.mime_type = new_mimetype
new_extension = self.__episode.extension()
# If the desired filename extension changed due to the new
# mimetype, we force an update of the local filename to fix the
# extension.
if old_extension != new_extension or util.wrong_extension(ext):
self.filename = self.__episode.local_filename(create=True, force_update=True)
# In some cases, the redirect of a URL causes the real filename to
# be revealed in the final URL (e.g. http://gpodder.org/bug/1423)
if real_url != url:
realname, realext = util.filename_from_url(real_url)
# Only update from redirect if the redirected-to filename has
# a proper extension (this is needed for e.g. YouTube)
if not util.wrong_extension(realext):
real_filename = ''.join((realname, realext))
self.filename = self.__episode.local_filename(create=True, force_update=True,
template=real_filename)
logger.info('Download was redirected (%s). New filename: %s', real_url,
os.path.basename(self.filename))
# Look at the Content-disposition header; use if if available
disposition_filename = get_header_param(headers, 'filename', 'content-disposition')
# Some servers do send the content-disposition header, but provide
# an empty filename, resulting in an empty string here (bug 1440)
if disposition_filename is not None and disposition_filename != '':
# The server specifies a download filename - try to use it
disposition_filename = os.path.basename(disposition_filename)
self.filename = self.__episode.local_filename(create=True, force_update=True,
template=disposition_filename)
new_mimetype, encoding = mimetypes.guess_type(self.filename)
if new_mimetype is not None:
logger.info('Using content-disposition mimetype: %s', new_mimetype)
self.__episode.mime_type = new_mimetype
# Re-evaluate filename and tempname to take care of podcast renames
# while downloads are running (which will change both file names)
self.filename = self.__episode.local_filename(create=False)
self.tempname = os.path.join(os.path.dirname(self.filename),
os.path.basename(self.tempname))
shutil.move(self.tempname, self.filename)
# Model- and database-related updates after a download has finished
self.__episode.on_downloaded(self.filename)
except DownloadCancelledException:
logger.info('Download has been cancelled/paused: %s', self)
if self.status == DownloadTask.CANCELLED:
util.delete_file(self.tempname)
self.progress = 0.0
self.speed = 0.0
except urllib.error.ContentTooShortError as ctse:
self.status = DownloadTask.FAILED
self.error_message = 'Missing content from server'
except IOError as ioe:
logger.error('%s while downloading "%s": %s', ioe.strerror,
self.__episode.title, ioe.filename, exc_info=True)
self.status = DownloadTask.FAILED
d = {'error': ioe.strerror, 'filename': ioe.filename}
self.error_message = 'I/O Error: %(error)s: %(filename)s' % d
except gPodderDownloadHTTPError as gdhe:
logger.error('HTTP %s while downloading "%s": %s',
gdhe.error_code, self.__episode.title, gdhe.error_message, exc_info=True)
self.status = DownloadTask.FAILED
d = {'code': gdhe.error_code, 'message': gdhe.error_message}
self.error_message = 'HTTP Error %(code)s: %(message)s' % d
except Exception as e:
self.status = DownloadTask.FAILED
logger.error('Download failed: %s', str(e), exc_info=True)
self.error_message = 'Error: %s' % (str(e),)
if self.status == DownloadTask.DOWNLOADING:
# Everything went well - we're done
self.status = DownloadTask.DONE
if self.total_size <= 0:
self.total_size = util.calculate_size(self.filename)
logger.info('Total size updated to %d', self.total_size)
self.progress = 1.0
registry.after_download.call_each(self.__episode)
return True
self.speed = 0.0
# We finished, but not successfully (at least not really)
return False
class SyncTask(download.DownloadTask):
# An object representing the synchronization task of an episode
# Possible states this sync task can be in
STATUS_MESSAGE = (_('Added'), _('Queued'), _('Synchronizing'),
_('Finished'), _('Failed'), _('Cancelled'), _('Paused'))
(INIT, QUEUED, DOWNLOADING, DONE, FAILED, CANCELLED, PAUSED) = range(7)
def __str__(self):
return self.__episode.title
def __get_status(self):
return self.__status
def __set_status(self, status):
if status != self.__status:
self.__status_changed = True
self.__status = status
status = property(fget=__get_status, fset=__set_status)
def __get_device(self):
return self.__device
def __set_device(self, device):
self.__device = device
device = property(fget=__get_device, fset=__set_device)
def __get_status_changed(self):
if self.__status_changed:
self.__status_changed = False
return True
else:
return False
status_changed = property(fget=__get_status_changed)
def __get_activity(self):
return self.__activity
def __set_activity(self, activity):
self.__activity = activity
activity = property(fget=__get_activity, fset=__set_activity)
def __get_empty_string(self):
return ''
url = property(fget=__get_empty_string)
podcast_url = property(fget=__get_empty_string)
def __get_episode(self):
return self.__episode
episode = property(fget=__get_episode)
def cancel(self):
if self.status in (self.DOWNLOADING, self.QUEUED):
self.status = self.CANCELLED
def removed_from_list(self):
# XXX: Should we delete temporary/incomplete files here?
pass
def __init__(self, episode):
self.__status = SyncTask.INIT
self.__activity = SyncTask.ACTIVITY_SYNCHRONIZE
self.__status_changed = True
self.__episode = episode
# Create the target filename and save it in the database
self.filename = self.__episode.local_filename(create=False)
self.tempname = self.filename + '.partial'
self.total_size = self.__episode.file_size
self.speed = 0.0
self.progress = 0.0
self.error_message = None
# Have we already shown this task in a notification?
self._notification_shown = False
# Variables for speed limit and speed calculation
self.__start_time = 0
self.__start_blocks = 0
self.__limit_rate_value = 999
self.__limit_rate = 999
# Callbacks
self._progress_updated = lambda x: None
def notify_as_finished(self):
if self.status == SyncTask.DONE:
if self._notification_shown:
return False
else:
self._notification_shown = True
return True
return False
def notify_as_failed(self):
if self.status == SyncTask.FAILED:
if self._notification_shown:
return False
else:
self._notification_shown = True
return True
return False
def add_progress_callback(self, callback):
self._progress_updated = callback
def status_updated(self, count, blockSize, totalSize):
# We see a different "total size" while downloading,
# so correct the total size variable in the thread
if totalSize != self.total_size and totalSize > 0:
self.total_size = float(totalSize)
if self.total_size > 0:
self.progress = max(
0.0, min(1.0,
float(count * blockSize) / self.total_size))
self._progress_updated(self.progress)
if self.status == SyncTask.CANCELLED:
raise SyncCancelledException()
if self.status == SyncTask.PAUSED:
raise SyncCancelledException()
def recycle(self):
self.episode.download_task = None
def run(self):
# Speed calculation (re-)starts here
self.__start_time = 0
self.__start_blocks = 0
# If the download has already been cancelled, skip it
if self.status == SyncTask.CANCELLED:
util.delete_file(self.tempname)
self.progress = 0.0
self.speed = 0.0
return False
# We only start this download if its status is "queued"
if self.status != SyncTask.QUEUED:
return False
# We are synching this file right now
self.status = SyncTask.DOWNLOADING
self._notification_shown = False
try:
logger.info('Starting SyncTask')
self.device.add_track(self.episode, reporthook=self.status_updated)
except Exception, e:
self.status = SyncTask.FAILED
logger.error('Download failed: %s', str(e), exc_info=True)
self.error_message = _('Error: %s') % (str(e), )
if self.status == SyncTask.DOWNLOADING:
# Everything went well - we're done
self.status = SyncTask.DONE
if self.total_size <= 0:
self.total_size = util.calculate_size(self.filename)
logger.info('Total size updated to %d', self.total_size)
self.progress = 1.0
gpodder.user_extensions.on_episode_downloaded(self.__episode)
return True
self.speed = 0.0
# We finished, but not successfully (at least not really)
return False
self.status = DownloadTask.FAILED
d = {'error': ioe.strerror, 'filename': ioe.filename}
self.error_message = _('I/O Error: %(error)s: %(filename)s') % d
except gPodderDownloadHTTPError, gdhe:
logger.error('HTTP %s while downloading "%s": %s',
gdhe.error_code, self.__episode.title, gdhe.error_message,
exc_info=True)
self.status = DownloadTask.FAILED
d = {'code': gdhe.error_code, 'message': gdhe.error_message}
self.error_message = _('HTTP Error %(code)s: %(message)s') % d
except Exception, e:
self.status = DownloadTask.FAILED
logger.error('Download failed: %s', str(e), exc_info=True)
self.error_message = _('Error: %s') % (str(e),)
if self.status == DownloadTask.DOWNLOADING:
# Everything went well - we're done
self.status = DownloadTask.DONE
if self.total_size <= 0:
self.total_size = util.calculate_size(self.filename)
logger.info('Total size updated to %d', self.total_size)
self.progress = 1.0
gpodder.user_extensions.on_episode_downloaded(self.__episode)
return True
self.speed = 0.0
# We finished, but not successfully (at least not really)
return False
def update_save_dir_size(self):
self.save_dir_size = util.calculate_size(self.save_dir)