def callback(self, result, mdi):
mediatype, duration, got_screenshot = result
# Make sure this is unicode, or else database validation will
# fail on insert!
mediatype = unicode(mediatype)
if os.path.splitext(mdi.video_path)[1] == '.flv':
# bug #17266. if the extension is .flv, we ignore the mediatype
# we just got from the movie data program. this is
# specifically for .flv files which the movie data
# extractors have a hard time with.
mediatype = u'video'
if fileutil.exists(mdi.thumbnail_path) and got_screenshot:
screenshot = mdi.thumbnail_path
else:
screenshot = None
# bz:17364/bz:18072 HACK: need to avoid UnicodeDecodeError -
# until we do a proper pathname cleanup. Used to be a %s with a
# encode to utf-8 but then 18072 came up. It seems that this
# can either be a str OR a unicode. I don't really feel
# like dealing with this right now, so just use %r.
logging.debug("moviedata: mdp %s %s %s %r", duration, screenshot,
mediatype, mdi.video_path)
# Repack the thing, as we may have changed it
self.update_finished(mdi.item, duration, screenshot, mediatype)
self._path_processed(mdi)
def cleanup_incomplete_downloads():
download_dir = os.path.join(app.config.get(prefs.MOVIES_DIRECTORY),
'Incomplete Downloads')
if not fileutil.exists(download_dir):
return
files_in_use = set()
for downloader in RemoteDownloader.make_view():
if downloader.get_state() in ('downloading', 'paused',
'offline', 'uploading', 'finished',
'uploading-paused'):
filename = downloader.get_filename()
if filename:
if not fileutil.isabs(filename):
filename = os.path.join(download_dir, filename)
files_in_use.add(filename)
try:
entries = fileutil.listdir(download_dir)
except OSError:
entries = []
for f in entries:
f = os.path.join(download_dir, f)
if f not in files_in_use:
try:
if fileutil.isfile(f):
fileutil.remove(f)
elif fileutil.isdir(f):
fileutil.rmtree(f)
except OSError:
# FIXME - maybe a permissions error?
pass
def convert_mdp_result(source_path, screenshot, result):
"""Convert the movie data program result for the metadata manager
"""
converted_result = { 'source_path': source_path }
file_type, duration, success = result
if duration >= 0:
converted_result['duration'] = duration
# Return a file_type only if the duration is > 0. Otherwise it may be a
# false identification (#18840). Also, if moviedata reports other, that's
# a sign that it doesn't know what the file type is. Just leave out the
# file_type key so that we fallback to the mutagen guess.
if file_type != "other" and (duration not in (0, None)):
# Make file_type is unicode, or else database validation will fail on
# insert!
converted_result['file_type'] = unicode(file_type)
if os.path.splitext(source_path)[1] == '.flv':
# bug #17266. if the extension is .flv, we ignore the file type
# we just got from the movie data program. this is
# specifically for .flv files which the movie data
# extractors have a hard time with.
converted_result['file_type'] = u'video'
if (converted_result.get('file_type') == 'video' and success and
fileutil.exists(screenshot)):
converted_result['screenshot'] = screenshot
return converted_result
def cleanup_incomplete_downloads():
download_dir = os.path.join(app.config.get(prefs.MOVIES_DIRECTORY),
'Incomplete Downloads')
if not fileutil.exists(download_dir):
return
files_in_use = set()
for downloader in RemoteDownloader.make_view():
if downloader.get_state() in ('downloading', 'paused',
'offline', 'uploading', 'finished',
'uploading-paused'):
filename = downloader.get_filename()
if filename:
if not fileutil.isabs(filename):
filename = os.path.join(download_dir, filename)
files_in_use.add(filename)
try:
entries = fileutil.listdir(download_dir)
except OSError:
entries = []
for f in entries:
f = os.path.join(download_dir, f)
if f not in files_in_use:
try:
if fileutil.isfile(f):
fileutil.remove(f)
elif fileutil.isdir(f):
fileutil.rmtree(f)
except OSError:
# FIXME - maybe a permissions error?
pass
def callback(self, result, mdi):
mediatype, duration, got_screenshot = result
if os.path.splitext(mdi.video_path)[1] == '.flv':
# bug #17266. if the extension is .flv, we ignore the mediatype
# we just got from the movie data program. this is
# specifically for .flv files which the movie data
# extractors have a hard time with.
mediatype = u'video'
if fileutil.exists(mdi.thumbnail_path) and got_screenshot:
screenshot = mdi.thumbnail_path
else:
screenshot = None
# bz:17364/bz:18072 HACK: need to avoid UnicodeDecodeError -
# until we do a proper pathname cleanup. Used to be a %s with a
# encode to utf-8 but then 18072 came up. It seems that this
# can either be a str OR a unicode. I don't really feel
# like dealing with this right now, so just use %r.
logging.debug("moviedata: mdp %s %s %s %r", duration, screenshot,
mediatype, mdi.video_path)
# Repack the thing, as we may have changed it
self.update_finished(mdi.item, duration, screenshot, mediatype)
self._path_processed(mdi)
def thread_loop(self):
while not self.in_shutdown:
self.emit('begin-loop')
if self.queue.empty():
self.emit('queue-empty')
mdi = self.queue.get(block=True)
if mdi is None or mdi.program_info is None:
# shutdown() was called or there's no moviedata
# implemented.
self.emit('end-loop')
break
duration = -1
metadata = {}
(mime_mediatype, duration, metadata) = self.read_metadata(mdi.item)
if duration > -1 and mime_mediatype is not 'video':
mediatype = 'audio'
screenshot = mdi.item.screenshot or FilenameType("")
logging.debug("moviedata: mutagen %s %s", duration, mediatype)
self.update_finished(mdi.item, duration, screenshot, mediatype,
metadata)
else:
try:
screenshot_worked = False
screenshot = None
command_line, env = mdi.program_info
stdout = self.run_movie_data_program(command_line, env)
# if the moviedata program tells us to try again, we move
# along without updating the item at all
if TRY_AGAIN_RE.search(stdout):
continue
if duration == -1:
duration = self.parse_duration(stdout)
mediatype = self.parse_type(stdout)
if THUMBNAIL_SUCCESS_RE.search(stdout):
screenshot_worked = True
if ((screenshot_worked and
fileutil.exists(mdi.thumbnail_path))):
screenshot = mdi.thumbnail_path
else:
# All the programs failed, maybe it's an audio
# file? Setting it to "" instead of None, means
# that we won't try to take the screenshot again.
screenshot = FilenameType("")
logging.debug("moviedata: mdp %s %s %s", duration, screenshot,
mediatype)
self.update_finished(mdi.item, duration, screenshot,
mediatype, metadata)
except StandardError:
if self.in_shutdown:
break
signals.system.failed_exn(
"When running external movie data program")
self.update_finished(mdi.item, -1, None, None, metadata)
self.emit('end-loop')
def migrate(self, directory):
if app.download_state_manager.get_download(self.dlid):
c = command.MigrateDownloadCommand(RemoteDownloader.dldaemon,
self.dlid, directory)
c.send()
else:
# downloader doesn't have our dlid. Move the file ourself.
short_filename = self.status.get("shortFilename")
if not short_filename:
logging.warning(
"can't migrate download; no shortfilename! URL was %s",
self.url)
return
filename = self.status.get("filename")
if not filename:
logging.warning(
"can't migrate download; no filename! URL was %s",
self.url)
return
if fileutil.exists(filename):
if self.status.get('channelName', None) is not None:
channelName = filter_directory_name(
self.status['channelName'])
directory = os.path.join(directory, channelName)
if not os.path.exists(directory):
try:
fileutil.makedirs(directory)
except OSError:
# FIXME - what about permission issues?
pass
newfilename = os.path.join(directory, short_filename)
if newfilename == filename:
return
# create a file or directory to serve as a placeholder before
# we start to migrate. This helps ensure that the destination
# we're migrating too is not already taken.
try:
is_dir = fileutil.isdir(filename)
if is_dir:
newfilename = next_free_directory(newfilename)
fp = None
else:
newfilename, fp = next_free_filename(newfilename)
fp.close()
except ValueError:
func = ('next_free_directory'
if is_dir else 'next_free_filename')
logging.warn('migrate: %s failed. candidate = %r', func,
newfilename)
else:
def callback():
self.status['filename'] = newfilename
self.signal_change(needs_signal_item=False)
self._file_migrated(filename)
fileutil.migrate_file(filename, newfilename, callback)
for i in self.item_list:
i.migrate_children(directory)
def thumbnail(self):
if (self.cover_art_path_unicode is not None
and fileutil.exists(self.cover_art_path)):
return self.cover_art_path
if (self.icon_cache_path_unicode is not None and
fileutil.exists(self.icon_cache_path)):
return self.icon_cache_path
if (self.screenshot_path_unicode is not None
and fileutil.exists(self.screenshot_path)):
return self.screenshot_path
if self.is_container_item:
return resources.path("images/thumb-default-folder.png")
else:
# TODO: check for feed thumbnail here
if self.file_type == u'audio':
return resources.path("images/thumb-default-audio.png")
else:
return resources.path("images/thumb-default-video.png")
def migrate(self, directory):
if app.download_state_manager.get_download(self.dlid):
c = command.MigrateDownloadCommand(RemoteDownloader.dldaemon,
self.dlid, directory)
c.send()
else:
# downloader doesn't have our dlid. Move the file ourself.
short_filename = self.short_filename
if not short_filename:
logging.warning(
"can't migrate download; no shortfilename! URL was %s",
self.url)
return
filename = self.filename
if not filename:
logging.warning(
"can't migrate download; no filename! URL was %s",
self.url)
return
if fileutil.exists(filename):
if self.channel_name is not None:
channel_name = filter_directory_name(self.channel_name)
directory = os.path.join(directory, channel_name)
if not os.path.exists(directory):
try:
fileutil.makedirs(directory)
except OSError:
# FIXME - what about permission issues?
pass
newfilename = os.path.join(directory, short_filename)
if newfilename == filename:
return
# create a file or directory to serve as a placeholder before
# we start to migrate. This helps ensure that the destination
# we're migrating too is not already taken.
try:
is_dir = fileutil.isdir(filename)
if is_dir:
newfilename = next_free_directory(newfilename)
fp = None
else:
newfilename, fp = next_free_filename(newfilename)
fp.close()
except ValueError:
func = ('next_free_directory' if is_dir
else 'next_free_filename')
logging.warn('migrate: %s failed. candidate = %r',
func, newfilename)
else:
def callback():
self.filename = newfilename
self.signal_change(needs_signal_item=False)
self._file_migrated(filename)
fileutil.migrate_file(filename, newfilename, callback)
for i in self.item_list:
i.migrate_children(directory)
def add_videos(paths):
# filter out non-existent paths
paths = [p for p in paths if fileutil.exists(p)]
path_iter = iter(paths)
finished = False
yield # yield after doing prep work
with app.local_metadata_manager.bulk_add():
while not finished:
finished = _add_batch_of_videos(path_iter, 0.1)
yield # yield after each batch
def thumbnail(self):
if (self.cover_art_path_unicode is not None
and fileutil.exists(self.cover_art_path)):
return self.cover_art_path
if (self.icon_cache_path_unicode is not None and
fileutil.exists(self.icon_cache_path)):
return self.icon_cache_path
if (self.screenshot_path_unicode is not None
and fileutil.exists(self.screenshot_path)):
return self.screenshot_path
if self.is_container_item:
return resources.path("images/thumb-default-folder.png")
if self.feed_thumbnail_path is not None:
return self.feed_thumbnail_path
# default
if self.file_type == u'audio':
return resources.path("images/thumb-default-audio.png")
else:
return resources.path("images/thumb-default-video.png")
def add_videos(paths):
# filter out non-existent paths
paths = [p for p in paths if fileutil.exists(p)]
path_iter = iter(paths)
finished = False
yield # yield after doing prep work
with app.local_metadata_manager.bulk_add():
while not finished:
finished = _add_batch_of_videos(path_iter, 0.1)
yield # yield after each batch
def add_videos(paths):
# filter out non-existent paths
paths = [p for p in paths if fileutil.exists(p)]
for path in paths:
app.metadata_progress_updater.will_process_path(path)
path_iter = iter(paths)
finished = False
yield # yield after doing prep work
while not finished:
finished = _add_batch_of_videos(path_iter, 0.5)
yield # yield after each batch
def add_videos(paths):
# filter out non-existent paths
paths = [p for p in paths if fileutil.exists(p)]
for path in paths:
app.metadata_progress_updater.will_process_path(path)
path_iter = iter(paths)
finished = False
yield # yield after doing prep work
while not finished:
finished = _add_batch_of_videos(path_iter, 0.1)
yield # yield after each batch
def migrate(self, directory):
if _downloads.has_key(self.dlid):
c = command.MigrateDownloadCommand(RemoteDownloader.dldaemon,
self.dlid, directory)
c.send()
else:
# downloader doesn't have our dlid. Move the file ourself.
short_filename = self.status.get("shortFilename")
if not short_filename:
logging.warning(
"can't migrate download; no shortfilename! URL was %s",
self.url)
return
filename = self.status.get("filename")
if not filename:
logging.warning(
"can't migrate download; no filename! URL was %s",
self.url)
return
if fileutil.exists(filename):
if self.status.get('channelName', None) is not None:
channelName = filter_directory_name(self.status['channelName'])
directory = os.path.join(directory, channelName)
if not os.path.exists(directory):
try:
fileutil.makedirs(directory)
except OSError:
# FIXME - what about permission issues?
pass
newfilename = os.path.join(directory, short_filename)
if newfilename == filename:
return
# create a file or directory to serve as a placeholder before
# we start to migrate. This helps ensure that the destination
# we're migrating too is not already taken.
if fileutil.isdir(filename):
newfilename = next_free_directory(newfilename)
fp = None
else:
newfilename, fp = next_free_filename(newfilename)
def callback():
self.status['filename'] = newfilename
self.signal_change(needs_signal_item=False)
self._file_migrated(filename)
fileutil.migrate_file(filename, newfilename, callback)
if fp is not None:
fp.close() # clean up if we called next_free_filename()
for i in self.item_list:
i.migrate_children(directory)
def migrate(self, directory):
if _downloads.has_key(self.dlid):
c = command.MigrateDownloadCommand(RemoteDownloader.dldaemon,
self.dlid, directory)
c.send()
else:
# downloader doesn't have our dlid. Move the file ourself.
short_filename = self.status.get("shortFilename")
if not short_filename:
logging.warning(
"can't migrate download; no shortfilename! URL was %s",
self.url)
return
filename = self.status.get("filename")
if not filename:
logging.warning(
"can't migrate download; no filename! URL was %s",
self.url)
return
if fileutil.exists(filename):
if self.status.get('channelName', None) is not None:
channelName = filter_directory_name(self.status['channelName'])
directory = os.path.join(directory, channelName)
if not os.path.exists(directory):
try:
fileutil.makedirs(directory)
except OSError:
# FIXME - what about permission issues?
pass
newfilename = os.path.join(directory, short_filename)
if newfilename == filename:
return
newfilename, fp = next_free_filename(newfilename)
def callback():
self.status['filename'] = newfilename
self.signal_change(needs_signal_item=False)
self._file_migrated(filename)
fileutil.migrate_file(filename, newfilename, callback)
fp.close()
for i in self.item_list:
i.migrate_children(directory)
def is_valid(self):
self.dbItem.confirm_db_thread()
return self.filename and fileutil.exists(self.filename)
def is_valid(self):
self.dbItem.confirm_db_thread()
return self.filename and fileutil.exists(self.filename)
def parse_screenshot(self, stdout, mdi):
if (THUMBNAIL_SUCCESS_RE.search(stdout) and
fileutil.exists(mdi.thumbnail_path)):
return mdi.thumbnail_path
else:
return None
def parse_screenshot(self, stdout, mdi):
if (THUMBNAIL_SUCCESS_RE.search(stdout) and
fileutil.exists(mdi.thumbnail_path)):
return mdi.thumbnail_path
else:
return None
