def move_to_directory(self, directory):
check_f(directory)
if self.channelName:
channel_name = filter_directory_name(self.channelName)
# bug 10769: shutil and windows has problems with long
# filenames, so we clip the directory name.
if len(channel_name) > 80:
channel_name = channel_name[:80]
directory = os.path.join(directory, channel_name)
if not os.path.exists(directory):
try:
fileutil.makedirs(directory)
except (SystemExit, KeyboardInterrupt):
raise
except:
pass
newfilename = os.path.join(directory, self.shortFilename)
if newfilename == self.filename:
return
newfilename, fp = next_free_filename(newfilename)
def callback():
self.filename = newfilename
self.update_client()
fileutil.migrate_file(self.filename, newfilename, callback)
fp.close()
def add_items(self, item_infos):
for info in item_infos:
if self.stopping:
self._check_finished()
return
if self.device.database.item_exists(info):
continue # don't recopy stuff
conversion = self.conversion_for_info(info)
if not conversion:
continue
if info.file_type == 'audio':
target_folder = self.audio_target_folder
elif info.file_type == 'video':
target_folder = self.video_target_folder
else:
continue
self.total += 1
if conversion == 'copy':
final_path = os.path.join(target_folder,
os.path.basename(
info.video_path))
if os.path.exists(final_path):
logging.debug('%r exists, getting a new one precopy',
final_path)
final_path, fp = next_free_filename(final_path)
self.copy_file(info, final_path)
else:
self.start_conversion(conversion,
info,
target_folder)
self._check_finished()
def pick_initial_filename(self, suffix=".part", torrent=False):
"""Pick a path to download to based on self.shortFilename.
This method sets self.filename, as well as creates any leading
paths needed to start downloading there.
If the torrent flag is true, then the filename we're working
with is utf-8 and shouldn't be transformed in any way.
If the torrent flag is false, then the filename we're working
with is ascii and needs to be transformed into something sane.
(default)
"""
download_dir = os.path.join(app.config.get(prefs.MOVIES_DIRECTORY),
'Incomplete Downloads')
# Create the download directory if it doesn't already exist.
if not os.path.exists(download_dir):
fileutil.makedirs(download_dir)
filename = self.shortFilename + suffix
if not torrent:
# this is an ascii filename and needs to be fixed
filename = clean_filename(filename)
self.filename, fp = next_free_filename(
os.path.join(download_dir, filename))
# We can close this object now the file's been created, I guess.
# This will linger in the filesystem namespace, caller is responsible
# for cleaning this up (which I think it does).
fp.close()
def dumpDatabase(self):
output = next_free_filename(os.path.join(app.config.get(prefs.SUPPORT_DIRECTORY), "database-dump.xml"))
def indent(level):
output.write(' ' * level)
def output_object(table_name, values):
indent(1)
if 'id' in values:
output.write('<%s id="%s">\n' % (table_name, values['id']))
else:
output.write('<%s>\n' % (table_name,))
for key, value in values.items():
if key == 'id':
continue
indent(2)
output.write('<%s>' % (key,))
if isinstance (value, unicode):
output.write (value.encode('ascii', 'xmlcharrefreplace'))
else:
output.write (str(value))
output.write ('</%s>\n' % (key,))
indent(1)
output.write ('</%s>\n' % (table_name))
output.write ('<?xml version="1.0"?>\n')
output.write ('<database schema="%d">\n' % (self._schema_version,))
for schema in self._object_schemas:
self.cursor.execute("SELECT * FROM %s" % schema.table_name)
column_names = [d[0] for d in self.cursor.description]
for row in self.cursor:
output_object(schema.table_name.replace('_', '-'),
dict(zip(column_names, row)))
output.write ('</database>\n')
output.close()
def migrate(self, directory):
if app.download_state_manager.get_download(self.dlid):
c = command.MigrateDownloadCommand(RemoteDownloader.dldaemon,
self.dlid, directory)
c.send()
else:
# downloader doesn't have our dlid. Move the file ourself.
short_filename = self.status.get("shortFilename")
if not short_filename:
logging.warning(
"can't migrate download; no shortfilename! URL was %s",
self.url)
return
filename = self.status.get("filename")
if not filename:
logging.warning(
"can't migrate download; no filename! URL was %s",
self.url)
return
if fileutil.exists(filename):
if self.status.get('channelName', None) is not None:
channelName = filter_directory_name(
self.status['channelName'])
directory = os.path.join(directory, channelName)
if not os.path.exists(directory):
try:
fileutil.makedirs(directory)
except OSError:
# FIXME - what about permission issues?
pass
newfilename = os.path.join(directory, short_filename)
if newfilename == filename:
return
# create a file or directory to serve as a placeholder before
# we start to migrate. This helps ensure that the destination
# we're migrating too is not already taken.
try:
is_dir = fileutil.isdir(filename)
if is_dir:
newfilename = next_free_directory(newfilename)
fp = None
else:
newfilename, fp = next_free_filename(newfilename)
fp.close()
except ValueError:
func = ('next_free_directory'
if is_dir else 'next_free_filename')
logging.warn('migrate: %s failed. candidate = %r', func,
newfilename)
else:
def callback():
self.status['filename'] = newfilename
self.signal_change(needs_signal_item=False)
self._file_migrated(filename)
fileutil.migrate_file(filename, newfilename, callback)
for i in self.item_list:
i.migrate_children(directory)
def _add_item(self, final_path, item_info):
dirname, basename = os.path.split(final_path)
_, extension = os.path.splitext(basename)
new_basename = "%s%s" % (unicode_to_filename(item_info.name,
self.device.mount),
extension)
new_path = os.path.join(dirname, new_basename)
if os.path.exists(new_path):
logging.debug('final destination %r exists, making a new one',
new_path)
new_path, fp = next_free_filename(new_path)
def callback():
if not os.path.exists(new_path):
return # copy failed, just give up
device_item = item.DeviceItem(
device=self.device,
file_type=item_info.file_type,
video_path=new_path[len(self.device.mount):],
title=item_info.name,
feed_name=item_info.feed_name,
feed_url=item_info.feed_url,
description=item_info.description,
release_date=time.mktime(item_info.release_date.timetuple()),
duration=(item_info.duration and item_info.duration * 1000 or
None),
permalink=item_info.permalink,
commentslink=item_info.commentslink,
payment_link=item_info.payment_link,
screenshot=item_info.thumbnail,
thumbnail_url=item_info.thumbnail_url,
file_format=item_info.file_format,
license=item_info.license,
url=item_info.file_url,
media_type_checked=item_info.media_type_checked,
mime_type=item_info.mime_type,
creation_time=time.mktime(item_info.date_added.timetuple()),
title_tag=item_info.title_tag,
artist=item_info.artist,
album=item_info.album,
track=item_info.track,
year=item_info.year,
genre=item_info.genre,
metadata_version=item_info.metadata_version,
mdp_state=item_info.mdp_state,
auto_sync=getattr(item_info, 'auto_sync', False)
)
device_item._migrate_thumbnail()
database = self.device.database
database.setdefault(device_item.file_type, {})
database[device_item.file_type][device_item.id] = \
device_item.to_dict()
database.emit('item-added', device_item)
self.device.remaining -= device_item.size
fileutil.migrate_file(final_path, new_path, callback)
def migrate(self, directory):
if app.download_state_manager.get_download(self.dlid):
c = command.MigrateDownloadCommand(RemoteDownloader.dldaemon,
self.dlid, directory)
c.send()
else:
# downloader doesn't have our dlid. Move the file ourself.
short_filename = self.short_filename
if not short_filename:
logging.warning(
"can't migrate download; no shortfilename! URL was %s",
self.url)
return
filename = self.filename
if not filename:
logging.warning(
"can't migrate download; no filename! URL was %s",
self.url)
return
if fileutil.exists(filename):
if self.channel_name is not None:
channel_name = filter_directory_name(self.channel_name)
directory = os.path.join(directory, channel_name)
if not os.path.exists(directory):
try:
fileutil.makedirs(directory)
except OSError:
# FIXME - what about permission issues?
pass
newfilename = os.path.join(directory, short_filename)
if newfilename == filename:
return
# create a file or directory to serve as a placeholder before
# we start to migrate. This helps ensure that the destination
# we're migrating too is not already taken.
try:
is_dir = fileutil.isdir(filename)
if is_dir:
newfilename = next_free_directory(newfilename)
fp = None
else:
newfilename, fp = next_free_filename(newfilename)
fp.close()
except ValueError:
func = ('next_free_directory' if is_dir
else 'next_free_filename')
logging.warn('migrate: %s failed. candidate = %r',
func, newfilename)
else:
def callback():
self.filename = newfilename
self.signal_change(needs_signal_item=False)
self._file_migrated(filename)
fileutil.migrate_file(filename, newfilename, callback)
for i in self.item_list:
i.migrate_children(directory)
def test_next_free_filename(self):
# make a bunch of files that we should skip over
for name in ('foo', 'foo.1', 'foo.2', 'bar.jpg', 'bar.1.jpg'):
path = os.path.join(self.tempdir, name)
open(path, 'wt').write("FAKE FILE")
path1 = os.path.join(self.tempdir, 'foo')
# test we find the a nonexistent file
returned_path, fp = download_utils.next_free_filename(path1)
self.assertEquals(returned_path, os.path.join(self.tempdir, 'foo.3'))
# test that we create the file
self.assert_(os.path.exists(returned_path))
# try with an extension
path2 = os.path.join(self.tempdir, 'bar.jpg')
returned_path, fp = download_utils.next_free_filename(path2)
self.assertEquals(returned_path, os.path.join(self.tempdir,
'bar.2.jpg'))
self.assert_(os.path.exists(returned_path))
def _make_screenshot_path(source_path, image_directory):
"""Get a unique path to put a screenshot at
This function creates a unique path to put a screenshot file at.
:param source_path: path to the input video file
:param image_directory: directory to put screenshots in
"""
filename = os.path.basename(source_path) + ".png"
path = os.path.join(image_directory, filename)
# we have to use next_free_filename_no_create() here because we are
# passing the file path to the movie data process, so keeping the file
# open is not an option. We'll just have to live with the race condition
return download_utils.next_free_filename(path)
def migrate(self, directory):
if _downloads.has_key(self.dlid):
c = command.MigrateDownloadCommand(RemoteDownloader.dldaemon,
self.dlid, directory)
c.send()
else:
# downloader doesn't have our dlid. Move the file ourself.
short_filename = self.status.get("shortFilename")
if not short_filename:
logging.warning(
"can't migrate download; no shortfilename! URL was %s",
self.url)
return
filename = self.status.get("filename")
if not filename:
logging.warning(
"can't migrate download; no filename! URL was %s",
self.url)
return
if fileutil.exists(filename):
if self.status.get('channelName', None) is not None:
channelName = filter_directory_name(self.status['channelName'])
directory = os.path.join(directory, channelName)
if not os.path.exists(directory):
try:
fileutil.makedirs(directory)
except OSError:
# FIXME - what about permission issues?
pass
newfilename = os.path.join(directory, short_filename)
if newfilename == filename:
return
# create a file or directory to serve as a placeholder before
# we start to migrate. This helps ensure that the destination
# we're migrating too is not already taken.
if fileutil.isdir(filename):
newfilename = next_free_directory(newfilename)
fp = None
else:
newfilename, fp = next_free_filename(newfilename)
def callback():
self.status['filename'] = newfilename
self.signal_change(needs_signal_item=False)
self._file_migrated(filename)
fileutil.migrate_file(filename, newfilename, callback)
if fp is not None:
fp.close() # clean up if we called next_free_filename()
for i in self.item_list:
i.migrate_children(directory)
def migrate(self, directory):
if _downloads.has_key(self.dlid):
c = command.MigrateDownloadCommand(RemoteDownloader.dldaemon,
self.dlid, directory)
c.send()
else:
# downloader doesn't have our dlid. Move the file ourself.
short_filename = self.status.get("shortFilename")
if not short_filename:
logging.warning(
"can't migrate download; no shortfilename! URL was %s",
self.url)
return
filename = self.status.get("filename")
if not filename:
logging.warning(
"can't migrate download; no filename! URL was %s",
self.url)
return
if fileutil.exists(filename):
if self.status.get('channelName', None) is not None:
channelName = filter_directory_name(self.status['channelName'])
directory = os.path.join(directory, channelName)
if not os.path.exists(directory):
try:
fileutil.makedirs(directory)
except OSError:
# FIXME - what about permission issues?
pass
newfilename = os.path.join(directory, short_filename)
if newfilename == filename:
return
newfilename, fp = next_free_filename(newfilename)
def callback():
self.status['filename'] = newfilename
self.signal_change(needs_signal_item=False)
self._file_migrated(filename)
fileutil.migrate_file(filename, newfilename, callback)
fp.close()
for i in self.item_list:
i.migrate_children(directory)
def _process_message_queue(self):
try:
msg = self.message_queue.get_nowait()
except Queue.Empty:
return
if msg['message'] == 'get_tasks_list':
self._notify_tasks_list()
elif msg['message'] == 'cancel':
try:
task = self._lookup_task(msg['key'])
except KeyError:
logging.warn("Couldn't find task for key %s", msg['key'])
return
if task.is_pending():
task_list = self.pending_tasks
elif task.is_running():
task_list = self.running_tasks
else:
task_list = self.finished_tasks
try:
task_list.remove(task)
except ValueError:
logging.warn("Task not in list: %s", msg['key'])
else:
self._notify_task_removed(task)
self._notify_tasks_count()
task.interrupt()
elif msg['message'] == 'clear_all_finished':
for task in self.finished_tasks:
self._notify_task_removed(task)
self.finished_tasks = []
self._notify_tasks_count()
elif msg['message'] == 'clear_finished':
try:
task = self._lookup_task(msg['key'])
except KeyError:
logging.warn("Couldn't find task for key %s", msg['key'])
return
self.finished_tasks.remove(task)
self._notify_task_removed(task)
self._notify_tasks_count()
elif msg['message'] == 'cancel_all':
self._terminate()
elif msg['message'] == 'stage_conversion':
try:
task = self._lookup_task(msg['key'])
except KeyError:
logging.warn("Couldn't find task for key %s", msg['key'])
return
source = task.temp_output_path
destination, fp = next_free_filename(task.final_output_path)
source_info = task.item_info
conversion_name = task.get_display_name()
if os.path.exists(source):
self._move_finished_file(source, destination)
if task.create_item:
_create_item_for_conversion(destination,
source_info,
conversion_name)
if not task.temp_output_path.endswith('.tmp'): # temp dir
clean_up(task.temp_output_path,
file_and_directory=True)
else:
task.error = _("Reason unknown--check log")
self._notify_tasks_count()
fp.close()
self.emit('task-staged', task)
def update_icon_cache(self, url, info):
self.dbItem.confirm_db_thread()
if self.removed:
app.icon_cache_updater.update_finished()
return
needs_save = False
needsChange = False
if info == None or (info['status'] != 304 and info['status'] != 200):
self.error_callback(url, "bad response")
return
try:
# Our cache is good. Hooray!
if info['status'] == 304:
return
needsChange = True
# We have to update it, and if we can't write to the file, we
# should pick a new filename.
if ((self.filename and
not fileutil.access(self.filename, os.R_OK | os.W_OK))):
self.filename = None
cachedir = app.config.get(prefs.ICON_CACHE_DIRECTORY)
try:
fileutil.makedirs(cachedir)
except OSError:
pass
try:
# Write to a temp file.
if self.filename:
tmp_filename = self.filename + ".part"
else:
tmp_filename = os.path.join(cachedir, info["filename"]) + ".part"
tmp_filename, output = next_free_filename(tmp_filename)
output.write(info["body"])
output.close()
except IOError:
self.remove_file(tmp_filename)
return
except ValueError:
logging.warn('update_icon_cache: next_free_filename failed '
'#1, candidate = %r', tmp_filename)
return
filename = unicode(info["filename"])
filename = unicode_to_filename(filename, cachedir)
filename = os.path.join(cachedir, filename)
needs_save = True
try:
filename, fp = next_free_filename(filename)
except ValueError:
logging.warn('update_icon_cache: next_free_filename failed '
'#2, candidate = %r', filename)
return
if self.filename:
filename = self.filename
self.filename = None
self.remove_file(filename)
# we need to move the file here--so we close the file
# pointer and then move the file.
fp.close()
try:
self.remove_file(filename)
fileutil.rename(tmp_filename, filename)
except (IOError, OSError):
logging.exception("iconcache: fileutil.move failed")
filename = None
self.filename = filename
etag = unicodify(info.get("etag"))
modified = unicodify(info.get("modified"))
if self.etag != etag:
needs_save = True
self.etag = etag
if self.modified != modified:
needs_save = True
self.modified = modified
if self.url != url:
needs_save = True
self.url = url
finally:
if needsChange:
self.icon_changed(needs_save=needs_save)
self.updating = False
if self.needsUpdate:
self.needsUpdate = False
self.request_update(True)
app.icon_cache_updater.update_finished()
def _process_message_queue(self):
try:
msg = self.message_queue.get_nowait()
except Queue.Empty:
return
if msg['message'] == 'get_tasks_list':
self._notify_tasks_list()
elif msg['message'] == 'cancel':
try:
task = self._lookup_task(msg['key'])
except KeyError:
logging.warn("Couldn't find task for key %s", msg['key'])
return
if task.is_pending():
task_list = self.pending_tasks
elif task.is_running():
task_list = self.running_tasks
else:
task_list = self.finished_tasks
try:
task_list.remove(task)
except ValueError:
logging.warn("Task not in list: %s", msg['key'])
else:
self._notify_task_removed(task)
self._notify_tasks_count()
task.interrupt()
elif msg['message'] == 'clear_all_finished':
for task in self.finished_tasks:
self._notify_task_removed(task)
self.finished_tasks = []
self._notify_tasks_count()
elif msg['message'] == 'clear_finished':
try:
task = self._lookup_task(msg['key'])
except KeyError:
logging.warn("Couldn't find task for key %s", msg['key'])
return
self.finished_tasks.remove(task)
self._notify_task_removed(task)
self._notify_tasks_count()
elif msg['message'] == 'cancel_all':
self._terminate()
elif msg['message'] == 'stage_conversion':
try:
task = self._lookup_task(msg['key'])
except KeyError:
logging.warn("Couldn't find task for key %s", msg['key'])
return
source = task.temp_output_path
try:
destination, fp = next_free_filename(task.final_output_path)
fp.close()
except ValueError:
logging.warn(
'_process_message_queue: '
'next_free_filename failed. Candidate = %r',
task.final_output_path)
return
source_info = task.item_info
conversion_name = task.get_display_name()
if os.path.exists(source):
self._move_finished_file(source, destination)
if task.create_item:
_create_item_for_conversion(destination, source_info,
conversion_name)
if not task.temp_output_path.endswith('.tmp'): # temp dir
clean_up(task.temp_output_path, file_and_directory=True)
else:
task.error = _("Reason unknown--check log")
self._notify_tasks_count()
self.emit('task-staged', task)
def update_icon_cache(self, url, info):
self.dbItem.confirm_db_thread()
if self.removed:
icon_cache_updater.update_finished()
return
needs_save = False
needsChange = False
if info == None or (info['status'] != 304 and info['status'] != 200):
self.error_callback(url, "bad response")
return
try:
# Our cache is good. Hooray!
if info['status'] == 304:
return
needsChange = True
# We have to update it, and if we can't write to the file, we
# should pick a new filename.
if ((self.filename and
not fileutil.access(self.filename, os.R_OK | os.W_OK))):
self.filename = None
cachedir = app.config.get(prefs.ICON_CACHE_DIRECTORY)
try:
fileutil.makedirs(cachedir)
except OSError:
pass
try:
# Write to a temp file.
if self.filename:
tmp_filename = self.filename + ".part"
else:
tmp_filename = os.path.join(cachedir, info["filename"]) + ".part"
tmp_filename, output = next_free_filename(tmp_filename)
output.write(info["body"])
output.close()
except IOError:
self.remove_file(tmp_filename)
return
except ValueError:
logging.warn('update_icon_cache: next_free_filename failed '
'#1, candidate = %r', tmp_filename)
return
filename = unicode(info["filename"])
filename = unicode_to_filename(filename, cachedir)
filename = os.path.join(cachedir, filename)
needs_save = True
try:
filename, fp = next_free_filename(filename)
except ValueError:
logging.warn('update_icon_cache: next_free_filename failed '
'#2, candidate = %r', filename)
return
if self.filename:
filename = self.filename
self.filename = None
self.remove_file(filename)
# we need to move the file here--so we close the file
# pointer and then move the file.
fp.close()
try:
self.remove_file(filename)
fileutil.rename(tmp_filename, filename)
except (IOError, OSError):
logging.exception("iconcache: fileutil.move failed")
filename = None
self.filename = filename
etag = unicodify(info.get("etag"))
modified = unicodify(info.get("modified"))
if self.etag != etag:
needs_save = True
self.etag = etag
if self.modified != modified:
needs_save = True
self.modified = modified
if self.url != url:
needs_save = True
self.url = url
finally:
if needsChange:
self.icon_changed(needs_save=needs_save)
self.updating = False
if self.needsUpdate:
self.needsUpdate = False
self.request_update(True)
icon_cache_updater.update_finished()
def update_icon_cache(self, url, info):
self.dbItem.confirm_db_thread()
if self.removed:
iconCacheUpdater.update_finished()
return
needs_save = False
needsChange = False
if info == None or (info['status'] != 304 and info['status'] != 200):
self.error_callback(url, "bad response")
return
try:
# Our cache is good. Hooray!
if info['status'] == 304:
return
needsChange = True
# We have to update it, and if we can't write to the file, we
# should pick a new filename.
if self.filename and not fileutil.access(self.filename, os.R_OK | os.W_OK):
self.filename = None
cachedir = app.config.get(prefs.ICON_CACHE_DIRECTORY)
try:
fileutil.makedirs(cachedir)
except OSError:
pass
try:
# Write to a temp file.
if self.filename:
tmp_filename = self.filename + ".part"
else:
tmp_filename = os.path.join(cachedir, info["filename"]) + ".part"
tmp_filename, output = next_free_filename(tmp_filename)
output.write(info["body"])
output.close()
except IOError:
self.remove_file(tmp_filename)
return
if self.filename:
self.remove_file(self.filename)
# Create a new filename always to avoid browser caching in case a
# file changes.
# Add a random unique id
parts = unicodify(info["filename"]).split('.')
uid = u"%08d" % random.randint(0, 99999999)
if len(parts) == 1:
parts.append(uid)
else:
parts[-1:-1] = [uid]
self.filename = u'.'.join(parts)
self.filename = unicode_to_filename(self.filename, cachedir)
self.filename = os.path.join(cachedir, self.filename)
self.filename, fp = next_free_filename(self.filename)
needs_save = True
try:
fileutil.rename(tmp_filename, self.filename)
except OSError:
self.filename = None
needs_save = True
fp.close()
etag = unicodify(info.get("etag"))
modified = unicodify(info.get("modified"))
if self.etag != etag:
needs_save = True
self.etag = etag
if self.modified != modified:
needs_save = True
self.modified = modified
if self.url != url:
needs_save = True
self.url = url
finally:
if needsChange:
self.icon_changed(needs_save=needs_save)
self.updating = False
if self.needsUpdate:
self.needsUpdate = False
self.request_update(True)
iconCacheUpdater.update_finished()