def filename_from_url(url, clean=False):
"""Returns a reasonable filename for saving the given url.
"""
check_u(url)
try:
match = URI_PATTERN.match(url)
if match is None:
# This code path will never be executed.
return unicode_to_filename(url)
filename = match.group(2)
query = match.group(4)
if not filename:
ret = query
elif not query:
ret = filename
else:
root, ext = os.path.splitext(filename)
ret = u"%s-%s%s" % (root, query, ext)
ret = unquote(ret)
if ret is None:
ret = u'unknown'
if clean:
return clean_filename(ret)
else:
return unicode_to_filename(ret)
except (TypeError, KeyError, AttributeError, UnicodeDecodeError):
return unicode_to_filename(u'unknown')
def filename_from_url(url, clean=False):
"""Returns a reasonable filename for saving the given url.
"""
check_u(url)
try:
match = URI_PATTERN.match(url)
if match is None:
# This code path will never be executed.
return unicode_to_filename(url)
filename = match.group(2)
query = match.group(4)
if not filename:
ret = query
elif not query:
ret = filename
else:
root, ext = os.path.splitext(filename)
ret = u"%s-%s%s" % (root, query, ext)
ret = unquote(ret)
if ret is None:
ret = u'unknown'
if clean:
return clean_filename(ret)
else:
return unicode_to_filename(ret)
except (SystemExit, KeyboardInterrupt):
raise
except:
return unicode_to_filename(u'unknown')
def __init__(self, name, **kwargs):
self.name = name
self.__dict__.update(kwargs)
if 'audio_path' in kwargs:
self.audio_path = unicode_to_filename(self.audio_path)
if 'video_path' in kwargs:
self.video_path = unicode_to_filename(self.video_path)
def __init__(self, device_name, children, **kwargs):
self.device_name = self.name = device_name
self.__dict__.update(kwargs)
if 'audio_path' in kwargs:
self.audio_path = unicode_to_filename(self.audio_path)
if 'video_path' in kwargs:
self.video_path = unicode_to_filename(self.video_path)
self.devices = {}
for info in children:
self.add_device(info)
def get_downloader_for_item(item):
existing = get_existing_downloader(item)
if existing:
return existing
url = item.get_url()
existing = get_existing_downloader_by_url(url)
if existing:
return existing
channel_name = unicode_to_filename(item.get_channel_title(True))
if not channel_name:
channel_name = None
if url.startswith(u'file://'):
path = get_file_url_path(url)
try:
get_torrent_info_hash(path)
except ValueError:
raise ValueError("Don't know how to handle %s" % url)
except (OSError, IOError):
return None
else:
return RemoteDownloader(url, item, u'application/x-bittorrent',
channel_name=channel_name)
elif is_magnet_uri(url):
return RemoteDownloader(url, item, u'application/x-magnet')
else:
return RemoteDownloader(url, item, channel_name=channel_name)
def build_output_paths(item_info, target_folder, converter_info):
"""Returns final_output_path and temp_output_path.
We base the temp path on temp filenames.
We base the final path on the item title.
"""
if target_folder is None:
use_temp_dir = True
target_folder = get_conversions_folder()
else:
use_temp_dir = False
input_path = item_info.video_path
basename = os.path.basename(input_path)
title = utils.unicode_to_filename(item_info.name, target_folder).strip()
if not title:
title = basename
if converter_info:
target_name = "%s.%s.%s" % (title, converter_info.identifier,
converter_info.extension)
else:
target_name = basename
final_path = FilenameType(os.path.join(target_folder, target_name))
if not use_temp_dir:
# convert directly onto the device
temp_path = final_path + '.tmp'
else:
temp_dir = FilenameType(tempfile.mkdtemp("miro-conversion"))
temp_path = os.path.join(temp_dir, basename)
return (final_path, temp_path)
def get_downloader_for_item(item):
existing = get_existing_downloader(item)
if existing:
return existing
url = item.get_url()
existing = get_existing_downloader_by_url(url)
if existing:
return existing
channel_name = unicode_to_filename(item.get_channel_title(True))
if not channel_name:
channel_name = None
if url.startswith(u'file://'):
path = get_file_url_path(url)
try:
get_torrent_info_hash(path)
except ValueError:
raise ValueError("Don't know how to handle %s" % url)
except (OSError, IOError):
return None
else:
return RemoteDownloader(url, item, u'application/x-bittorrent',
channel_name=channel_name)
elif is_magnet_uri(url):
return RemoteDownloader(url, item, u'application/x-magnet')
else:
return RemoteDownloader(url, item, channel_name=channel_name)
def build_output_paths(item_info, target_folder, converter_info):
"""Returns final_output_path and temp_output_path.
We base the temp path on temp filenames.
We base the final path on the item title.
"""
if target_folder is None:
use_temp_dir = True
target_folder = get_conversions_folder()
else:
use_temp_dir = False
input_path = item_info.filename
basename = os.path.basename(input_path)
title = utils.unicode_to_filename(item_info.title, target_folder).strip()
if not title:
title = basename
if converter_info:
target_name = "%s.%s.%s" % (title, converter_info.identifier,
converter_info.extension)
else:
target_name = basename
final_path = FilenameType(os.path.join(target_folder, target_name))
if not use_temp_dir:
# convert directly onto the device
temp_path = final_path + '.tmp'
else:
temp_dir = FilenameType(tempfile.mkdtemp("miro-conversion"))
temp_path = os.path.join(temp_dir, basename)
return (final_path, temp_path)
def clean_filename(filename):
"""Given either a filename or a unicode "filename" return a valid
clean version of it.
"""
for char in (':', '?', '<', '>', '|', '*', '\\', '/', '"', '\'', '%'):
filename = filename.replace(char, '')
if len(filename) == 0:
return unicode_to_filename(u'_')
if len(filename) > MAX_FILENAME_LENGTH:
base, ext = os.path.splitext(filename)
ext = ext[:MAX_FILENAME_EXTENSION_LENGTH]
base = base[:MAX_FILENAME_LENGTH-len(ext)]
filename = base + ext
if isinstance(filename, str):
return unicode_to_filename(filename.decode('ascii', 'replace'))
else:
return unicode_to_filename(filename)
def get_filename(self):
self.dbItem.confirm_db_thread()
if self.url and self.url.startswith(u"file://"):
return get_file_url_path(self.url)
elif self.url and self.url.startswith(u"/"):
return unicode_to_filename(self.url)
else:
return self.filename
def clean_filename(filename):
"""Given either a filename or a unicode "filename" return a valid
clean version of it.
"""
for char in (':', '?', '<', '>', '|', '*', '\\', '/', '"', '\'', '%'):
filename = filename.replace(char, '')
if len(filename) == 0:
return unicode_to_filename(u'_')
if len(filename) > MAX_FILENAME_LENGTH:
base, ext = os.path.splitext(filename)
ext = ext[:MAX_FILENAME_EXTENSION_LENGTH]
base = base[:MAX_FILENAME_LENGTH-len(ext)]
filename = base + ext
if type(filename) == str:
return unicode_to_filename(filename.decode('ascii', 'replace'))
else:
return unicode_to_filename(filename)
def get_filename(self):
self.dbItem.confirm_db_thread()
if self.url and self.url.startswith(u"file://"):
return get_file_url_path(self.url)
elif self.url and self.url.startswith(u"/"):
return unicode_to_filename(self.url)
else:
return self.filename
def test_file_urlize_with_unicode(self):
# contrived way of getting unicode characters in a filename
basename = unicode_to_filename(u'b\u0103r')
directory = fileobject.FilenameType('/foo/')
filename = fileobject.FilenameType(directory + basename)
# "/foo/bar" with a breve over the "a"
self.assertEquals(filename.urlize(), "file:///foo/b%C4%83r")
# urlize() should convert it to utf-8 then quote it
self.assertEquals(type(filename.urlize()), str)
def add_fake_media_files_to_device(self):
self.device_item_filenames = [
unicode_to_filename(f.decode('utf-8'))
for f in ['foo.mp3', 'bar.mp3', 'foo.avi', 'bar.ogg']
]
for filename in self.device_item_filenames:
path = os.path.join(self.device.mount, filename)
with open(path, 'w') as f:
f.write("fake-data")
def _add_item(self, final_path, item_info):
dirname, basename = os.path.split(final_path)
_, extension = os.path.splitext(basename)
new_basename = "%s%s" % (unicode_to_filename(item_info.name,
self.device.mount),
extension)
new_path = os.path.join(dirname, new_basename)
if os.path.exists(new_path):
logging.debug('final destination %r exists, making a new one',
new_path)
new_path, fp = next_free_filename(new_path)
def callback():
if not os.path.exists(new_path):
return # copy failed, just give up
device_item = item.DeviceItem(
device=self.device,
file_type=item_info.file_type,
video_path=new_path[len(self.device.mount):],
title=item_info.name,
feed_name=item_info.feed_name,
feed_url=item_info.feed_url,
description=item_info.description,
release_date=time.mktime(item_info.release_date.timetuple()),
duration=(item_info.duration and item_info.duration * 1000 or
None),
permalink=item_info.permalink,
commentslink=item_info.commentslink,
payment_link=item_info.payment_link,
screenshot=item_info.thumbnail,
thumbnail_url=item_info.thumbnail_url,
file_format=item_info.file_format,
license=item_info.license,
url=item_info.file_url,
media_type_checked=item_info.media_type_checked,
mime_type=item_info.mime_type,
creation_time=time.mktime(item_info.date_added.timetuple()),
title_tag=item_info.title_tag,
artist=item_info.artist,
album=item_info.album,
track=item_info.track,
year=item_info.year,
genre=item_info.genre,
metadata_version=item_info.metadata_version,
mdp_state=item_info.mdp_state,
auto_sync=getattr(item_info, 'auto_sync', False)
)
device_item._migrate_thumbnail()
database = self.device.database
database.setdefault(device_item.file_type, {})
database[device_item.file_type][device_item.id] = \
device_item.to_dict()
database.emit('item-added', device_item)
self.device.remaining -= device_item.size
fileutil.migrate_file(final_path, new_path, callback)
def add_fake_media_files_to_device(self):
self.device_item_filenames = [
unicode_to_filename(f.decode('utf-8')) for f in
['foo.mp3', 'bar.mp3', 'foo.avi', 'bar.ogg']
]
for filename in self.device_item_filenames:
path = os.path.join(self.device.mount, filename)
with open(path, 'w') as f:
f.write("fake-data")
def test_file_urlize_with_unicode(self):
# contrived way of getting unicode characters in a filename
basename = unicode_to_filename(u'b\u0103r')
directory = fileobject.FilenameType('/foo/')
filename = fileobject.FilenameType(directory + basename)
# "/foo/bar" with a breve over the "a"
self.assertEquals(filename.urlize(), "file:///foo/b%C4%83r")
# urlize() should convert it to utf-8 then quote it
self.assertEquals(type(filename.urlize()), str)
def icon_path_for_engine(engine):
engine_name = unicode_to_filename(engine.name)
icon_path = resources.path('images/search_icon_%s.png' % engine_name)
if app.config.get(prefs.THEME_NAME):
logging.debug('engine %s filename: %s' % (engine.name, engine.filename))
test_icon_path = resources.theme_path(app.config.get(prefs.THEME_NAME),
'images/search_icon_%s.png' %
engine_name)
if os.path.exists(test_icon_path):
# this search engine came from a theme; look up the icon in the
# theme directory instead
icon_path = test_icon_path
return icon_path
def _add_item(self, final_path, item_info):
dirname, basename = os.path.split(final_path)
_, extension = os.path.splitext(basename)
new_basename = "%s%s" % (unicode_to_filename(
item_info.name, self.device.mount), extension)
new_path = os.path.join(dirname, new_basename)
def callback():
if not os.path.exists(new_path):
return # copy failed, just give up
device_item = DeviceItem(
device=self.device,
file_type=item_info.file_type,
video_path=new_path[len(self.device.mount):],
title=item_info.name,
feed_name=item_info.feed_name,
feed_url=item_info.feed_url,
description=item_info.description,
release_date=time.mktime(item_info.release_date.timetuple()),
duration=(item_info.duration and item_info.duration * 1000
or None),
permalink=item_info.permalink,
commentslink=item_info.commentslink,
payment_link=item_info.payment_link,
screenshot=item_info.thumbnail,
thumbnail_url=item_info.thumbnail_url,
file_format=item_info.file_format,
license=item_info.license,
url=item_info.file_url,
media_type_checked=item_info.media_type_checked,
mime_type=item_info.mime_type,
creation_time=time.mktime(item_info.date_added.timetuple()),
title_tag=item_info.title_tag,
artist=item_info.artist,
album=item_info.album,
track=item_info.track,
year=item_info.year,
genre=item_info.genre,
metadata_version=item_info.metadata_version,
mdp_state=item_info.mdp_state,
)
device_item._migrate_thumbnail()
database = self.device.database
database.setdefault(device_item.file_type, {})
database[device_item.file_type][device_item.id] = \
device_item.to_dict()
database.emit('item-added', device_item)
fileutil.migrate_file(final_path, new_path, callback)
def _add_item(self, final_path, item_info):
dirname, basename = os.path.split(final_path)
_, extension = os.path.splitext(basename)
new_basename = "%s%s" % (unicode_to_filename(item_info.name),
extension)
new_path = os.path.join(dirname, new_basename)
os.rename(final_path, new_path)
device_item = item.DeviceItem(
device=self.device,
file_type=item_info.file_type,
video_path=new_path[len(self.device.mount):],
name=item_info.name,
feed_name=item_info.feed_name,
feed_url=item_info.feed_url,
description=item_info.description,
release_date=time.mktime(item_info.release_date.timetuple()),
duration=item_info.duration * 1000,
permalink=item_info.permalink,
commentslink=item_info.commentslink,
payment_link=item_info.payment_link,
screenshot=item_info.thumbnail,
thumbnail_url=item_info.thumbnail_url,
file_format=item_info.file_format,
license=item_info.license,
url=item_info.file_url,
media_type_checked=item_info.media_type_checked,
mime_type=item_info.mime_type)
device_item._migrate_thumbnail()
database = self.device.database
database.setdefault(device_item.file_type, [])
database[device_item.file_type][device_item.video_path] = \
device_item.to_dict()
messages.ItemsChanged('device', '%s-%s' % (self.device.id,
device_item.file_type),
[messages.ItemInfo(device_item)], # added
[], []).send_to_frontend() # changed, removed
def make_device_item(device, filename):
# ensure that filename is the correct type for our platform
filename = unicode_to_filename(unicode(filename))
ensure_file_exists(os.path.join(device.mount, filename))
return item.DeviceItem(device, filename)
def test_extendend_chars(self):
filename = unicode_to_filename(u'\ufffdxtended Chars')
self.add_file_to_support_dir(filename, 'xtended Chars')
self.check_backup()
def add_item(self, filename):
path = os.path.join(self.tempdir, unicode_to_filename(filename))
# create a bogus file so we don't get a warning when we create a
# filename.
open(path, 'wb').write("data")
self.added_items[path] = FileItem(path, self.feed.id)
def update_icon_cache(self, url, info):
self.dbItem.confirm_db_thread()
if self.removed:
app.icon_cache_updater.update_finished()
return
needs_save = False
needsChange = False
if info == None or (info['status'] != 304 and info['status'] != 200):
self.error_callback(url, "bad response")
return
try:
# Our cache is good. Hooray!
if info['status'] == 304:
return
needsChange = True
# We have to update it, and if we can't write to the file, we
# should pick a new filename.
if ((self.filename and
not fileutil.access(self.filename, os.R_OK | os.W_OK))):
self.filename = None
cachedir = app.config.get(prefs.ICON_CACHE_DIRECTORY)
try:
fileutil.makedirs(cachedir)
except OSError:
pass
try:
# Write to a temp file.
if self.filename:
tmp_filename = self.filename + ".part"
else:
tmp_filename = os.path.join(cachedir, info["filename"]) + ".part"
tmp_filename, output = next_free_filename(tmp_filename)
output.write(info["body"])
output.close()
except IOError:
self.remove_file(tmp_filename)
return
except ValueError:
logging.warn('update_icon_cache: next_free_filename failed '
'#1, candidate = %r', tmp_filename)
return
filename = unicode(info["filename"])
filename = unicode_to_filename(filename, cachedir)
filename = os.path.join(cachedir, filename)
needs_save = True
try:
filename, fp = next_free_filename(filename)
except ValueError:
logging.warn('update_icon_cache: next_free_filename failed '
'#2, candidate = %r', filename)
return
if self.filename:
filename = self.filename
self.filename = None
self.remove_file(filename)
# we need to move the file here--so we close the file
# pointer and then move the file.
fp.close()
try:
self.remove_file(filename)
fileutil.rename(tmp_filename, filename)
except (IOError, OSError):
logging.exception("iconcache: fileutil.move failed")
filename = None
self.filename = filename
etag = unicodify(info.get("etag"))
modified = unicodify(info.get("modified"))
if self.etag != etag:
needs_save = True
self.etag = etag
if self.modified != modified:
needs_save = True
self.modified = modified
if self.url != url:
needs_save = True
self.url = url
finally:
if needsChange:
self.icon_changed(needs_save=needs_save)
self.updating = False
if self.needsUpdate:
self.needsUpdate = False
self.request_update(True)
app.icon_cache_updater.update_finished()
def remove_item(self, filename):
path = os.path.join(self.tempdir, unicode_to_filename(filename))
self.added_items[path].remove()
del self.added_items[path]
self.deleted_paths.append(path)
def update_icon_cache(self, url, info):
self.dbItem.confirm_db_thread()
if self.removed:
icon_cache_updater.update_finished()
return
needs_save = False
needsChange = False
if info == None or (info['status'] != 304 and info['status'] != 200):
self.error_callback(url, "bad response")
return
try:
# Our cache is good. Hooray!
if info['status'] == 304:
return
needsChange = True
# We have to update it, and if we can't write to the file, we
# should pick a new filename.
if ((self.filename and
not fileutil.access(self.filename, os.R_OK | os.W_OK))):
self.filename = None
cachedir = app.config.get(prefs.ICON_CACHE_DIRECTORY)
try:
fileutil.makedirs(cachedir)
except OSError:
pass
try:
# Write to a temp file.
if self.filename:
tmp_filename = self.filename + ".part"
else:
tmp_filename = os.path.join(cachedir, info["filename"]) + ".part"
tmp_filename, output = next_free_filename(tmp_filename)
output.write(info["body"])
output.close()
except IOError:
self.remove_file(tmp_filename)
return
except ValueError:
logging.warn('update_icon_cache: next_free_filename failed '
'#1, candidate = %r', tmp_filename)
return
filename = unicode(info["filename"])
filename = unicode_to_filename(filename, cachedir)
filename = os.path.join(cachedir, filename)
needs_save = True
try:
filename, fp = next_free_filename(filename)
except ValueError:
logging.warn('update_icon_cache: next_free_filename failed '
'#2, candidate = %r', filename)
return
if self.filename:
filename = self.filename
self.filename = None
self.remove_file(filename)
# we need to move the file here--so we close the file
# pointer and then move the file.
fp.close()
try:
self.remove_file(filename)
fileutil.rename(tmp_filename, filename)
except (IOError, OSError):
logging.exception("iconcache: fileutil.move failed")
filename = None
self.filename = filename
etag = unicodify(info.get("etag"))
modified = unicodify(info.get("modified"))
if self.etag != etag:
needs_save = True
self.etag = etag
if self.modified != modified:
needs_save = True
self.modified = modified
if self.url != url:
needs_save = True
self.url = url
finally:
if needsChange:
self.icon_changed(needs_save=needs_save)
self.updating = False
if self.needsUpdate:
self.needsUpdate = False
self.request_update(True)
icon_cache_updater.update_finished()
def add_item(self, filename):
path = '/videos/' + unicode_to_filename(filename)
self.added_items[path] = FileItem(path, self.feed.id)
def make_device_item(device, filename):
# ensure that filename is the correct type for our platform
filename = unicode_to_filename(unicode(filename))
ensure_file_exists(os.path.join(device.mount, filename))
return item.DeviceItem(device, filename)
def remove_item(self, filename):
path = os.path.join(self.tempdir, unicode_to_filename(filename))
self.added_items[path].remove()
del self.added_items[path]
self.deleted_paths.append(path)
def test_extendend_chars(self):
filename = unicode_to_filename(u'\ufffdxtended Chars')
self.add_file_to_support_dir(filename, 'xtended Chars')
self.check_backup()
def remove_item(self, filename):
path = '/videos/' + unicode_to_filename(filename)
self.added_items[path].remove()
del self.added_items[path]
self.deleted_paths.append(path)
def add_item(self, filename):
path = os.path.join(self.tempdir, unicode_to_filename(filename))
# create a bogus file so we don't get a warning when we create a
# filename.
open(path, 'wb').write("data")
self.added_items[path] = FileItem(path, self.feed.id)
def update_icon_cache(self, url, info):
self.dbItem.confirm_db_thread()
if self.removed:
iconCacheUpdater.update_finished()
return
needs_save = False
needsChange = False
if info == None or (info['status'] != 304 and info['status'] != 200):
self.error_callback(url, "bad response")
return
try:
# Our cache is good. Hooray!
if info['status'] == 304:
return
needsChange = True
# We have to update it, and if we can't write to the file, we
# should pick a new filename.
if self.filename and not fileutil.access(self.filename, os.R_OK | os.W_OK):
self.filename = None
cachedir = app.config.get(prefs.ICON_CACHE_DIRECTORY)
try:
fileutil.makedirs(cachedir)
except OSError:
pass
try:
# Write to a temp file.
if self.filename:
tmp_filename = self.filename + ".part"
else:
tmp_filename = os.path.join(cachedir, info["filename"]) + ".part"
tmp_filename, output = next_free_filename(tmp_filename)
output.write(info["body"])
output.close()
except IOError:
self.remove_file(tmp_filename)
return
if self.filename:
self.remove_file(self.filename)
# Create a new filename always to avoid browser caching in case a
# file changes.
# Add a random unique id
parts = unicodify(info["filename"]).split('.')
uid = u"%08d" % random.randint(0, 99999999)
if len(parts) == 1:
parts.append(uid)
else:
parts[-1:-1] = [uid]
self.filename = u'.'.join(parts)
self.filename = unicode_to_filename(self.filename, cachedir)
self.filename = os.path.join(cachedir, self.filename)
self.filename, fp = next_free_filename(self.filename)
needs_save = True
try:
fileutil.rename(tmp_filename, self.filename)
except OSError:
self.filename = None
needs_save = True
fp.close()
etag = unicodify(info.get("etag"))
modified = unicodify(info.get("modified"))
if self.etag != etag:
needs_save = True
self.etag = etag
if self.modified != modified:
needs_save = True
self.modified = modified
if self.url != url:
needs_save = True
self.url = url
finally:
if needsChange:
self.icon_changed(needs_save=needs_save)
self.updating = False
if self.needsUpdate:
self.needsUpdate = False
self.request_update(True)
iconCacheUpdater.update_finished()