def filename_from_url(url, clean=False): """Returns a reasonable filename for saving the given url. """ check_u(url) try: match = URI_PATTERN.match(url) if match is None: # This code path will never be executed. return unicode_to_filename(url) filename = match.group(2) query = match.group(4) if not filename: ret = query elif not query: ret = filename else: root, ext = os.path.splitext(filename) ret = u"%s-%s%s" % (root, query, ext) ret = unquote(ret) if ret is None: ret = u'unknown' if clean: return clean_filename(ret) else: return unicode_to_filename(ret) except (TypeError, KeyError, AttributeError, UnicodeDecodeError): return unicode_to_filename(u'unknown')
def filename_from_url(url, clean=False): """Returns a reasonable filename for saving the given url. """ check_u(url) try: match = URI_PATTERN.match(url) if match is None: # This code path will never be executed. return unicode_to_filename(url) filename = match.group(2) query = match.group(4) if not filename: ret = query elif not query: ret = filename else: root, ext = os.path.splitext(filename) ret = u"%s-%s%s" % (root, query, ext) ret = unquote(ret) if ret is None: ret = u'unknown' if clean: return clean_filename(ret) else: return unicode_to_filename(ret) except (SystemExit, KeyboardInterrupt): raise except: return unicode_to_filename(u'unknown')
def __init__(self, name, **kwargs): self.name = name self.__dict__.update(kwargs) if 'audio_path' in kwargs: self.audio_path = unicode_to_filename(self.audio_path) if 'video_path' in kwargs: self.video_path = unicode_to_filename(self.video_path)
def __init__(self, device_name, children, **kwargs): self.device_name = self.name = device_name self.__dict__.update(kwargs) if 'audio_path' in kwargs: self.audio_path = unicode_to_filename(self.audio_path) if 'video_path' in kwargs: self.video_path = unicode_to_filename(self.video_path) self.devices = {} for info in children: self.add_device(info)
def get_downloader_for_item(item): existing = get_existing_downloader(item) if existing: return existing url = item.get_url() existing = get_existing_downloader_by_url(url) if existing: return existing channel_name = unicode_to_filename(item.get_channel_title(True)) if not channel_name: channel_name = None if url.startswith(u'file://'): path = get_file_url_path(url) try: get_torrent_info_hash(path) except ValueError: raise ValueError("Don't know how to handle %s" % url) except (OSError, IOError): return None else: return RemoteDownloader(url, item, u'application/x-bittorrent', channel_name=channel_name) elif is_magnet_uri(url): return RemoteDownloader(url, item, u'application/x-magnet') else: return RemoteDownloader(url, item, channel_name=channel_name)
def build_output_paths(item_info, target_folder, converter_info): """Returns final_output_path and temp_output_path. We base the temp path on temp filenames. We base the final path on the item title. """ if target_folder is None: use_temp_dir = True target_folder = get_conversions_folder() else: use_temp_dir = False input_path = item_info.video_path basename = os.path.basename(input_path) title = utils.unicode_to_filename(item_info.name, target_folder).strip() if not title: title = basename if converter_info: target_name = "%s.%s.%s" % (title, converter_info.identifier, converter_info.extension) else: target_name = basename final_path = FilenameType(os.path.join(target_folder, target_name)) if not use_temp_dir: # convert directly onto the device temp_path = final_path + '.tmp' else: temp_dir = FilenameType(tempfile.mkdtemp("miro-conversion")) temp_path = os.path.join(temp_dir, basename) return (final_path, temp_path)
def build_output_paths(item_info, target_folder, converter_info): """Returns final_output_path and temp_output_path. We base the temp path on temp filenames. We base the final path on the item title. """ if target_folder is None: use_temp_dir = True target_folder = get_conversions_folder() else: use_temp_dir = False input_path = item_info.filename basename = os.path.basename(input_path) title = utils.unicode_to_filename(item_info.title, target_folder).strip() if not title: title = basename if converter_info: target_name = "%s.%s.%s" % (title, converter_info.identifier, converter_info.extension) else: target_name = basename final_path = FilenameType(os.path.join(target_folder, target_name)) if not use_temp_dir: # convert directly onto the device temp_path = final_path + '.tmp' else: temp_dir = FilenameType(tempfile.mkdtemp("miro-conversion")) temp_path = os.path.join(temp_dir, basename) return (final_path, temp_path)
def clean_filename(filename): """Given either a filename or a unicode "filename" return a valid clean version of it. """ for char in (':', '?', '<', '>', '|', '*', '\\', '/', '"', '\'', '%'): filename = filename.replace(char, '') if len(filename) == 0: return unicode_to_filename(u'_') if len(filename) > MAX_FILENAME_LENGTH: base, ext = os.path.splitext(filename) ext = ext[:MAX_FILENAME_EXTENSION_LENGTH] base = base[:MAX_FILENAME_LENGTH-len(ext)] filename = base + ext if isinstance(filename, str): return unicode_to_filename(filename.decode('ascii', 'replace')) else: return unicode_to_filename(filename)
def get_filename(self): self.dbItem.confirm_db_thread() if self.url and self.url.startswith(u"file://"): return get_file_url_path(self.url) elif self.url and self.url.startswith(u"/"): return unicode_to_filename(self.url) else: return self.filename
def clean_filename(filename): """Given either a filename or a unicode "filename" return a valid clean version of it. """ for char in (':', '?', '<', '>', '|', '*', '\\', '/', '"', '\'', '%'): filename = filename.replace(char, '') if len(filename) == 0: return unicode_to_filename(u'_') if len(filename) > MAX_FILENAME_LENGTH: base, ext = os.path.splitext(filename) ext = ext[:MAX_FILENAME_EXTENSION_LENGTH] base = base[:MAX_FILENAME_LENGTH-len(ext)] filename = base + ext if type(filename) == str: return unicode_to_filename(filename.decode('ascii', 'replace')) else: return unicode_to_filename(filename)
def test_file_urlize_with_unicode(self): # contrived way of getting unicode characters in a filename basename = unicode_to_filename(u'b\u0103r') directory = fileobject.FilenameType('/foo/') filename = fileobject.FilenameType(directory + basename) # "/foo/bar" with a breve over the "a" self.assertEquals(filename.urlize(), "file:///foo/b%C4%83r") # urlize() should convert it to utf-8 then quote it self.assertEquals(type(filename.urlize()), str)
def add_fake_media_files_to_device(self): self.device_item_filenames = [ unicode_to_filename(f.decode('utf-8')) for f in ['foo.mp3', 'bar.mp3', 'foo.avi', 'bar.ogg'] ] for filename in self.device_item_filenames: path = os.path.join(self.device.mount, filename) with open(path, 'w') as f: f.write("fake-data")
def _add_item(self, final_path, item_info): dirname, basename = os.path.split(final_path) _, extension = os.path.splitext(basename) new_basename = "%s%s" % (unicode_to_filename(item_info.name, self.device.mount), extension) new_path = os.path.join(dirname, new_basename) if os.path.exists(new_path): logging.debug('final destination %r exists, making a new one', new_path) new_path, fp = next_free_filename(new_path) def callback(): if not os.path.exists(new_path): return # copy failed, just give up device_item = item.DeviceItem( device=self.device, file_type=item_info.file_type, video_path=new_path[len(self.device.mount):], title=item_info.name, feed_name=item_info.feed_name, feed_url=item_info.feed_url, description=item_info.description, release_date=time.mktime(item_info.release_date.timetuple()), duration=(item_info.duration and item_info.duration * 1000 or None), permalink=item_info.permalink, commentslink=item_info.commentslink, payment_link=item_info.payment_link, screenshot=item_info.thumbnail, thumbnail_url=item_info.thumbnail_url, file_format=item_info.file_format, license=item_info.license, url=item_info.file_url, media_type_checked=item_info.media_type_checked, mime_type=item_info.mime_type, creation_time=time.mktime(item_info.date_added.timetuple()), title_tag=item_info.title_tag, artist=item_info.artist, album=item_info.album, track=item_info.track, year=item_info.year, genre=item_info.genre, metadata_version=item_info.metadata_version, mdp_state=item_info.mdp_state, auto_sync=getattr(item_info, 'auto_sync', False) ) device_item._migrate_thumbnail() database = self.device.database database.setdefault(device_item.file_type, {}) database[device_item.file_type][device_item.id] = \ device_item.to_dict() database.emit('item-added', device_item) self.device.remaining -= device_item.size fileutil.migrate_file(final_path, new_path, callback)
def icon_path_for_engine(engine): engine_name = unicode_to_filename(engine.name) icon_path = resources.path('images/search_icon_%s.png' % engine_name) if app.config.get(prefs.THEME_NAME): logging.debug('engine %s filename: %s' % (engine.name, engine.filename)) test_icon_path = resources.theme_path(app.config.get(prefs.THEME_NAME), 'images/search_icon_%s.png' % engine_name) if os.path.exists(test_icon_path): # this search engine came from a theme; look up the icon in the # theme directory instead icon_path = test_icon_path return icon_path
def _add_item(self, final_path, item_info): dirname, basename = os.path.split(final_path) _, extension = os.path.splitext(basename) new_basename = "%s%s" % (unicode_to_filename( item_info.name, self.device.mount), extension) new_path = os.path.join(dirname, new_basename) def callback(): if not os.path.exists(new_path): return # copy failed, just give up device_item = DeviceItem( device=self.device, file_type=item_info.file_type, video_path=new_path[len(self.device.mount):], title=item_info.name, feed_name=item_info.feed_name, feed_url=item_info.feed_url, description=item_info.description, release_date=time.mktime(item_info.release_date.timetuple()), duration=(item_info.duration and item_info.duration * 1000 or None), permalink=item_info.permalink, commentslink=item_info.commentslink, payment_link=item_info.payment_link, screenshot=item_info.thumbnail, thumbnail_url=item_info.thumbnail_url, file_format=item_info.file_format, license=item_info.license, url=item_info.file_url, media_type_checked=item_info.media_type_checked, mime_type=item_info.mime_type, creation_time=time.mktime(item_info.date_added.timetuple()), title_tag=item_info.title_tag, artist=item_info.artist, album=item_info.album, track=item_info.track, year=item_info.year, genre=item_info.genre, metadata_version=item_info.metadata_version, mdp_state=item_info.mdp_state, ) device_item._migrate_thumbnail() database = self.device.database database.setdefault(device_item.file_type, {}) database[device_item.file_type][device_item.id] = \ device_item.to_dict() database.emit('item-added', device_item) fileutil.migrate_file(final_path, new_path, callback)
def _add_item(self, final_path, item_info): dirname, basename = os.path.split(final_path) _, extension = os.path.splitext(basename) new_basename = "%s%s" % (unicode_to_filename(item_info.name), extension) new_path = os.path.join(dirname, new_basename) os.rename(final_path, new_path) device_item = item.DeviceItem( device=self.device, file_type=item_info.file_type, video_path=new_path[len(self.device.mount):], name=item_info.name, feed_name=item_info.feed_name, feed_url=item_info.feed_url, description=item_info.description, release_date=time.mktime(item_info.release_date.timetuple()), duration=item_info.duration * 1000, permalink=item_info.permalink, commentslink=item_info.commentslink, payment_link=item_info.payment_link, screenshot=item_info.thumbnail, thumbnail_url=item_info.thumbnail_url, file_format=item_info.file_format, license=item_info.license, url=item_info.file_url, media_type_checked=item_info.media_type_checked, mime_type=item_info.mime_type) device_item._migrate_thumbnail() database = self.device.database database.setdefault(device_item.file_type, []) database[device_item.file_type][device_item.video_path] = \ device_item.to_dict() messages.ItemsChanged('device', '%s-%s' % (self.device.id, device_item.file_type), [messages.ItemInfo(device_item)], # added [], []).send_to_frontend() # changed, removed
def make_device_item(device, filename): # ensure that filename is the correct type for our platform filename = unicode_to_filename(unicode(filename)) ensure_file_exists(os.path.join(device.mount, filename)) return item.DeviceItem(device, filename)
def test_extendend_chars(self): filename = unicode_to_filename(u'\ufffdxtended Chars') self.add_file_to_support_dir(filename, 'xtended Chars') self.check_backup()
def add_item(self, filename): path = os.path.join(self.tempdir, unicode_to_filename(filename)) # create a bogus file so we don't get a warning when we create a # filename. open(path, 'wb').write("data") self.added_items[path] = FileItem(path, self.feed.id)
def update_icon_cache(self, url, info): self.dbItem.confirm_db_thread() if self.removed: app.icon_cache_updater.update_finished() return needs_save = False needsChange = False if info == None or (info['status'] != 304 and info['status'] != 200): self.error_callback(url, "bad response") return try: # Our cache is good. Hooray! if info['status'] == 304: return needsChange = True # We have to update it, and if we can't write to the file, we # should pick a new filename. if ((self.filename and not fileutil.access(self.filename, os.R_OK | os.W_OK))): self.filename = None cachedir = app.config.get(prefs.ICON_CACHE_DIRECTORY) try: fileutil.makedirs(cachedir) except OSError: pass try: # Write to a temp file. if self.filename: tmp_filename = self.filename + ".part" else: tmp_filename = os.path.join(cachedir, info["filename"]) + ".part" tmp_filename, output = next_free_filename(tmp_filename) output.write(info["body"]) output.close() except IOError: self.remove_file(tmp_filename) return except ValueError: logging.warn('update_icon_cache: next_free_filename failed ' '#1, candidate = %r', tmp_filename) return filename = unicode(info["filename"]) filename = unicode_to_filename(filename, cachedir) filename = os.path.join(cachedir, filename) needs_save = True try: filename, fp = next_free_filename(filename) except ValueError: logging.warn('update_icon_cache: next_free_filename failed ' '#2, candidate = %r', filename) return if self.filename: filename = self.filename self.filename = None self.remove_file(filename) # we need to move the file here--so we close the file # pointer and then move the file. fp.close() try: self.remove_file(filename) fileutil.rename(tmp_filename, filename) except (IOError, OSError): logging.exception("iconcache: fileutil.move failed") filename = None self.filename = filename etag = unicodify(info.get("etag")) modified = unicodify(info.get("modified")) if self.etag != etag: needs_save = True self.etag = etag if self.modified != modified: needs_save = True self.modified = modified if self.url != url: needs_save = True self.url = url finally: if needsChange: self.icon_changed(needs_save=needs_save) self.updating = False if self.needsUpdate: self.needsUpdate = False self.request_update(True) app.icon_cache_updater.update_finished()
def remove_item(self, filename): path = os.path.join(self.tempdir, unicode_to_filename(filename)) self.added_items[path].remove() del self.added_items[path] self.deleted_paths.append(path)
def update_icon_cache(self, url, info): self.dbItem.confirm_db_thread() if self.removed: icon_cache_updater.update_finished() return needs_save = False needsChange = False if info == None or (info['status'] != 304 and info['status'] != 200): self.error_callback(url, "bad response") return try: # Our cache is good. Hooray! if info['status'] == 304: return needsChange = True # We have to update it, and if we can't write to the file, we # should pick a new filename. if ((self.filename and not fileutil.access(self.filename, os.R_OK | os.W_OK))): self.filename = None cachedir = app.config.get(prefs.ICON_CACHE_DIRECTORY) try: fileutil.makedirs(cachedir) except OSError: pass try: # Write to a temp file. if self.filename: tmp_filename = self.filename + ".part" else: tmp_filename = os.path.join(cachedir, info["filename"]) + ".part" tmp_filename, output = next_free_filename(tmp_filename) output.write(info["body"]) output.close() except IOError: self.remove_file(tmp_filename) return except ValueError: logging.warn('update_icon_cache: next_free_filename failed ' '#1, candidate = %r', tmp_filename) return filename = unicode(info["filename"]) filename = unicode_to_filename(filename, cachedir) filename = os.path.join(cachedir, filename) needs_save = True try: filename, fp = next_free_filename(filename) except ValueError: logging.warn('update_icon_cache: next_free_filename failed ' '#2, candidate = %r', filename) return if self.filename: filename = self.filename self.filename = None self.remove_file(filename) # we need to move the file here--so we close the file # pointer and then move the file. fp.close() try: self.remove_file(filename) fileutil.rename(tmp_filename, filename) except (IOError, OSError): logging.exception("iconcache: fileutil.move failed") filename = None self.filename = filename etag = unicodify(info.get("etag")) modified = unicodify(info.get("modified")) if self.etag != etag: needs_save = True self.etag = etag if self.modified != modified: needs_save = True self.modified = modified if self.url != url: needs_save = True self.url = url finally: if needsChange: self.icon_changed(needs_save=needs_save) self.updating = False if self.needsUpdate: self.needsUpdate = False self.request_update(True) icon_cache_updater.update_finished()
def add_item(self, filename): path = '/videos/' + unicode_to_filename(filename) self.added_items[path] = FileItem(path, self.feed.id)
def remove_item(self, filename): path = '/videos/' + unicode_to_filename(filename) self.added_items[path].remove() del self.added_items[path] self.deleted_paths.append(path)
def update_icon_cache(self, url, info): self.dbItem.confirm_db_thread() if self.removed: iconCacheUpdater.update_finished() return needs_save = False needsChange = False if info == None or (info['status'] != 304 and info['status'] != 200): self.error_callback(url, "bad response") return try: # Our cache is good. Hooray! if info['status'] == 304: return needsChange = True # We have to update it, and if we can't write to the file, we # should pick a new filename. if self.filename and not fileutil.access(self.filename, os.R_OK | os.W_OK): self.filename = None cachedir = app.config.get(prefs.ICON_CACHE_DIRECTORY) try: fileutil.makedirs(cachedir) except OSError: pass try: # Write to a temp file. if self.filename: tmp_filename = self.filename + ".part" else: tmp_filename = os.path.join(cachedir, info["filename"]) + ".part" tmp_filename, output = next_free_filename(tmp_filename) output.write(info["body"]) output.close() except IOError: self.remove_file(tmp_filename) return if self.filename: self.remove_file(self.filename) # Create a new filename always to avoid browser caching in case a # file changes. # Add a random unique id parts = unicodify(info["filename"]).split('.') uid = u"%08d" % random.randint(0, 99999999) if len(parts) == 1: parts.append(uid) else: parts[-1:-1] = [uid] self.filename = u'.'.join(parts) self.filename = unicode_to_filename(self.filename, cachedir) self.filename = os.path.join(cachedir, self.filename) self.filename, fp = next_free_filename(self.filename) needs_save = True try: fileutil.rename(tmp_filename, self.filename) except OSError: self.filename = None needs_save = True fp.close() etag = unicodify(info.get("etag")) modified = unicodify(info.get("modified")) if self.etag != etag: needs_save = True self.etag = etag if self.modified != modified: needs_save = True self.modified = modified if self.url != url: needs_save = True self.url = url finally: if needsChange: self.icon_changed(needs_save=needs_save) self.updating = False if self.needsUpdate: self.needsUpdate = False self.request_update(True) iconCacheUpdater.update_finished()