def test_response(self):
with temp_filename() as fn:
mock = Mock(resp=bytes2fsn(b"resp", None))
remote = QuodLibetUnixRemote(None, mock)
remote._callback(b"\x00foo\x00" + fsn2bytes(fn, None) + b"\x00")
self.assertEqual(mock.lines, [bytes2fsn(b"foo", None)])
with open(fn, "rb") as h:
self.assertEqual(h.read(), b"resp")
def _callback(self, data):
try:
messages = list(fifo.split_message(data))
except ValueError:
print_w("invalid message: %r" % data)
return
for command, path in messages:
command = bytes2fsn(command, None)
response = self._cmd_registry.handle_line(self._app, command)
if path is not None:
path = bytes2fsn(path, None)
with open(path, "wb") as h:
if response is not None:
assert isinstance(response, fsnative)
h.write(fsn2bytes(response, None))
def from_dump(self, text):
"""Parses the text created with to_dump and adds the found tags.
Args:
text (bytes)
"""
for line in text.split(b"\n"):
if not line:
continue
parts = line.split(b"=")
key = decode(parts[0])
val = b"=".join(parts[1:])
if key == "~format":
pass
elif key in FILESYSTEM_TAGS:
self.add(key, bytes2fsn(val, "utf-8"))
elif key.startswith("~#"):
try:
self.add(key, int(val))
except ValueError:
try:
self.add(key, float(val))
except ValueError:
pass
else:
self.add(key, decode(val))
def test_path(self):
try:
path = bytes2fsn(b"\xff\xff", "utf-8")
except ValueError:
return
assert decode_value("~filename", path) == fsn2text(path)
def parse_xdg_user_dirs(data):
"""Parses xdg-user-dirs and returns a dict of keys and paths.
The paths depend on the content of environ while calling this function.
See http://www.freedesktop.org/wiki/Software/xdg-user-dirs/
Args:
data (bytes)
Can't fail (but might return garbage).
"""
assert isinstance(data, bytes)
paths = {}
for line in data.splitlines():
if line.startswith(b"#"):
continue
parts = line.split(b"=", 1)
if len(parts) <= 1:
continue
key = parts[0]
try:
values = shlex.split(bytes2fsn(parts[1], "utf-8"))
except ValueError:
continue
if len(values) != 1:
continue
paths[key] = os.path.normpath(expandvars(values[0]))
return paths
def test_fifo(self):
mock = Mock()
remote = QuodLibetUnixRemote(None, mock)
remote._callback(b"foo\n")
remote._callback(b"bar\nbaz")
self.assertEqual(
mock.lines, [bytes2fsn(b, None) for b in [b"foo", b"bar", b"baz"]])
def glib2fsn(path):
"""Takes a glib filename and returns a fsnative path"""
if PY2:
return bytes2fsn(path, "utf-8")
else:
return path
def test_file_encoding(self):
if os.name == "nt":
return
f = self.add_file(bytes2fsn(b"\xff\xff\xff\xff - cover.jpg", None))
self.assertTrue(isinstance(self.song("album"), text_type))
h = self._find_cover(self.song)
self.assertEqual(h.name, normalize_path(f))
def get_exclude_dirs():
"""Returns a list of paths which should be ignored during scanning
Returns:
list
"""
paths = split_scan_dirs(bytes2fsn(config.getbytes("library", "exclude"), "utf-8"))
return [expanduser(p) for p in paths]
def restore(self):
data = config.getbytes("browsers", "filesystem", b"")
try:
paths = bytes2fsn(data, "utf-8")
except ValueError:
return
if not paths:
return
self.__select_paths(paths.split("\n"))
def get_scan_dirs():
"""Returns a list of paths which should be scanned
Returns:
list
"""
joined_paths = bytes2fsn(config.getbytes("settings", "scan"), "utf-8")
return [expanduser(p) for p in split_scan_dirs(joined_paths)]
def selection_get_filenames(selection_data):
"""Extracts the filenames of songs set with selection_set_songs()
from a Gtk.SelectionData.
"""
data_type = selection_data.get_data_type()
assert data_type.name() == "text/x-quodlibet-songs"
items = selection_data.get_data().split(b"\x00")
return [bytes2fsn(i, "utf-8") for i in items]
def test_main(self):
v = fsnative(u"foo")
self.assertTrue(isinstance(v, fsnative))
v2 = glib2fsn(fsn2glib(v))
self.assertTrue(isinstance(v2, fsnative))
self.assertEqual(v, v2)
v3 = bytes2fsn(fsn2bytes(v, "utf-8"), "utf-8")
self.assertTrue(isinstance(v3, fsnative))
self.assertEqual(v, v3)
def escape_filename(s):
"""Escape a string in a manner suitable for a filename.
Args:
s (text_type)
Returns:
fsnative
"""
s = text_type(s)
s = quote(s.encode("utf-8"), safe=b"")
if isinstance(s, text_type):
s = s.encode("ascii")
return bytes2fsn(s, "utf-8")
def test_uri(self):
# On windows where we have unicode paths (windows encoding is utf-16)
# we need to encode to utf-8 first, then escape.
# On linux we take the byte stream and escape it.
# see g_filename_to_uri
if os.name == "nt":
f = AudioFile({"~filename": u"/\xf6\xe4.mp3", "title": "win"})
self.failUnlessEqual(f("~uri"), "file:///%C3%B6%C3%A4.mp3")
else:
f = AudioFile({
"~filename": bytes2fsn(b"/\x87\x12.mp3", None),
"title": "linux",
})
self.failUnlessEqual(f("~uri"), "file:///%87%12.mp3")
def _normalize_darwin_path(filename, canonicalise=False):
filename = path2fsn(filename)
if canonicalise:
filename = os.path.realpath(filename)
filename = os.path.normpath(filename)
data = fsn2bytes(filename, "utf-8")
decoded = data.decode("utf-8", "quodlibet-osx-path-decode")
try:
return bytes2fsn(
NSString.fileSystemRepresentation(decoded), "utf-8")
except ValueError:
return filename
def parse_m3u(filename, library=None):
plname = fsn2text(path2fsn(os.path.basename(
os.path.splitext(filename)[0])))
filenames = []
with open(filename, "rb") as h:
for line in h:
line = line.strip()
if line.startswith(b"#"):
continue
else:
try:
filenames.append(bytes2fsn(line, "utf-8"))
except ValueError:
continue
return __parse_playlist(plname, filename, filenames, library)
def parse_gtk_bookmarks(data):
"""
Args:
data (bytes)
Retruns:
List[fsnative]
Raises:
ValueError
"""
assert isinstance(data, bytes)
paths = []
for line in data.splitlines():
parts = line.split()
if not parts:
continue
folder_url = parts[0]
paths.append(bytes2fsn(urlsplit(folder_url)[2], "utf-8"))
return paths
def __populate_from_file(self):
library = self.library
try:
with open(self.filename, "rb") as h:
for line in h:
assert library is not None
try:
line = bytes2fsn(line.rstrip(), "utf-8")
except ValueError:
# decoding failed
continue
if line in library:
self._list.append(library[line])
elif library and library.masked(line):
self._list.append(line)
except IOError:
if self.name:
util.print_d(
"Playlist '%s' not found, creating new." % self.name)
self.write()
def parse_pls(filename, name="", library=None):
plname = fsn2text(path2fsn(os.path.basename(
os.path.splitext(filename)[0])))
filenames = []
with open(filename, "rb") as h:
for line in h:
line = line.strip()
if not line.lower().startswith(b"file"):
continue
else:
try:
line = line[line.index(b"=") + 1:].strip()
except ValueError:
pass
else:
try:
filenames.append(bytes2fsn(line, "utf-8"))
except ValueError:
continue
return __parse_playlist(plname, filename, filenames, library)
def _py2_to_py3(items):
assert PY3
for i in items:
try:
l = list(i.items())
except AttributeError:
raise SerializationError
i.clear()
for k, v in l:
if isinstance(k, bytes):
k = k.decode("utf-8", "replace")
else:
# strip surrogates
try:
k.encode("utf-8")
except UnicodeEncodeError:
k = k.encode("utf-8", "replace").decode("utf-8")
if k == "~filename" or k == "~mountpoint":
if isinstance(v, bytes):
try:
v = bytes2fsn(v, "utf-8")
except ValueError:
# just in case, only on Windows
assert is_windows()
v = v.decode("utf-8", "replace")
elif isinstance(v, bytes):
v = v.decode("utf-8", "replace")
elif isinstance(v, text_type):
# strip surrogates
try:
v.encode("utf-8")
except UnicodeEncodeError:
v = v.encode("utf-8", "replace").decode("utf-8")
i[k] = v
return items
def from_dump(self, text):
"""Parses the text created with to_dump and adds the found tags.
Args:
text (bytes)
"""
def decode_key(key):
"""str if ascii, otherwise decode using utf-8"""
if PY3:
return decode(key)
try:
key.decode("ascii")
except ValueError:
return decode(key)
return key
for line in text.split(b"\n"):
if not line:
continue
parts = line.split(b"=")
key = decode_key(parts[0])
val = b"=".join(parts[1:])
if key == "~format":
pass
elif key in FILESYSTEM_TAGS:
self.add(key, bytes2fsn(val, "utf-8"))
elif key.startswith("~#"):
try:
self.add(key, int(val))
except ValueError:
try:
self.add(key, float(val))
except ValueError:
pass
else:
self.add(key, decode(val))
def get_current_dir():
"""Returns the currently active chooser directory path.
The path might not actually exist.
Returns:
fsnative
"""
data = config.getbytes("memory", "chooser_dir", b"")
try:
path = bytes2fsn(data, "utf-8") or None
except ValueError:
path = None
# the last user dir might not be there any more, try showing a parent
# instead
if path is not None:
path = find_nearest_dir(path)
if path is None:
path = get_home_dir()
return path
def parse_command(line):
"""Parses a MPD command (without trailing newline)
Returns (command, [arguments]) or raises ParseError in case of an error.
"""
assert isinstance(line, bytes)
parts = re.split(b"[ \\t]+", line, maxsplit=1)
if not parts:
raise ParseError("empty command")
command = parts[0]
if len(parts) > 1:
lex = shlex.shlex(bytes2fsn(parts[1], "utf-8"), posix=True)
lex.whitespace_split = True
lex.commenters = ""
lex.quotes = "\""
lex.whitespace = " \t"
args = [fsn2bytes(a, "utf-8") for a in lex]
else:
args = []
try:
command = command.decode("utf-8")
except ValueError as e:
raise ParseError(e)
dec_args = []
for arg in args:
try:
arg = arg.decode("utf-8")
except ValueError as e:
raise ParseError(e)
dec_args.append(arg)
return command, dec_args
def _py2_to_py3(items):
for i in items:
try:
l = list(i.items())
except AttributeError:
raise SerializationError
i.clear()
for k, v in l:
if isinstance(k, bytes):
k = k.decode("utf-8", "replace")
else:
# strip surrogates
try:
k.encode("utf-8")
except UnicodeEncodeError:
k = k.encode("utf-8", "replace").decode("utf-8")
if k == "~filename" or k == "~mountpoint":
if isinstance(v, bytes):
try:
v = bytes2fsn(v, "utf-8")
except ValueError:
# just in case, only on Windows
assert is_windows()
v = v.decode("utf-8", "replace")
elif isinstance(v, bytes):
v = v.decode("utf-8", "replace")
elif isinstance(v, str):
# strip surrogates
try:
v.encode("utf-8")
except UnicodeEncodeError:
v = v.encode("utf-8", "replace").decode("utf-8")
i[k] = v
return items
def parse_command(line):
"""Parses a MPD command (without trailing newline)
Returns (command, [arguments]) or raises ParseError in case of an error.
"""
assert isinstance(line, bytes)
parts = re.split(b"[ \\t]+", line, maxsplit=1)
if not parts:
raise ParseError("empty command")
command = parts[0]
if len(parts) > 1:
lex = shlex.shlex(bytes2fsn(parts[1], "utf-8"), posix=True)
lex.whitespace_split = True
lex.commenters = ""
lex.quotes = "\""
lex.whitespace = " \t"
args = [fsn2bytes(a, "utf-8") for a in lex]
else:
args = []
try:
command = command.decode("utf-8")
except ValueError as e:
raise ParseError(e)
dec_args = []
for arg in args:
try:
arg = arg.decode("utf-8")
except ValueError as e:
raise ParseError(e)
dec_args.append(arg)
return command, dec_args
def __fill(self, library):
try:
with open(QUEUE, "rb") as f:
lines = f.readlines()
except EnvironmentError:
return
filenames = []
for line in lines:
line = line.strip()
if not line:
continue
try:
filename = bytes2fsn(line, "utf-8")
except ValueError:
print_exc()
continue
filenames.append(filename)
if library.librarian:
library = library.librarian
songs = filter(None, map(library.get, filenames))
for song in songs:
self.model.append([song])
def __fill(self, library):
try:
with open(QUEUE, "rb") as f:
lines = f.readlines()
except EnvironmentError:
return
filenames = []
for line in lines:
line = line.strip()
if not line:
continue
try:
filename = bytes2fsn(line, "utf-8")
except ValueError:
print_exc()
continue
filenames.append(filename)
if library.librarian:
library = library.librarian
songs = filter(None, map(library.get, filenames))
for song in songs:
self.model.append([song])
def _callback(self, data):
message = bytes2fsn(data, "utf-8")
self._cmd_registry.handle_line(self._app, message)
def tmp_fifo_path(self, tmp_path):
self.registry = Mock(resp=bytes2fsn(b"response", None))
self.tmp_path = tmp_path
with mock.patch.object(QuodLibetUnixRemote, "_PATH",
str(tmp_path / "control")):
yield
def test_remote_send_message(self):
response = self.send_message(b"foo 42")
self.assertEqual(self.registry.lines, [bytes2fsn(b"foo 42", None)])
self.assertEqual(response, b"response")
def __attempt_add(filename, filenames):
try:
filenames.append(bytes2fsn(filename, 'utf-8'))
except ValueError:
return
def test_fsn2bytes_ill_formed_utf16():
p = bytes2fsn(b"a\x00\xe9\x00 \x00=\xd8=\xd8\xa9\xdc", "utf-16-le")
assert fsn2bytes(p, "utf-8") == b"a\xC3\xA9 \xED\xA0\xBD\xF0\x9F\x92\xA9"
assert fsn2bytes(u"aé " + u"\uD83D" + u"💩", "utf-16-le") == \
b'a\x00\xe9\x00 \x00=\xd8=\xd8\xa9\xdc'
def get_scan_dirs():
dirs = split_scan_dirs(config.get("settings", "scan"))
return [bytes2fsn(d, "utf-8") for d in dirs if d]
def cat(*args):
return fsn2bytes("".join([bytes2fsn(a, "utf-8") for a in args]),
"utf-8")
def __attempt_add(filename, filenames):
try:
filenames.append(bytes2fsn(filename, 'utf-8'))
except ValueError:
print_w(f"Ignoring invalid filename {filename!r}")
def test_surrogates():
if os.name == "nt":
assert fsn2bytes(u"\ud83d", "utf-16-le") == b"=\xd8"
assert bytes2fsn(b"\xd8=", "utf-16-be") == u"\ud83d"
with pytest.raises(ValueError):
bytes2fsn(b"\xd8=a", "utf-16-be")
with pytest.raises(ValueError):
bytes2fsn(b"=\xd8a", "utf-16-le")
# for utf-16-le we have a workaround
assert bytes2fsn(b"=\xd8", "utf-16-le") == u"\ud83d"
assert bytes2fsn(b"=\xd8=\xd8", "utf-16-le") == u"\ud83d\ud83d"
with pytest.raises(ValueError):
bytes2fsn(b"=\xd8\x00\x00", "utf-16-le")
# 4 byte code point
assert fsn2bytes(u"\U0001f600", "utf-16-le") == b"=\xd8\x00\xde"
assert bytes2fsn(b"=\xd8\x00\xde", "utf-16-le") == u"\U0001f600"
# 4 byte codepoint + lone surrogate
assert bytes2fsn(b"=\xd8\x00\xde=\xd8", "utf-16-le") == \
u"\U0001f600\ud83d"
with pytest.raises(UnicodeDecodeError):
bytes2fsn(b"a", "utf-16-le")
assert fsn2bytes(u"\ud83d", "utf-8") == b"\xed\xa0\xbd"
assert bytes2fsn(b"\xed\xa0\xbd", "utf-8") == u"\ud83d"
assert fsnative(u"\ud83d") == u"\ud83d"
assert fsn2text(u"\ud83d") == u"\ufffd"
# at least don't fail...
assert fsn2uri(u"C:\\\ud83d") == u"file:///C:/%ED%A0%BD"
else:
# this shouldn't fail and produce the same result on py2/3 at least.
assert fsn2bytes(fsnative(u"\ud83d"), None) == b"\xed\xa0\xbd"
text2fsn(fsn2text(fsnative(u"\ud83d")))
if PY2 and isunicodeencoding():
# under Python 2 we get surrogates, but we can't do anything about
# it since most codecs don't treat that as an error
assert fsn2text(fsnative(u"\ud83d")) == u"\ud83d"
else:
# under Python 3 the decoder don't allow surrogates
assert fsn2text(fsnative(u"\ud83d")) == u"\ufffd\ufffd\ufffd"
def __attempt_add(filename, filenames):
try:
filenames.append(bytes2fsn(filename, 'utf-8'))
except ValueError:
return
def _callback(self, data):
message = bytes2fsn(data, "utf-8")
self._cmd_registry.handle_line(self._app, message)
def glib2fsnative(path):
if PY2:
return bytes2fsn(path, "utf-8")
else:
return path
def bytes2fsnative(data):
return bytes2fsn(data, "utf-8")
def get_exclude_dirs() -> Iterable[fsnative]:
""":return: a list of paths which should be ignored during scanning"""
paths = split_scan_dirs(
bytes2fsn(config.getbytes("library", "exclude"), "utf-8"))
return [expanduser(p) for p in paths] # type: ignore
def test_bytes2fsn():
assert bytes2fsn(b"foo", "utf-8") == fsnative(u"foo")
assert (bytes2fsn(fsn2bytes(fsnative(u"\u1234"), "utf-8"),
"utf-8") == fsnative(u"\u1234"))
with pytest.raises(ValueError):
bytes2fsn(b"\x00", "utf-8")
with pytest.raises(ValueError):
bytes2fsn(b"\x00\x00", "utf-16-le")
with pytest.raises(TypeError):
bytes2fsn(object(), "utf-8")
with pytest.raises(TypeError):
bytes2fsn(u"data", "utf-8")
if os.name == "nt":
with pytest.raises(ValueError):
bytes2fsn(b"data", "notanencoding")
with pytest.raises(ValueError):
bytes2fsn(b"data", None)
with pytest.raises(TypeError):
bytes2fsn(b"data", object())
