def encode(self, command, source, dest, pretend=False):
"""Encode `source` to `dest` using command template `command`.
Raises `subprocess.CalledProcessError` if the command exited with a
non-zero status code.
"""
# The paths and arguments must be bytes.
assert isinstance(command, bytes)
assert isinstance(source, bytes)
assert isinstance(dest, bytes)
quiet = self.config['quiet'].get(bool)
if not quiet and not pretend:
self._log.info('Encoding {0}', util.displayable_path(source))
command = command.decode(arg_encoding(), 'surrogateescape')
source = decode_commandline_path(source)
dest = decode_commandline_path(dest)
# Substitute $source and $dest in the argument list.
args = shlex.split(command)
encode_cmd = []
for i, arg in enumerate(args):
args[i] = Template(arg).safe_substitute({
'source': source,
'dest': dest,
})
encode_cmd.append(args[i].encode(util.arg_encoding()))
if pretend:
self._log.info('{0}', ' '.join(ui.decargs(args)))
return
try:
util.command_output(encode_cmd)
except subprocess.CalledProcessError as exc:
# Something went wrong (probably Ctrl+C), remove temporary files
self._log.info('Encoding {0} failed. Cleaning up...',
util.displayable_path(source))
self._log.debug('Command {0} exited with status {1}: {2}',
args,
exc.returncode,
exc.output)
util.remove(dest)
util.prune_dirs(os.path.dirname(dest))
raise
except OSError as exc:
raise ui.UserError(
"convert: couldn't invoke '{}': {}".format(
' '.join(ui.decargs(args)), exc
)
)
if not quiet and not pretend:
self._log.info('Finished encoding {0}',
util.displayable_path(source))
def _convert_args(args):
"""Convert args to bytestrings for Python 2 and convert them to strings
on Python 3.
"""
for i, elem in enumerate(args):
if six.PY2:
if isinstance(elem, six.text_type):
args[i] = elem.encode(util.arg_encoding())
else:
if isinstance(elem, bytes):
args[i] = elem.decode(util.arg_encoding())
return args
def _convert_args(args):
"""Convert args to bytestrings for Python 2 and convert them to strings
on Python 3.
"""
for i, elem in enumerate(args):
if six.PY2:
if isinstance(elem, six.text_type):
args[i] = elem.encode(util.arg_encoding())
else:
if isinstance(elem, bytes):
args[i] = elem.decode(util.arg_encoding())
return args
def hook_function(**kwargs):
if command is None or len(command) == 0:
self._log.error('invalid command "{0}"', command)
return
# Use a string formatter that works on Unicode strings.
if six.PY2:
formatter = CodingFormatter(arg_encoding())
else:
formatter = string.Formatter()
command_pieces = shlex_split(command)
for i, piece in enumerate(command_pieces):
command_pieces[i] = formatter.format(piece,
event=event,
**kwargs)
self._log.debug(u'running command "{0}" for event {1}',
u' '.join(command_pieces), event)
try:
subprocess.Popen(command_pieces).wait()
except OSError as exc:
self._log.error(u'hook for {0} failed: {1}', event, exc)
def _store_dict(option, opt_str, value, parser):
"""Custom action callback to parse options which have ``key=value``
pairs as values. All such pairs passed for this option are
aggregated into a dictionary.
"""
dest = option.dest
option_values = getattr(parser.values, dest, None)
if option_values is None:
# This is the first supplied ``key=value`` pair of option.
# Initialize empty dictionary and get a reference to it.
setattr(parser.values, dest, {})
option_values = getattr(parser.values, dest)
# Decode the argument using the platform's argument encoding.
value = util.text_string(value, util.arg_encoding())
try:
key, value = value.split('=', 1)
if not (key and value):
raise ValueError
except ValueError:
raise UserError(
"supplied argument `{}' is not of the form `key=value'".format(
value))
option_values[key] = value
def hook_function(**kwargs):
if command is None or len(command) == 0:
self._log.error('invalid command "{0}"', command)
return
# Use a string formatter that works on Unicode strings.
formatter = CodingFormatter(arg_encoding())
command_pieces = shlex.split(command)
for i, piece in enumerate(command_pieces):
command_pieces[i] = formatter.format(piece,
event=event,
**kwargs)
self._log.debug('running command "{0}" for event {1}',
' '.join(command_pieces), event)
try:
subprocess.check_call(command_pieces)
except subprocess.CalledProcessError as exc:
self._log.error('hook for {0} exited with status {1}', event,
exc.returncode)
except OSError as exc:
self._log.error('hook for {0} failed: {1}', event, exc)
def _convert_args(args):
"""Convert args to strings
"""
for i, elem in enumerate(args):
if isinstance(elem, bytes):
args[i] = elem.decode(util.arg_encoding())
return args
def decargs(arglist):
"""Given a list of command-line argument bytestrings, attempts to
decode them to Unicode strings when running under Python 2.
"""
if six.PY2:
return [s.decode(util.arg_encoding()) for s in arglist]
else:
return arglist
def run_convert_path(self, path, *args):
"""Run the `convert` command on a given path."""
# The path is currently a filesystem bytestring. Convert it to
# an argument bytestring.
path = path.decode(util._fsencoding()).encode(util.arg_encoding())
args = args + (b'path:' + path,)
return self.run_command('convert', *args)
def decargs(arglist):
"""Given a list of command-line argument bytestrings, attempts to
decode them to Unicode strings when running under Python 2.
"""
if six.PY2:
return [s.decode(util.arg_encoding()) for s in arglist]
else:
return arglist
def run_convert_path(self, path, *args):
"""Run the `convert` command on a given path."""
# The path is currently a filesystem bytestring. Convert it to
# an argument bytestring.
path = path.decode(util._fsencoding()).encode(util.arg_encoding())
args = args + (b'path:' + path, )
return self.run_command('convert', *args)
def convert(in_file, out_file, tag):
"""Copy `in_file` to `out_file` and append the string `tag`.
"""
# On Python 3, encode the tag argument as bytes.
if not isinstance(tag, bytes):
tag = tag.encode("utf8")
# On Windows, use Unicode paths. (The test harness gives them to us
# as UTF-8 bytes.)
if platform.system() == "Windows":
if not six.PY2:
in_file = in_file.encode(arg_encoding())
out_file = out_file.encode(arg_encoding())
in_file = in_file.decode("utf-8")
out_file = out_file.decode("utf-8")
with open(out_file, "wb") as out_f:
with open(in_file, "rb") as in_f:
out_f.write(in_f.read())
out_f.write(tag)
def convert(in_file, out_file, tag):
"""Copy `in_file` to `out_file` and append the string `tag`.
"""
# On Python 3, encode the tag argument as bytes.
if not isinstance(tag, bytes):
tag = tag.encode('utf8')
# On Windows, use Unicode paths. (The test harness gives them to us
# as UTF-8 bytes.)
if platform.system() == 'Windows':
if not six.PY2:
in_file = in_file.encode(arg_encoding())
out_file = out_file.encode(arg_encoding())
in_file = in_file.decode('utf-8')
out_file = out_file.decode('utf-8')
with open(out_file, 'wb') as out_f:
with open(in_file, 'rb') as in_f:
out_f.write(in_f.read())
out_f.write(tag)
def setUp(self):
super(ExternalConvertWorkerTest, self).setUp(mock_worker=False)
external_dir = self.mkdtemp()
self.config['convert']['formats'] = {
'ogg': 'bash -c "cp \'{source}\' \'$dest\'"'.format(
# The convert plugin will encode this again using arg_encoding
source=self.item_fixture_path('ogg').decode(
util.arg_encoding()))
}
self.config['alternatives'] = {
'myexternal': {
'directory': external_dir,
'query': u'myexternal:true',
'formats': 'ogg'
}
}
def hook_function(**kwargs):
if command is None or len(command) == 0:
self._log.error('invalid command "{0}"', command)
return
# Use a string formatter that works on Unicode strings.
formatter = CodingFormatter(arg_encoding())
command_pieces = shlex_split(command)
for i, piece in enumerate(command_pieces):
command_pieces[i] = formatter.format(piece, event=event,
**kwargs)
self._log.debug(u'running command "{0}" for event {1}',
u' '.join(command_pieces), event)
try:
subprocess.Popen(command_pieces).wait()
except OSError as exc:
self._log.error(u'hook for {0} failed: {1}', event, exc)
def test_convert_command_args_keeps_undecodeable_bytes(self):
arg = b'\x82' # non-ascii bytes
cmd_args = util.convert_command_args([arg])
self.assertEqual(cmd_args[0],
arg.decode(util.arg_encoding(), 'surrogateescape'))
def test_convert_command_args_keeps_undecodeable_bytes(self):
arg = b'\x82' # non-ascii bytes
cmd_args = util.convert_command_args([arg])
self.assertEqual(cmd_args[0],
arg.decode(util.arg_encoding(), 'surrogateescape'))
def encode(self, command, source, dest, pretend=False):
"""Encode `source` to `dest` using command template `command`.
Raises `subprocess.CalledProcessError` if the command exited with a
non-zero status code.
"""
# The paths and arguments must be bytes.
assert isinstance(command, bytes)
assert isinstance(source, bytes)
assert isinstance(dest, bytes)
quiet = self.config['quiet'].get(bool)
if not quiet and not pretend:
self._log.info(u'Encoding {0}', util.displayable_path(source))
# On Python 3, we need to construct the command to invoke as a
# Unicode string. On Unix, this is a little unfortunate---the OS is
# expecting bytes---so we use surrogate escaping and decode with the
# argument encoding, which is the same encoding that will then be
# *reversed* to recover the same bytes before invoking the OS. On
# Windows, we want to preserve the Unicode filename "as is."
if not six.PY2:
command = command.decode(util.arg_encoding(), 'surrogateescape')
if platform.system() == 'Windows':
source = source.decode(util._fsencoding())
dest = dest.decode(util._fsencoding())
else:
source = source.decode(util.arg_encoding(), 'surrogateescape')
dest = dest.decode(util.arg_encoding(), 'surrogateescape')
# Substitute $source and $dest in the argument list.
args = shlex.split(command)
encode_cmd = []
for i, arg in enumerate(args):
args[i] = Template(arg).safe_substitute({
'source': source,
'dest': dest,
})
if six.PY2:
encode_cmd.append(args[i])
else:
encode_cmd.append(args[i].encode(util.arg_encoding()))
if pretend:
self._log.info(u'{0}', u' '.join(ui.decargs(args)))
return
try:
util.command_output(encode_cmd)
except subprocess.CalledProcessError as exc:
# Something went wrong (probably Ctrl+C), remove temporary files
self._log.info(u'Encoding {0} failed. Cleaning up...',
util.displayable_path(source))
self._log.debug(u'Command {0} exited with status {1}: {2}',
args,
exc.returncode,
exc.output)
util.remove(dest)
util.prune_dirs(os.path.dirname(dest))
raise
except OSError as exc:
raise ui.UserError(
u"convert: couldn't invoke '{0}': {1}".format(
u' '.join(ui.decargs(args)), exc
)
)
if not quiet and not pretend:
self._log.info(u'Finished encoding {0}',
util.displayable_path(source))
def test_format_option_unicode(self):
l = self.run_with_output(b'ls', b'-f',
u'caf\xe9'.encode(util.arg_encoding()))
self.assertEqual(l, u'caf\xe9\n')
def encode(self, command, source, dest, pretend=False):
"""Encode `source` to `dest` using command template `command`.
Raises `subprocess.CalledProcessError` if the command exited with a
non-zero status code.
"""
# The paths and arguments must be bytes.
assert isinstance(command, bytes)
assert isinstance(source, bytes)
assert isinstance(dest, bytes)
quiet = self.config['quiet'].get(bool)
if not quiet and not pretend:
self._log.info(u'Encoding {0}', util.displayable_path(source))
# Substitute $source and $dest in the argument list.
if not six.PY2:
command = command.decode(util.arg_encoding(), 'surrogateescape')
source = source.decode(util.arg_encoding(), 'surrogateescape')
dest = dest.decode(util.arg_encoding(), 'surrogateescape')
args = shlex.split(command)
encode_cmd = []
for i, arg in enumerate(args):
args[i] = Template(arg).safe_substitute({
'source': source,
'dest': dest,
})
if six.PY2:
encode_cmd.append(args[i])
else:
encode_cmd.append(args[i].encode(util.arg_encoding()))
if pretend:
self._log.info(u' '.join(ui.decargs(args)))
return
try:
util.command_output(encode_cmd)
except subprocess.CalledProcessError as exc:
# Something went wrong (probably Ctrl+C), remove temporary files
self._log.info(u'Encoding {0} failed. Cleaning up...',
util.displayable_path(source))
self._log.debug(u'Command {0} exited with status {1}: {2}',
args,
exc.returncode,
exc.output)
util.remove(dest)
util.prune_dirs(os.path.dirname(dest))
raise
except OSError as exc:
raise ui.UserError(
u"convert: couldn't invoke '{0}': {1}".format(
u' '.join(ui.decargs(args)), exc
)
)
if not quiet and not pretend:
self._log.info(u'Finished encoding {0}',
util.displayable_path(source))
def encode(self, command, source, dest, pretend=False):
"""Encode `source` to `dest` using command template `command`.
Raises `subprocess.CalledProcessError` if the command exited with a
non-zero status code.
"""
# The paths and arguments must be bytes.
assert isinstance(command, bytes)
assert isinstance(source, bytes)
assert isinstance(dest, bytes)
quiet = self.config['quiet'].get(bool)
if not quiet and not pretend:
self._log.info(u'Encoding {0}', util.displayable_path(source))
# On Python 3, we need to construct the command to invoke as a
# Unicode string. On Unix, this is a little unfortunate---the OS is
# expecting bytes---so we use surrogate escaping and decode with the
# argument encoding, which is the same encoding that will then be
# *reversed* to recover the same bytes before invoking the OS. On
# Windows, we want to preserve the Unicode filename "as is."
if not six.PY2:
command = command.decode(util.arg_encoding(), 'surrogateescape')
if platform.system() == 'Windows':
source = source.decode(util._fsencoding())
dest = dest.decode(util._fsencoding())
else:
source = source.decode(util.arg_encoding(), 'surrogateescape')
dest = dest.decode(util.arg_encoding(), 'surrogateescape')
# Substitute $source and $dest in the argument list.
args = shlex.split(command)
encode_cmd = []
for i, arg in enumerate(args):
args[i] = Template(arg).safe_substitute({
'source': source,
'dest': dest,
})
if six.PY2:
encode_cmd.append(args[i])
else:
encode_cmd.append(args[i].encode(util.arg_encoding()))
if pretend:
self._log.info(u'{0}', u' '.join(ui.decargs(args)))
return
try:
util.command_output(encode_cmd)
except subprocess.CalledProcessError as exc:
# Something went wrong (probably Ctrl+C), remove temporary files
self._log.info(u'Encoding {0} failed. Cleaning up...',
util.displayable_path(source))
self._log.debug(u'Command {0} exited with status {1}: {2}',
args,
exc.returncode,
exc.output)
util.remove(dest)
util.prune_dirs(os.path.dirname(dest))
raise
except OSError as exc:
raise ui.UserError(
u"convert: couldn't invoke '{0}': {1}".format(
u' '.join(ui.decargs(args)), exc
)
)
if not quiet and not pretend:
self._log.info(u'Finished encoding {0}',
util.displayable_path(source))
def test_format_option_unicode(self):
l = self.run_with_output(b'ls', b'-f',
u'caf\xe9'.encode(util.arg_encoding()))
self.assertEqual(l, u'caf\xe9\n')
