def add_file(self, name, filename, content, mimetype=None):
"""Add an uploaded file for the request.
Args:
name (bytes or unicode):
The name of the field representing the file.
filename (bytes or unicode):
The filename.
content (bytes or unicode):
The contents of the file.
mimetype (bytes or unicode, optional):
The optional mimetype of the content. If not provided, it
will be guessed.
"""
mimetype = force_bytes(
mimetypes.guess_type(force_unicode(filename))[0]
or b'application/octet-stream')
self._files[force_bytes(name)] = {
'filename': force_bytes(filename),
'content': force_bytes(content),
'mimetype': mimetype,
}
def encode_url_value(self, key, value):
"""Encode the given value for inclusion in a URL.
Args:
key (unicode):
The field name for which the value is being encoded.
This argument is only used to generate an error message.
value (object):
The value to be encoded.
Raises:
ValueError:
The given value could not be encoded.
Returns:
unicode:
The value encoded as a unicode string.
"""
if isinstance(value, bool):
if value:
value = '1'
else:
value = '0'
elif isinstance(value, six.integer_types + (float, )):
value = six.text_type(value)
elif isinstance(value, (bytes, six.text_type)):
value = force_unicode(value)
else:
raise ValueError(
'Could not encode value %r for key %s: expected int, float, '
'bool, or string type; got %s instead' %
(key, value, type(value).__name__))
return value
def __str__(self):
"""Return the error message as a unicode string.
Returns:
unicode:
The error message as a unicode string.
"""
return force_unicode(self.msg)
def _hg_get_tip(self):
"""Return the revision at the tip of the branch.
Returns:
unicode:
The tip revision.
"""
return force_unicode(self.run_hg(['identify']).split()[0])
def __init__(self, url, method='GET', query_args={}, headers={}):
"""Initialize the HTTP request.
Args:
url (bytes or unicode):
The URL to request.
method (bytes or unicode, optional):
The HTTP method to send to the server.
query_args (dict, optional):
Any query arguments to add to the URL.
All keys and values are expected to be strings (either
byte strings or unicode strings).
headers (dict, optional):
Any HTTP headers to provide in the request.
All keys and values are expected to be strings (either
byte strings or unicode strings).
"""
self.method = method
self._fields = OrderedDict()
self._files = OrderedDict()
# Replace all underscores in each query argument
# key with dashes.
query_args = {
force_unicode(key).replace('_', '-'): force_unicode(value)
for key, value in six.iteritems(query_args)
}
# Make sure headers are always in the native string type.
self.headers = {
str(key): str(value)
for key, value in six.iteritems(headers)
}
# Add the query arguments to the url
url_parts = list(urlparse(str(url)))
query = dict(parse_qsl(url_parts[4]))
query.update(query_args)
url_parts[4] = urlencode(query)
self.url = urlunparse(url_parts)
def edit_file(filename):
"""Run a user-configured editor to edit an existing file.
This will run a configured text editor (trying the :envvar:`VISUAL` or
:envvar:`EDITOR` environment variables, falling back on :program:`vi`)
to request text for use in a commit message or some other purpose.
Args:
filename (unicode):
The file to edit.
Returns:
unicode:
The resulting content.
Raises:
rbcommons.utils.errors.EditorError:
The configured editor could not be run, or it failed with an
error.
"""
if not os.path.exists(filename):
raise EditorError('The file "%s" does not exist or is not accessible.'
% filename)
editor = force_unicode(
os.environ.get(str('RBTOOLS_EDITOR')) or
os.environ.get(str('VISUAL')) or
os.environ.get(str('EDITOR')) or
'vi'
)
try:
subprocess.call(editor.split() + [filename])
except OSError:
raise EditorError('The editor "%s" was not found or could not be run. '
'Make sure the EDITOR environment variable is set '
'to your preferred editor.'
% editor)
try:
with open(filename, 'r') as fp:
return force_unicode(fp.read())
except IOError:
raise EditorError('The edited file "%s" was deleted during edit.'
% filename)
def log_command_line(fmt, command):
"""Log a command line.
Args:
fmt (unicode):
A format string to use for the log message.
command (list):
A command line in list form.
"""
# While most of the subprocess library can deal with bytes objects in
# command lines, list2cmdline can't. Decode each part if necessary.
logging.debug(
fmt,
subprocess.list2cmdline([force_unicode(part) for part in command]))
def encode_url_key(self, key):
"""Encode the given key for inclusion in a URL.
Args:
key (unicode):
The key that is being encoded.
Raises:
ValueError:
The given key was neither a unicode string or byte string.
Returns:
unicode:
The key encoded as a unicode string.
"""
return force_unicode(key).replace('_', '-')
def process_error(self, http_status, data):
"""Processes an error, raising an APIError with the information."""
# In Python 3, the data can be bytes, not str, and json.loads
# explicitly requires decoded strings.
data = force_unicode(data)
try:
rsp = json_loads(data)
assert rsp['stat'] == 'fail'
logging.debug('Got API Error %d (HTTP code %d): %s',
rsp['err']['code'], http_status, rsp['err']['msg'])
logging.debug('Error data: %r', rsp)
raise create_api_error(http_status, rsp['err']['code'], rsp,
rsp['err']['msg'])
except ValueError:
logging.debug('Got HTTP error: %s: %s', http_status, data)
raise APIError(http_status, None, None, data)
def process_error(self, http_status, data):
"""Processes an error, raising an APIError with the information."""
# In Python 3, the data can be bytes, not str, and json.loads
# explicitly requires decoded strings.
data = force_unicode(data)
try:
rsp = json_loads(data)
assert rsp['stat'] == 'fail'
logging.debug('Got API Error %d (HTTP code %d): %s',
rsp['err']['code'], http_status, rsp['err']['msg'])
logging.debug('Error data: %r', rsp)
raise create_api_error(http_status, rsp['err']['code'], rsp,
rsp['err']['msg'])
except ValueError:
logging.debug('Got HTTP error: %s: %s', http_status, data)
raise APIError(http_status, None, None, data)
def JsonDecoder(payload):
# In Python 3, the payload can be bytes, not str, and json.loads explicitly
# requires decoded strings.
return json.loads(force_unicode(payload))
def execute(command,
env=None,
cwd=None,
split_lines=False,
ignore_errors=False,
extra_ignore_errors=(),
with_errors=True,
none_on_ignored_error=False,
return_error_code=False,
log_output_on_error=True,
results_unicode=True,
results_encoding=None,
return_errors=False):
"""Execute a command and return the output.
Args:
command (unicode or list of unicode):
The command to execute.
env (dict, optional):
Environment variables to pass to the called executable. These will
be added to the current environment.
cwd (unicode, optional):
An optional working directory to change to before executing the
process.
split_lines (bool, optional):
Whether to return the output as a list of lines or a single string.
ignore_errors (bool, optional):
Whether to ignore errors. If ``False``, this will raise an
exception.
extra_ignore_errors (tuple, optional):
A set of errors to ignore even when ``ignore_errors`` is False.
This is used because some commands (such as diff) use non-zero
return codes even when the command was successful.
with_errors (bool, optional):
Whether to combine the output and error streams of the command
together into a single return value. This argument is mutually
exclusive with the ``return_errors`` argument.
none_on_ignored_error (bool, optional):
Whether to return ``None`` in the case that an error was ignored
(instead of the output of the command).
return_error_code (bool, optional):
Whether to include the exit status of the executed command in
addition to the output
log_output_on_error (bool, optional):
If ``True``, the output from the command will be logged in the case
that the command returned a non-zero exit code.
results_unicode (bool, optional):
If ``True``, the output will be treated as text and returned as
unicode strings instead of bytes.
results_encoding (unicode, optional):
Optional encoding setting for command results.
return_errors (bool, optional):
Whether to return the content of the stderr stream. This argument
is mutually exclusive with the ``with_errors`` argument.
Returns:
This returns a single value, 2-tuple, or 3-tuple depending on the
arguments.
If ``return_error_code`` is True, the error code of the process will be
returned as the first element of the tuple.
If ``return_errors`` is True, the process' standard error stream will
be returned as the last element of the tuple.
If both of ``return_error_code`` and ``return_errors`` are ``False``,
then the process' output will be returned. If either or both of them
are ``True``, then this is the other element of the returned tuple.
"""
assert not (with_errors and return_errors)
if isinstance(command, list):
log_command_line('Running: %s', command)
else:
logging.debug('Running: %s', command)
new_env = os.environ.copy()
if env:
new_env.update(env)
# TODO: This can break on systems that don't have the en_US locale
# installed (which isn't very many). Ideally in this case, we could
# put something in the config file, but that's not plumbed through to here.
new_env['LC_ALL'] = 'en_US.UTF-8'
new_env['LANGUAGE'] = 'en_US.UTF-8'
if with_errors:
errors_output = subprocess.STDOUT
else:
errors_output = subprocess.PIPE
popen_encoding_args = {}
if results_unicode:
# Popen before Python 3.6 doesn't support the ``encoding`` parameter,
# so we have to use ``universal_newlines`` and then decode later.
if six.PY3 and sys.version_info.minor >= 6:
if results_encoding is None:
popen_encoding_args['encoding'] = 'utf-8'
else:
popen_encoding_args['encoding'] = results_encoding
else:
popen_encoding_args['universal_newlines'] = True
if sys.platform.startswith('win'):
# Convert all environment variables to the native string type, so that
# subprocess doesn't blow up on Windows.
new_env = dict(
(str(key), str(value)) for key, value in six.iteritems(new_env))
p = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=errors_output,
shell=False,
env=new_env,
cwd=cwd,
**popen_encoding_args)
else:
p = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=errors_output,
shell=False,
close_fds=True,
env=new_env,
cwd=cwd,
**popen_encoding_args)
data, errors = p.communicate()
# We did not specify `encoding` to Popen earlier, so we must decode now.
if results_unicode and 'encoding' not in popen_encoding_args:
if results_encoding is None:
data = force_unicode(data)
else:
data = force_unicode(data, encoding=results_encoding)
if split_lines:
data = data.splitlines(True)
if return_errors:
if split_lines:
errors = errors.splitlines(True)
else:
errors = None
rc = p.wait()
if rc and not ignore_errors and rc not in extra_ignore_errors:
if log_output_on_error:
logging.debug('Command exited with rc %s: %s\n%s---', rc, command,
data)
raise Exception('Failed to execute command: %s' % command)
elif rc:
if log_output_on_error:
logging.debug('Command exited with rc %s: %s\n%s---', rc, command,
data)
else:
logging.debug('Command exited with rc %s: %s', rc, command)
if rc and none_on_ignored_error:
data = None
if return_error_code and return_errors:
return rc, data, errors
elif return_error_code:
return rc, data
elif return_errors:
return data, errors
else:
return data
def JsonDecoder(payload):
# In Python 3, the payload can be bytes, not str, and json.loads explicitly
# requires decoded strings.
return json.loads(force_unicode(payload))
