def run_svn_command(self, wire, cmd, **opts):
path = wire.get('path', None)
if path and os.path.isdir(path):
opts['cwd'] = path
safe_call = False
if '_safe' in opts:
safe_call = True
svnenv = os.environ.copy()
svnenv.update(opts.pop('extra_env', {}))
_opts = {'env': svnenv, 'shell': False}
try:
_opts.update(opts)
p = subprocessio.SubprocessIOChunker(cmd, **_opts)
return ''.join(p), ''.join(p.error)
except (EnvironmentError, OSError) as err:
cmd = ' '.join(cmd) # human friendly CMD
tb_err = ("Couldn't run svn command (%s).\n"
"Original error was:%s\n"
"Call options:%s\n" % (cmd, err, _opts))
log.exception(tb_err)
if safe_call:
return '', err
else:
raise exceptions.VcsException()(tb_err)
def test_output_with_no_input(size, environ):
print type(environ)
data = 'X'
args = _get_python_args('sys.stdout.write("%s" * %d)' % (data, size))
output = ''.join(subprocessio.SubprocessIOChunker(
args, shell=False, env=environ))
assert output == data * size
def test_output_with_no_input_does_not_fail(size, environ):
data = 'X'
args = _get_python_args(
'sys.stdout.write("%s" * %d); sys.exit(1)' % (data, size))
output = ''.join(subprocessio.SubprocessIOChunker(
args, shell=False, fail_on_return_code=False, env=environ))
print len(data * size), len(output)
assert output == data * size
def test_does_not_fail_on_stderr(environ):
args = _get_python_args('sys.stderr.write("X"); time.sleep(1);')
output = ''.join(
subprocessio.SubprocessIOChunker(args,
shell=False,
fail_on_stderr=False,
env=environ))
assert output == ''
def test_does_not_fail_on_non_zero_return_code(environ):
args = _get_python_args('sys.exit(1)')
output = ''.join(
subprocessio.SubprocessIOChunker(args,
shell=False,
fail_on_return_code=False,
env=environ))
assert output == ''
def test_output_with_input(size, environ):
data = 'X' * size
inputstream = io.BytesIO(data)
# This acts like the cat command.
args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
output = ''.join(subprocessio.SubprocessIOChunker(
args, shell=False, inputstream=inputstream, env=environ))
print len(data), len(output)
assert output == data
def run_git_command(self, wire, cmd, **opts):
path = wire.get('path', None)
if path and os.path.isdir(path):
opts['cwd'] = path
if '_bare' in opts:
_copts = []
del opts['_bare']
else:
_copts = [
'-c',
'core.quotepath=false',
]
safe_call = False
if '_safe' in opts:
# no exc on failure
del opts['_safe']
safe_call = True
if '_copts' in opts:
_copts.extend(opts['_copts'] or [])
del opts['_copts']
gitenv = os.environ.copy()
gitenv.update(opts.pop('extra_env', {}))
# need to clean fix GIT_DIR !
if 'GIT_DIR' in gitenv:
del gitenv['GIT_DIR']
gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1'
cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
_opts = {'env': gitenv, 'shell': False}
proc = None
try:
_opts.update(opts)
proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
return ''.join(proc), ''.join(proc.error)
except (EnvironmentError, OSError) as err:
cmd = ' '.join(cmd) # human friendly CMD
tb_err = ("Couldn't run git command (%s).\n"
"Original error was:%s\n"
"Call options:%s\n" % (cmd, err, _opts))
log.exception(tb_err)
if safe_call:
return '', err
else:
raise exceptions.VcsException()(tb_err)
finally:
if proc:
proc.close()
def test_output_with_input_skipping_iterator(size, environ):
data = 'X' * size
inputstream = io.BytesIO(data)
# This acts like the cat command.
args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
# Note: assigning the chunker makes sure that it is not deleted too early
chunker = subprocessio.SubprocessIOChunker(
args, shell=False, inputstream=inputstream, env=environ)
output = ''.join(chunker.output)
print len(data), len(output)
assert output == data
def test_output_with_input(size, environ):
data_len = size
inputstream = KindaFilelike('X', size)
# This acts like the cat command.
args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)')
output = ''.join(
subprocessio.SubprocessIOChunker(args,
shell=False,
inputstream=inputstream,
env=environ))
assert len(output) == data_len
def inforefs(self, request, unused_environ):
"""
WSGI Response producer for HTTP GET Git Smart
HTTP /info/refs request.
"""
git_command = request.GET.get('service')
if git_command not in self.commands:
log.debug('command %s not allowed', git_command)
return exc.HTTPForbidden()
# please, resist the urge to add '\n' to git capture and increment
# line count by 1.
# by git docs: Documentation/technical/http-protocol.txt#L214 \n is
# a part of protocol.
# The code in Git client not only does NOT need '\n', but actually
# blows up if you sprinkle "flush" (0000) as "0001\n".
# It reads binary, per number of bytes specified.
# if you do add '\n' as part of data, count it.
server_advert = '# service=%s\n' % git_command
packet_len = str(hex(len(server_advert) + 4)[2:].rjust(4, '0')).lower()
try:
gitenv = dict(os.environ)
# forget all configs
gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
command = [
self.git_path, git_command[4:], '--stateless-rpc',
'--advertise-refs', self.content_path
]
out = subprocessio.SubprocessIOChunker(
command,
env=gitenv,
starting_values=[packet_len + server_advert + '0000'],
shell=False)
except EnvironmentError:
log.exception('Error processing command')
raise exc.HTTPExpectationFailed()
resp = Response()
resp.content_type = 'application/x-%s-advertisement' % str(git_command)
resp.charset = None
resp.app_iter = out
return resp
def backend(self, request, environ):
"""
WSGI Response producer for HTTP POST Git Smart HTTP requests.
Reads commands and data from HTTP POST's body.
returns an iterator obj with contents of git command's
response to stdout
"""
# TODO(skreft): think how we could detect an HTTPLockedException, as
# we probably want to have the same mechanism used by mercurial and
# simplevcs.
# For that we would need to parse the output of the command looking for
# some signs of the HTTPLockedError, parse the data and reraise it in
# pygrack. However, that would interfere with the streaming.
#
# Now the output of a blocked push is:
# Pushing to http://test_regular:[email protected]:5001/vcs_test_git
# POST git-receive-pack (1047 bytes)
# remote: ERROR: Repository `vcs_test_git` locked by user `test_admin`. Reason:`lock_auto`
# To http://test_regular:[email protected]:5001/vcs_test_git
# ! [remote rejected] master -> master (pre-receive hook declined)
# error: failed to push some refs to 'http://*****:*****@127.0.0.1:5001/vcs_test_git'
git_command = self._get_fixedpath(request.path_info)
if git_command not in self.commands:
log.debug('command %s not allowed', git_command)
return exc.HTTPForbidden()
capabilities = None
if git_command == 'git-upload-pack':
capabilities = self._get_want_capabilities(request)
if 'CONTENT_LENGTH' in environ:
inputstream = FileWrapper(request.body_file_seekable,
request.content_length)
else:
inputstream = request.body_file_seekable
resp = Response()
resp.content_type = ('application/x-%s-result' %
git_command.encode('utf8'))
resp.charset = None
pre_pull_messages = ''
if git_command == 'git-upload-pack':
status, pre_pull_messages = hooks.git_pre_pull(self.extras)
if status != 0:
resp.app_iter = self._build_failed_pre_pull_response(
capabilities, pre_pull_messages)
return resp
gitenv = dict(os.environ)
# forget all configs
gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
cmd = [
self.git_path, git_command[4:], '--stateless-rpc',
self.content_path
]
log.debug('handling cmd %s', cmd)
out = subprocessio.SubprocessIOChunker(cmd,
inputstream=inputstream,
env=gitenv,
cwd=self.content_path,
shell=False,
fail_on_stderr=False,
fail_on_return_code=False)
if self.update_server_info and git_command == 'git-receive-pack':
# We need to fully consume the iterator here, as the
# update-server-info command needs to be run after the push.
out = list(out)
# Updating refs manually after each push.
# This is required as some clients are exposing Git repos internally
# with the dumb protocol.
cmd = [self.git_path, 'update-server-info']
log.debug('handling cmd %s', cmd)
output = subprocessio.SubprocessIOChunker(
cmd,
inputstream=inputstream,
env=gitenv,
cwd=self.content_path,
shell=False,
fail_on_stderr=False,
fail_on_return_code=False)
# Consume all the output so the subprocess finishes
for _ in output:
pass
if git_command == 'git-upload-pack':
unused_status, post_pull_messages = hooks.git_post_pull(
self.extras)
resp.app_iter = self._inject_messages_to_response(
out, capabilities, pre_pull_messages, post_pull_messages)
else:
resp.app_iter = out
return resp
def test_raise_exception_on_stderr(environ):
args = _get_python_args('sys.stderr.write("X"); time.sleep(1);')
with pytest.raises(EnvironmentError) as excinfo:
list(subprocessio.SubprocessIOChunker(args, shell=False, env=environ))
assert 'exited due to an error:\nX' in str(excinfo.value)
def test_raise_exception_on_non_zero_return_code(environ):
args = _get_python_args('sys.exit(1)')
with pytest.raises(EnvironmentError):
list(subprocessio.SubprocessIOChunker(args, shell=False, env=environ))