def run_svn_command(self, wire, cmd, **opts): path = wire.get('path', None) if path and os.path.isdir(path): opts['cwd'] = path safe_call = False if '_safe' in opts: safe_call = True svnenv = os.environ.copy() svnenv.update(opts.pop('extra_env', {})) _opts = {'env': svnenv, 'shell': False} try: _opts.update(opts) p = subprocessio.SubprocessIOChunker(cmd, **_opts) return ''.join(p), ''.join(p.error) except (EnvironmentError, OSError) as err: cmd = ' '.join(cmd) # human friendly CMD tb_err = ("Couldn't run svn command (%s).\n" "Original error was:%s\n" "Call options:%s\n" % (cmd, err, _opts)) log.exception(tb_err) if safe_call: return '', err else: raise exceptions.VcsException()(tb_err)
def test_output_with_no_input(size, environ): print type(environ) data = 'X' args = _get_python_args('sys.stdout.write("%s" * %d)' % (data, size)) output = ''.join(subprocessio.SubprocessIOChunker( args, shell=False, env=environ)) assert output == data * size
def test_output_with_no_input_does_not_fail(size, environ): data = 'X' args = _get_python_args( 'sys.stdout.write("%s" * %d); sys.exit(1)' % (data, size)) output = ''.join(subprocessio.SubprocessIOChunker( args, shell=False, fail_on_return_code=False, env=environ)) print len(data * size), len(output) assert output == data * size
def test_does_not_fail_on_stderr(environ): args = _get_python_args('sys.stderr.write("X"); time.sleep(1);') output = ''.join( subprocessio.SubprocessIOChunker(args, shell=False, fail_on_stderr=False, env=environ)) assert output == ''
def test_does_not_fail_on_non_zero_return_code(environ): args = _get_python_args('sys.exit(1)') output = ''.join( subprocessio.SubprocessIOChunker(args, shell=False, fail_on_return_code=False, env=environ)) assert output == ''
def test_output_with_input(size, environ): data = 'X' * size inputstream = io.BytesIO(data) # This acts like the cat command. args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)') output = ''.join(subprocessio.SubprocessIOChunker( args, shell=False, inputstream=inputstream, env=environ)) print len(data), len(output) assert output == data
def run_git_command(self, wire, cmd, **opts): path = wire.get('path', None) if path and os.path.isdir(path): opts['cwd'] = path if '_bare' in opts: _copts = [] del opts['_bare'] else: _copts = [ '-c', 'core.quotepath=false', ] safe_call = False if '_safe' in opts: # no exc on failure del opts['_safe'] safe_call = True if '_copts' in opts: _copts.extend(opts['_copts'] or []) del opts['_copts'] gitenv = os.environ.copy() gitenv.update(opts.pop('extra_env', {})) # need to clean fix GIT_DIR ! if 'GIT_DIR' in gitenv: del gitenv['GIT_DIR'] gitenv['GIT_CONFIG_NOGLOBAL'] = '1' gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1' cmd = [settings.GIT_EXECUTABLE] + _copts + cmd _opts = {'env': gitenv, 'shell': False} proc = None try: _opts.update(opts) proc = subprocessio.SubprocessIOChunker(cmd, **_opts) return ''.join(proc), ''.join(proc.error) except (EnvironmentError, OSError) as err: cmd = ' '.join(cmd) # human friendly CMD tb_err = ("Couldn't run git command (%s).\n" "Original error was:%s\n" "Call options:%s\n" % (cmd, err, _opts)) log.exception(tb_err) if safe_call: return '', err else: raise exceptions.VcsException()(tb_err) finally: if proc: proc.close()
def test_output_with_input_skipping_iterator(size, environ): data = 'X' * size inputstream = io.BytesIO(data) # This acts like the cat command. args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)') # Note: assigning the chunker makes sure that it is not deleted too early chunker = subprocessio.SubprocessIOChunker( args, shell=False, inputstream=inputstream, env=environ) output = ''.join(chunker.output) print len(data), len(output) assert output == data
def test_output_with_input(size, environ): data_len = size inputstream = KindaFilelike('X', size) # This acts like the cat command. args = _get_python_args('shutil.copyfileobj(sys.stdin, sys.stdout)') output = ''.join( subprocessio.SubprocessIOChunker(args, shell=False, inputstream=inputstream, env=environ)) assert len(output) == data_len
def inforefs(self, request, unused_environ): """ WSGI Response producer for HTTP GET Git Smart HTTP /info/refs request. """ git_command = request.GET.get('service') if git_command not in self.commands: log.debug('command %s not allowed', git_command) return exc.HTTPForbidden() # please, resist the urge to add '\n' to git capture and increment # line count by 1. # by git docs: Documentation/technical/http-protocol.txt#L214 \n is # a part of protocol. # The code in Git client not only does NOT need '\n', but actually # blows up if you sprinkle "flush" (0000) as "0001\n". # It reads binary, per number of bytes specified. # if you do add '\n' as part of data, count it. server_advert = '# service=%s\n' % git_command packet_len = str(hex(len(server_advert) + 4)[2:].rjust(4, '0')).lower() try: gitenv = dict(os.environ) # forget all configs gitenv['RC_SCM_DATA'] = json.dumps(self.extras) command = [ self.git_path, git_command[4:], '--stateless-rpc', '--advertise-refs', self.content_path ] out = subprocessio.SubprocessIOChunker( command, env=gitenv, starting_values=[packet_len + server_advert + '0000'], shell=False) except EnvironmentError: log.exception('Error processing command') raise exc.HTTPExpectationFailed() resp = Response() resp.content_type = 'application/x-%s-advertisement' % str(git_command) resp.charset = None resp.app_iter = out return resp
def backend(self, request, environ): """ WSGI Response producer for HTTP POST Git Smart HTTP requests. Reads commands and data from HTTP POST's body. returns an iterator obj with contents of git command's response to stdout """ # TODO(skreft): think how we could detect an HTTPLockedException, as # we probably want to have the same mechanism used by mercurial and # simplevcs. # For that we would need to parse the output of the command looking for # some signs of the HTTPLockedError, parse the data and reraise it in # pygrack. However, that would interfere with the streaming. # # Now the output of a blocked push is: # Pushing to http://test_regular:[email protected]:5001/vcs_test_git # POST git-receive-pack (1047 bytes) # remote: ERROR: Repository `vcs_test_git` locked by user `test_admin`. Reason:`lock_auto` # To http://test_regular:[email protected]:5001/vcs_test_git # ! [remote rejected] master -> master (pre-receive hook declined) # error: failed to push some refs to 'http://*****:*****@127.0.0.1:5001/vcs_test_git' git_command = self._get_fixedpath(request.path_info) if git_command not in self.commands: log.debug('command %s not allowed', git_command) return exc.HTTPForbidden() capabilities = None if git_command == 'git-upload-pack': capabilities = self._get_want_capabilities(request) if 'CONTENT_LENGTH' in environ: inputstream = FileWrapper(request.body_file_seekable, request.content_length) else: inputstream = request.body_file_seekable resp = Response() resp.content_type = ('application/x-%s-result' % git_command.encode('utf8')) resp.charset = None pre_pull_messages = '' if git_command == 'git-upload-pack': status, pre_pull_messages = hooks.git_pre_pull(self.extras) if status != 0: resp.app_iter = self._build_failed_pre_pull_response( capabilities, pre_pull_messages) return resp gitenv = dict(os.environ) # forget all configs gitenv['GIT_CONFIG_NOGLOBAL'] = '1' gitenv['RC_SCM_DATA'] = json.dumps(self.extras) cmd = [ self.git_path, git_command[4:], '--stateless-rpc', self.content_path ] log.debug('handling cmd %s', cmd) out = subprocessio.SubprocessIOChunker(cmd, inputstream=inputstream, env=gitenv, cwd=self.content_path, shell=False, fail_on_stderr=False, fail_on_return_code=False) if self.update_server_info and git_command == 'git-receive-pack': # We need to fully consume the iterator here, as the # update-server-info command needs to be run after the push. out = list(out) # Updating refs manually after each push. # This is required as some clients are exposing Git repos internally # with the dumb protocol. cmd = [self.git_path, 'update-server-info'] log.debug('handling cmd %s', cmd) output = subprocessio.SubprocessIOChunker( cmd, inputstream=inputstream, env=gitenv, cwd=self.content_path, shell=False, fail_on_stderr=False, fail_on_return_code=False) # Consume all the output so the subprocess finishes for _ in output: pass if git_command == 'git-upload-pack': unused_status, post_pull_messages = hooks.git_post_pull( self.extras) resp.app_iter = self._inject_messages_to_response( out, capabilities, pre_pull_messages, post_pull_messages) else: resp.app_iter = out return resp
def test_raise_exception_on_stderr(environ): args = _get_python_args('sys.stderr.write("X"); time.sleep(1);') with pytest.raises(EnvironmentError) as excinfo: list(subprocessio.SubprocessIOChunker(args, shell=False, env=environ)) assert 'exited due to an error:\nX' in str(excinfo.value)
def test_raise_exception_on_non_zero_return_code(environ): args = _get_python_args('sys.exit(1)') with pytest.raises(EnvironmentError): list(subprocessio.SubprocessIOChunker(args, shell=False, env=environ))