def test_setup_python_multiple_transitive_markers(): py27_interpreter = ensure_python_interpreter(PY27) py36_interpreter = ensure_python_interpreter(PY36) with temporary_dir() as out: pex = os.path.join(out, 'pex.pex') results = run_pex_command([ 'jsonschema==2.6.0', '--disable-cache', '--python-shebang=#!/usr/bin/env python', '--python={}'.format(py27_interpreter), '--python={}'.format(py36_interpreter), '-o', pex ]) results.assert_success() pex_program = [pex, '-c'] py2_only_program = pex_program + ['import functools32'] both_program = pex_program + [ 'import jsonschema, os, sys; print(os.path.realpath(sys.executable))' ] with environment_as(PATH=os.path.dirname(py27_interpreter)): subprocess.check_call(py2_only_program) stdout = subprocess.check_output(both_program) assert to_bytes( os.path.realpath(py27_interpreter)) == stdout.strip() with environment_as(PATH=os.path.dirname(py36_interpreter)): with pytest.raises(subprocess.CalledProcessError): subprocess.check_call(py2_only_program) stdout = subprocess.check_output(both_program) assert to_bytes( os.path.realpath(py36_interpreter)) == stdout.strip()
def test_environment_negation(): with temporary_file() as output: with environment_as(HORK = 'BORK'): with environment_as(HORK = None): # test that the variable is cleared subprocess.Popen([sys.executable, '-c', 'import os; print os.environ.has_key("HORK")'], stdout=output).wait() output.seek(0) assert output.read() == 'False\n'
def test_environment_negation(): with temporary_file() as output: with environment_as(HORK='BORK'): with environment_as(HORK=None): # test that the variable is cleared subprocess.Popen([ sys.executable, '-c', 'import os; print("HORK" in os.environ)' ], stdout=output).wait() output.seek(0) assert output.read() == 'False\n'
def _maybe_scrubbed_env(cls): for env_var in cls._SCRUBBED_ENV: value = os.getenv(env_var) if value: log.warn('Scrubbing {env_var}={value}'.format(env_var=env_var, value=value)) with environment_as(**cls._SCRUBBED_ENV): yield
def _bootstrap_pex(self, pex_path: pathlib.PosixPath): """Bootstraps a pex with widget UI display.""" title = f'[Bootstrap] {pex_path.name}' with self._accordion_widget(title) as (expand, collapse, set_output_glyph): try: with environment_as(PEX_VERBOSE='2'): # Scrub the environment. _scrub_import_environment( self._ORIGINATING_SYS_MODULES_KEYS, self._display_line) # Bootstrap pex. bootstrap_pex_env(pex_path) except Exception: try: set_output_glyph(FAIL_GLYPH) expand() finally: raise else: self._display_line( f'Successfully bootstrapped pex environment from {pex_path}\n' ) set_output_glyph(SUCCESS_GLYPH) collapse()
def build_egg(self, egg_root, target): """Build an egg containing the files at egg_root for the specified target. There must be an egg_root/setup.py file.""" # TODO(Brian Wickman): Do a sanity check somewhere to ensure that # setuptools is on the path? args = [ sys.executable, 'setup.py', 'bdist_egg', '--dist-dir=dist', '--bdist-dir=build.%s' % target.name ] with pushd(egg_root): with environment_as(PYTHONPATH=':'.join(sys.path)): po = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE) rv = po.wait() eggs = os.path.abspath(os.path.join('dist', '*.egg')) eggs = glob.glob(eggs) if rv != 0 or len(eggs) != 1: comm = po.communicate() print('egg generation failed (return value=%d, num eggs=%d)' % (rv, len(eggs)), file=sys.stderr) print('STDOUT', file=sys.stderr) print(comm[0], file=sys.stderr) print('STDERR', file=sys.stderr) print(comm[1], file=sys.stderr) raise EggBuilder.EggBuildingException( 'Generation of eggs failed for target = %s' % target) egg_path = eggs[0] return egg_path
def setUpClass(cls): cls.origin = safe_mkdtemp() with pushd(cls.origin): subprocess.check_call(['git', 'init', '--bare']) cls.gitdir = safe_mkdtemp() cls.worktree = safe_mkdtemp() cls.readme_file = os.path.join(cls.worktree, 'README') with environment_as(GIT_DIR=cls.gitdir, GIT_WORK_TREE=cls.worktree): cls.init_repo('depot', cls.origin) touch(cls.readme_file) subprocess.check_call(['git', 'add', 'README']) subprocess.check_call(['git', 'commit', '-am', 'initial commit with decode -> \x81b']) subprocess.check_call(['git', 'tag', 'first']) subprocess.check_call(['git', 'push', '--tags', 'depot', 'master']) subprocess.check_call(['git', 'branch', '--set-upstream', 'master', 'depot/master']) with safe_open(cls.readme_file, 'w') as readme: readme.write('Hello World.') subprocess.check_call(['git', 'commit', '-am', 'Update README.']) cls.clone2 = safe_mkdtemp() with pushd(cls.clone2): cls.init_repo('origin', cls.origin) subprocess.check_call(['git', 'pull', '--tags', 'origin', 'master:master']) with safe_open(os.path.realpath('README'), 'a') as readme: readme.write('--') subprocess.check_call(['git', 'commit', '-am', 'Update README 2.']) subprocess.check_call(['git', 'push', '--tags', 'origin', 'master']) cls.git = Git(gitdir=cls.gitdir, worktree=cls.worktree)
def build_egg(self, egg_root, target): """Build an egg containing the files at egg_root for the specified target. There must be an egg_root/setup.py file.""" # TODO(Brian Wickman): Do a sanity check somewhere to ensure that # setuptools is on the path? args = [ sys.executable, 'setup.py', 'bdist_egg', '--dist-dir=dist', '--bdist-dir=build.%s' % target.name] with pushd(egg_root): print 'EggBuilder executing: %s' % ' '.join(args) with environment_as(PYTHONPATH = ':'.join(sys.path)): po = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE) rv = po.wait() eggs = os.path.abspath(os.path.join('dist', '*.egg')) eggs = glob.glob(eggs) if rv != 0 or len(eggs) != 1: comm = po.communicate() print >> sys.stderr, 'egg generation failed (return value=%d, num eggs=%d)' % ( rv, len(eggs)) print >> sys.stderr, 'STDOUT' print >> sys.stderr, comm[0] print >> sys.stderr, 'STDERR' print >> sys.stderr, comm[1] raise EggBuilder.EggBuildingException( 'Generation of eggs failed for target = %s' % target) egg_path = eggs[0] return egg_path
def test(self): self.assertEqual(set(), self.git.changed_files()) self.assertEqual(set(['README']), self.git.changed_files(from_commit='HEAD^')) tip_sha = self.git.commit_id self.assertTrue(tip_sha) self.assertTrue(tip_sha in self.git.changelog()) self.assertTrue(self.git.tag_name.startswith('first-')) self.assertEqual('master', self.git.branch_name) def edit_readme(): with open(self.readme_file, 'a') as readme: readme.write('More data.') edit_readme() with open(os.path.join(self.worktree, 'INSTALL'), 'w') as untracked: untracked.write('make install') self.assertEqual(set(['README']), self.git.changed_files()) self.assertEqual(set(['README', 'INSTALL']), self.git.changed_files(include_untracked=True)) try: # These changes should be rejected because our branch point from origin is 1 commit behind # the changes pushed there in clone 2. self.git.commit('API Changes.') except Scm.RemoteException: with environment_as(GIT_DIR=self.gitdir, GIT_WORK_TREE=self.worktree): subprocess.check_call( ['git', 'reset', '--hard', 'depot/master']) self.git.refresh() edit_readme() self.git.commit('''API '"' " Changes.''') self.git.tag('second', message='''Tagged ' " Changes''') with temporary_dir() as clone: with pushd(clone): self.init_repo('origin', self.origin) subprocess.check_call( ['git', 'pull', '--tags', 'origin', 'master:master']) with open(os.path.realpath('README')) as readme: self.assertEqual('--More data.', readme.read()) git = Git() # Check that we can pick up committed and uncommitted changes. with safe_open(os.path.realpath('CHANGES'), 'w') as changes: changes.write('none') subprocess.check_call(['git', 'add', 'CHANGES']) self.assertEqual(set(['README', 'CHANGES']), git.changed_files(from_commit='first')) self.assertEqual('master', git.branch_name) self.assertEqual('second', git.tag_name)
def run(self): self._run_count += 1 atexit.register(self.cleanup) if self.script_filename: os.unlink(self.script_filename) with temporary_file(cleanup=False) as fp: self.script_filename = fp.name fp.write( self.RUN_JOB_SCRIPT % { 'filename': self.job_filename, 'sandbox': self.sandbox, 'root': self.tempdir, 'task_id': self.task_id, 'state_filename': self.state_filename, 'portmap': repr(self.portmap), 'success_rate': self.success_rate, 'random_seed': self.random_seed + self._run_count, }) with environment_as(PYTHONPATH=os.pathsep.join(sys.path)): self.po = subprocess.Popen([sys.executable, self.script_filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: so, se = self.po.communicate() except OSError as e: if e.errno == errno.ECHILD: so = se = 'Killed' else: raise rc = self.po.returncode if rc != 0: if os.path.exists(self.job_filename): config = open(self.job_filename).read() else: config = 'Nonexistent!' if 'THERMOS_DEBUG' in os.environ: print( "Runner failed!\n\n\nconfig:%s\n\n\nstdout:%s\n\n\nstderr:%s\n\n\n" % (config, so, se)) try: with open(self.state_filename, 'r') as fp: self.state = thrift_deserialize(RunnerState(), fp.read()) except Exception as e: if 'THERMOS_DEBUG' in os.environ: print('Failed to load Runner state: %s' % e, file=sys.stderr) self.state = RunnerState() try: self.reconstructed_state = CheckpointDispatcher.from_file( self.pathspec.getpath('runner_checkpoint')) except: self.reconstructed_state = None self.initialized = True return rc
def run(self): self._run_count += 1 atexit.register(self.cleanup) if self.script_filename: os.unlink(self.script_filename) with temporary_file(cleanup=False) as fp: self.script_filename = fp.name fp.write(self.RUN_JOB_SCRIPT % { 'filename': self.job_filename, 'sandbox': self.sandbox, 'root': self.tempdir, 'task_id': self.task_id, 'state_filename': self.state_filename, 'success_rate': self.success_rate, 'random_seed': self.random_seed + self._run_count, 'extra_task_runner_args': self.extra_task_runner_args, }) with environment_as(PYTHONPATH=os.pathsep.join(sys.path)): self.po = subprocess.Popen([sys.executable, self.script_filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: so, se = self.po.communicate() except OSError as e: if e.errno == errno.ECHILD: so = se = 'Killed' else: raise rc = self.po.returncode if rc != 0: if os.path.exists(self.job_filename): with open(self.job_filename) as fp: config = fp.read() else: config = 'Nonexistent!' if 'THERMOS_DEBUG' in os.environ: print("Runner failed!\n\n\nconfig:%s\n\n\nstdout:%s\n\n\nstderr:%s\n\n\n" % ( config, so, se)) try: with open(self.state_filename, 'r') as fp: self.state = thrift_deserialize(RunnerState(), fp.read()) except Exception as e: if 'THERMOS_DEBUG' in os.environ: print('Failed to load Runner state: %s' % e, file=sys.stderr) self.state = RunnerState() try: self.reconstructed_state = CheckpointDispatcher.from_file( self.pathspec.getpath('runner_checkpoint')) except Exception as e: print('Failed to replay checkpoint: %s' % e, file=sys.stderr) self.reconstructed_state = None self.initialized = True return rc
def test_pex_python(): py2_path_interpreter = ensure_python_interpreter('2.7.10') py3_path_interpreter = ensure_python_interpreter('3.6.3') path = ':'.join([os.path.dirname(py2_path_interpreter), os.path.dirname(py3_path_interpreter)]) with environment_as(PATH=path): with temporary_dir() as td: pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: pex_python = ensure_python_interpreter('3.6.3') pexrc.write("PEX_PYTHON=%s" % pex_python) # test PEX_PYTHON with valid constraints pex_out_path = os.path.join(td, 'pex.pex') res = run_pex_command(['--disable-cache', '--rcfile=%s' % pexrc_path, '--interpreter-constraint=>3', '--interpreter-constraint=<3.8', '-o', pex_out_path]) res.assert_success() stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) assert rc == 0 correct_interpreter_path = pex_python.encode() assert correct_interpreter_path in stdout # test PEX_PYTHON with incompatible constraints pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: pex_python = ensure_python_interpreter('2.7.10') pexrc.write("PEX_PYTHON=%s" % pex_python) pex_out_path = os.path.join(td, 'pex2.pex') res = run_pex_command(['--disable-cache', '--rcfile=%s' % pexrc_path, '--interpreter-constraint=>3', '--interpreter-constraint=<3.8', '-o', pex_out_path]) res.assert_success() stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) assert rc == 1 fail_str = 'not compatible with specified interpreter constraints'.encode() assert fail_str in stdout # test PEX_PYTHON with no constraints pex_out_path = os.path.join(td, 'pex3.pex') res = run_pex_command(['--disable-cache', '--rcfile=%s' % pexrc_path, '-o', pex_out_path]) res.assert_success() stdin_payload = b'import sys; print(sys.executable); sys.exit(0)' stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload) assert rc == 0 correct_interpreter_path = pex_python.encode() assert correct_interpreter_path in stdout
def _maybe_scrubbed_classpath(self): if self._scrub_classpath: classpath = os.getenv('CLASSPATH') if classpath: log.warn('Scrubbing CLASSPATH=%s' % classpath) with environment_as(CLASSPATH=None): yield else: yield
def config(self, overrides=''): """Returns a config valid for the test build root.""" if overrides: with temporary_file() as fp: fp.write(overrides) fp.close() with environment_as(PANTS_CONFIG_OVERRIDE=fp.name): return Config.load() else: return Config.load()
def test_setup_interpreter_constraint(): interpreter = ensure_python_interpreter(PY27) with temporary_dir() as out: pex = os.path.join(out, 'pex.pex') with environment_as(PATH=os.path.dirname(interpreter)): results = run_pex_command([ 'jsonschema==2.6.0', '--disable-cache', '--interpreter-constraint=CPython=={}'.format(PY27), '-o', pex ]) results.assert_success() subprocess.check_call([pex, '-c', 'import jsonschema'])
def test(self): self.assertEqual(set(), self.git.changed_files()) self.assertEqual(set(['README']), self.git.changed_files(from_commit='HEAD^')) tip_sha = self.git.commit_id self.assertTrue(tip_sha) self.assertTrue(tip_sha in self.git.changelog()) self.assertTrue(self.git.tag_name.startswith('first-')) self.assertEqual('master', self.git.branch_name) def edit_readme(): with open(self.readme_file, 'a') as readme: readme.write('More data.') edit_readme() with open(os.path.join(self.worktree, 'INSTALL'), 'w') as untracked: untracked.write('make install') self.assertEqual(set(['README']), self.git.changed_files()) self.assertEqual(set(['README', 'INSTALL']), self.git.changed_files(include_untracked=True)) try: # These changes should be rejected because our branch point from origin is 1 commit behind # the changes pushed there in clone 2. self.git.commit('API Changes.') except Scm.RemoteException: with environment_as(GIT_DIR=self.gitdir, GIT_WORK_TREE=self.worktree): subprocess.check_call(['git', 'reset', '--hard', 'depot/master']) self.git.refresh() edit_readme() self.git.commit('''API '"' " Changes.''') self.git.tag('second', message='''Tagged ' " Changes''') with temporary_dir() as clone: with pushd(clone): subprocess.check_call(['git', 'init']) subprocess.check_call(['git', 'remote', 'add', 'origin', self.origin]) subprocess.check_call(['git', 'pull', '--tags', 'origin', 'master:master']) with open(os.path.realpath('README')) as readme: self.assertEqual('--More data.', readme.read()) git = Git() # Check that we can pick up committed and uncommitted changes. with safe_open(os.path.realpath('CHANGES'), 'w') as changes: changes.write('none') subprocess.check_call(['git', 'add', 'CHANGES']) self.assertEqual(set(['README', 'CHANGES']), git.changed_files(from_commit='first')) self.assertEqual('master', git.branch_name) self.assertEqual('second', git.tag_name)
def safe_classpath(logger=None): """ Yields to a block in an environment with no CLASSPATH. This is useful to ensure hermetic java invocations. """ classpath = os.getenv('CLASSPATH') if classpath: logger = logger or log.warn logger('Scrubbing CLASSPATH=%s' % classpath) with environment_as(CLASSPATH=None): yield
def do_test_jre_env_var(self, env_var, env_value, scrubbed=True): with self.jre(env_var=env_var) as jre: executor = SubprocessExecutor(Distribution(bin_path=jre)) with environment_as(**{env_var: env_value}): self.assertEqual(env_value, os.getenv(env_var)) process = executor.spawn(classpath=['dummy/classpath'], main='dummy.main', stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, _ = process.communicate() self.assertEqual(0, process.returncode) self.assertEqual('' if scrubbed else env_value, stdout.strip())
def test_pex_python_symlink(): with temporary_dir() as td: with environment_as(HOME=td): symlink_path = os.path.join(td, 'python-symlink') os.symlink(sys.executable, symlink_path) pexrc_path = os.path.join(td, '.pexrc') with open(pexrc_path, 'w') as pexrc: pexrc.write("PEX_PYTHON=%s" % symlink_path) body = "print('Hello')" _, rc = run_simple_pex_test(body, coverage=True) assert rc == 0
def test_interpreter_constraints_to_pex_info_py3(): py3_interpreter = ensure_python_interpreter('3.6.3') with environment_as(PATH=os.path.dirname(py3_interpreter)): with temporary_dir() as output_dir: # target python 3 pex_out_path = os.path.join(output_dir, 'pex_py3.pex') res = run_pex_command(['--disable-cache', '--interpreter-constraint=>3', '-o', pex_out_path]) res.assert_success() pex_info = get_pex_info(pex_out_path) assert ['>3'] == pex_info.interpreter_constraints
def test_setup_python_multiple_direct_markers(): py36_interpreter = ensure_python_interpreter(PY36) py27_interpreter = ensure_python_interpreter(PY27) with temporary_dir() as out: pex = os.path.join(out, 'pex.pex') results = run_pex_command([ 'subprocess32==3.2.7; python_version<"3"', '--disable-cache', '--python-shebang=#!/usr/bin/env python', '--python={}'.format(py36_interpreter), '--python={}'.format(py27_interpreter), '-o', pex ]) results.assert_success() py2_only_program = [pex, '-c', 'import subprocess32'] with environment_as(PATH=os.path.dirname(py36_interpreter)): with pytest.raises(subprocess.CalledProcessError): subprocess.check_call(py2_only_program) with environment_as(PATH=os.path.dirname(py27_interpreter)): subprocess.check_call(py2_only_program)
def _call(self, cmd, *args, **kwargs): """Runs hadoop fs command with the given command and args. Checks the result of the call by default but this can be disabled with check=False. """ cmd = ['hadoop', '--config', self._config, 'dfs', cmd] + list(args) heapsize = str(int(self._heap_limit.as_(Data.MB))) with environment_as(HADOOP_HEAPSIZE=heapsize): if kwargs.get('check'): return self._cmd_class.check_call(cmd) elif kwargs.get('return_output'): return self._cmd_class.execute_and_get_output(cmd) elif kwargs.get('supress_output'): return self._cmd_class.execute_suppress_stdout(cmd) else: return self._cmd_class.execute(cmd)
def test_pex_root(): with temporary_dir() as tmp_home: with environment_as(HOME=tmp_home): with temporary_dir() as td: with temporary_dir() as output_dir: env = os.environ.copy() env['PEX_INTERPRETER'] = '1' output_path = os.path.join(output_dir, 'pex.pex') args = ['pex', '-o', output_path, '--not-zip-safe', '--pex-root={0}'.format(td)] results = run_pex_command(args=args, env=env) results.assert_success() assert ['pex.pex'] == os.listdir(output_dir), 'Expected built pex file.' assert [] == os.listdir(tmp_home), 'Expected empty temp home dir.' assert 'build' in os.listdir(td), 'Expected build directory in tmp pex root.'
def test_override_single_variable(): with temporary_file() as output: # test that the override takes place with environment_as(HORK = 'BORK'): subprocess.Popen([sys.executable, '-c', 'import os; print os.environ["HORK"]'], stdout=output).wait() output.seek(0) assert output.read() == 'BORK\n' # test that the variable is cleared with temporary_file() as new_output: subprocess.Popen([sys.executable, '-c', 'import os; print os.environ.has_key("HORK")'], stdout=new_output).wait() new_output.seek(0) assert new_output.read() == 'False\n'
def test_override_single_variable(): with temporary_file() as output: # test that the override takes place with environment_as(HORK='BORK'): subprocess.Popen( [sys.executable, '-c', 'import os; print(os.environ["HORK"])'], stdout=output).wait() output.seek(0) assert output.read() == 'BORK\n' # test that the variable is cleared with temporary_file() as new_output: subprocess.Popen([ sys.executable, '-c', 'import os; print("HORK" in os.environ)' ], stdout=new_output).wait() new_output.seek(0) assert new_output.read() == 'False\n'
def write_and_run_simple_pex(inheriting=False): """Write a pex file that contains an executable entry point :param inheriting: whether this pex should inherit site-packages paths :type inheriting: bool """ with temporary_dir() as td: pex_path = os.path.join(td, 'show_path.pex') with open(os.path.join(td, 'exe.py'), 'w') as fp: fp.write('') # No contents, we just want the startup messages pb = PEXBuilder(path=td, preamble=None) pb.info.inherit_path = inheriting pb.set_executable(os.path.join(td, 'exe.py')) pb.freeze() pb.build(pex_path) with environment_as(PEX_VERBOSE='1'): yield run_simple_pex(pex_path)[0]
def execute(self): def is_python_test(target): # Note that we ignore PythonTestSuite, because we'll see the PythonTests targets # it depends on anyway,so if we don't we'll end up running the tests twice. # TODO(benjy): Once we're off the 'build' command we can get rid of python_test_suite, # or make it an alias of dependencies(). return isinstance(target, PythonTests) test_targets = list(filter(is_python_test, self.context.targets())) if test_targets: self.context.lock.release() # TODO(benjy): Only color on terminals that support it. args = ['--color', 'yes'] # TODO(benjy): A less hacky way to find the log level. if self.context.options.log_level == 'debug': args.append( '-s' ) # Make pytest emit all stdout/stderr, even for successful tests. if self.context.options.pytest_run_options: for options in self.context.options.pytest_run_options: args.extend(shlex.split(options)) test_builder = PythonTestBuilder( targets=test_targets, args=args, interpreter=self.interpreter, conn_timeout=self.conn_timeout, fast=self.context.options.pytest_run_fast) with self.context.new_workunit( name='run', labels=[WorkUnit.TOOL, WorkUnit.TEST]) as workunit: # pytest uses py.io.terminalwriter for output. That class detects the terminal # width and attempts to use all of it. However we capture and indent the console # output, leading to weird-looking line wraps. So we trick the detection code # into thinking the terminal window is narrower than it is. cols = os.environ.get('COLUMNS', 80) with environment_as(COLUMNS=str(int(cols) - 30)): stdout = workunit.output('stdout') if workunit else None stderr = workunit.output('stderr') if workunit else None if test_builder.run(stdout=stdout, stderr=stderr): raise TaskError()
def run(self, binary=None, interpreter_args=[], args=[], extra_deps=[], with_chroot=False, kill_orphans=False): """ Run the PythonEnvironment in an interpreter in a subprocess. """ cmdline = self.cmdline(binary, interpreter_args, args) path = self.path(extras=extra_deps) with pushd(self._dir.path() if with_chroot else os.getcwd()): with environment_as(PYTHONPATH=":".join(path)): PythonLauncher.debug("With PYTHONPATH=%s, executing %s" % (":".join(path), " ".join(cmdline))) # Spawn in a new session so we can cleanup any orphans po = subprocess.Popen(cmdline, preexec_fn=os.setsid) rv = -1 try: rv = po.wait() finally: if kill_orphans and rv: self._reap_orphans(po.pid) return rv
def build_egg(self, egg_root, target): """Build an egg containing the files at egg_root for the specified target. There must be an egg_root/setup.py file.""" # TODO(Brian Wickman): Do a sanity check somewhere to ensure that # setuptools is on the path? args = [sys.executable, "setup.py", "bdist_egg", "--dist-dir=dist", "--bdist-dir=build.%s" % target.name] with pushd(egg_root): with environment_as(PYTHONPATH=":".join(sys.path)): po = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE) rv = po.wait() eggs = os.path.abspath(os.path.join("dist", "*.egg")) eggs = glob.glob(eggs) if rv != 0 or len(eggs) != 1: comm = po.communicate() print("egg generation failed (return value=%d, num eggs=%d)" % (rv, len(eggs)), file=sys.stderr) print("STDOUT", file=sys.stderr) print(comm[0], file=sys.stderr) print("STDERR", file=sys.stderr) print(comm[1], file=sys.stderr) raise EggBuilder.EggBuildingException("Generation of eggs failed for target = %s" % target) egg_path = eggs[0] return egg_path
def test_cache_disable(): with temporary_dir() as tmp_home: with environment_as(HOME=tmp_home): with temporary_dir() as td: with temporary_dir() as output_dir: env = make_env(PEX_INTERPRETER=1) output_path = os.path.join(output_dir, 'pex.pex') args = [ 'pex', '-o', output_path, '--not-zip-safe', '--disable-cache', '--pex-root={0}'.format(td), ] results = run_pex_command(args=args, env=env) results.assert_success() assert [ 'pex.pex' ] == os.listdir(output_dir), 'Expected built pex file.' assert [] == os.listdir( tmp_home), 'Expected empty temp home dir.'
def execute(self): def is_python_test(target): # Note that we ignore PythonTestSuite, because we'll see the PythonTests targets # it depends on anyway,so if we don't we'll end up running the tests twice. # TODO(benjy): Once we're off the 'build' command we can get rid of python_test_suite, # or make it an alias of dependencies(). return isinstance(target, PythonTests) test_targets = list(filter(is_python_test, self.context.targets())) if test_targets: self.context.lock.release() # TODO(benjy): Only color on terminals that support it. args = ['--color', 'yes'] # TODO(benjy): A less hacky way to find the log level. if self.context.options.log_level == 'debug': args.append('-s') # Make pytest emit all stdout/stderr, even for successful tests. if self.context.options.pytest_run_options: for options in self.context.options.pytest_run_options: args.extend(shlex.split(options)) test_builder = PythonTestBuilder(targets=test_targets, args=args, interpreter=self.interpreter, conn_timeout=self.conn_timeout, fast=self.context.options.pytest_run_fast) with self.context.new_workunit(name='run', labels=[WorkUnit.TOOL, WorkUnit.TEST]) as workunit: # pytest uses py.io.terminalwriter for output. That class detects the terminal # width and attempts to use all of it. However we capture and indent the console # output, leading to weird-looking line wraps. So we trick the detection code # into thinking the terminal window is narrower than it is. cols = os.environ.get('COLUMNS', 80) with environment_as(COLUMNS=str(int(cols) - 30)): stdout = workunit.output('stdout') if workunit else None stderr = workunit.output('stderr') if workunit else None if test_builder.run(stdout=stdout, stderr=stderr): raise TaskError()
def _bootstrap_pex(self, pex_path: pathlib.PosixPath) -> None: """Bootstraps a pex with widget UI display.""" title = f"[Bootstrap] {pex_path.name}" with self._accordion_widget(title) as (expand, collapse, set_output_glyph): try: with environment_as(PEX_VERBOSE="2"): # Scrub the environment. self._display_line( "Scrubbing sys.path and sys.modules in preparation for pex bootstrap\n" ) self._display_line( f"sys.path contains {len(sys.path)} items, " f"sys.modules contains {len(sys.modules)} keys\n") for path in self._pex_manager.unmount(): self._display_line(f"scrubbed sys.path entry {path}\n") self._display_line( f"sys.path now contains {len(sys.path)} items, " f"sys.modules now contains {len(sys.modules)} keys\n") # Bootstrap pex. for path in self._pex_manager.mount(pex_path): self._display_line(f"added sys.path entry {path}\n") except Exception: try: set_output_glyph(FAIL_GLYPH) expand() finally: raise else: self._display_line( f"Successfully bootstrapped pex environment from {pex_path}\n" ) set_output_glyph(SUCCESS_GLYPH) collapse()
def wrapped_app(*args, **kwargs): with environment_as(KRB5_KTNAME=self._keytab): return self.authorize(request, app, args, kwargs)
def env(**kwargs): environment = dict(ANDROID_HOME=None, ANDROID_SDK_HOME=None, ANDROID_SDK=None) environment.update(**kwargs) with environment_as(**environment): yield
def env(**kwargs): environment = dict(JDK_HOME=None, JAVA_HOME=None, PATH=None) environment.update(**kwargs) with environment_as(**environment): yield
def test_empty_environment(): with environment_as(): pass
def test_via_env(self): with environment_as(PANTS_BUILD_ROOT=self.new_root): self.assertEqual(self.new_root, BuildRoot().path)