Example #1
0
def test_setup_python_multiple_transitive_markers():
    py27_interpreter = ensure_python_interpreter(PY27)
    py36_interpreter = ensure_python_interpreter(PY36)
    with temporary_dir() as out:
        pex = os.path.join(out, 'pex.pex')
        results = run_pex_command([
            'jsonschema==2.6.0', '--disable-cache',
            '--python-shebang=#!/usr/bin/env python',
            '--python={}'.format(py27_interpreter),
            '--python={}'.format(py36_interpreter), '-o', pex
        ])
        results.assert_success()

        pex_program = [pex, '-c']
        py2_only_program = pex_program + ['import functools32']
        both_program = pex_program + [
            'import jsonschema, os, sys; print(os.path.realpath(sys.executable))'
        ]

        with environment_as(PATH=os.path.dirname(py27_interpreter)):
            subprocess.check_call(py2_only_program)

            stdout = subprocess.check_output(both_program)
            assert to_bytes(
                os.path.realpath(py27_interpreter)) == stdout.strip()

        with environment_as(PATH=os.path.dirname(py36_interpreter)):
            with pytest.raises(subprocess.CalledProcessError):
                subprocess.check_call(py2_only_program)

            stdout = subprocess.check_output(both_program)
            assert to_bytes(
                os.path.realpath(py36_interpreter)) == stdout.strip()
Example #2
0
def test_environment_negation():
  with temporary_file() as output:
    with environment_as(HORK = 'BORK'):
      with environment_as(HORK = None):
      # test that the variable is cleared
        subprocess.Popen([sys.executable, '-c', 'import os; print os.environ.has_key("HORK")'],
          stdout=output).wait()
        output.seek(0)
        assert output.read() == 'False\n'
Example #3
0
def test_environment_negation():
    with temporary_file() as output:
        with environment_as(HORK='BORK'):
            with environment_as(HORK=None):
                # test that the variable is cleared
                subprocess.Popen([
                    sys.executable, '-c',
                    'import os; print("HORK" in os.environ)'
                ],
                                 stdout=output).wait()
                output.seek(0)
                assert output.read() == 'False\n'
Example #4
0
 def _maybe_scrubbed_env(cls):
   for env_var in cls._SCRUBBED_ENV:
     value = os.getenv(env_var)
     if value:
       log.warn('Scrubbing {env_var}={value}'.format(env_var=env_var, value=value))
   with environment_as(**cls._SCRUBBED_ENV):
     yield
Example #5
0
    def _bootstrap_pex(self, pex_path: pathlib.PosixPath):
        """Bootstraps a pex with widget UI display."""
        title = f'[Bootstrap] {pex_path.name}'
        with self._accordion_widget(title) as (expand, collapse,
                                               set_output_glyph):
            try:
                with environment_as(PEX_VERBOSE='2'):
                    # Scrub the environment.
                    _scrub_import_environment(
                        self._ORIGINATING_SYS_MODULES_KEYS, self._display_line)

                    # Bootstrap pex.
                    bootstrap_pex_env(pex_path)
            except Exception:
                try:
                    set_output_glyph(FAIL_GLYPH)
                    expand()
                finally:
                    raise
            else:
                self._display_line(
                    f'Successfully bootstrapped pex environment from {pex_path}\n'
                )
                set_output_glyph(SUCCESS_GLYPH)
                collapse()
Example #6
0
 def build_egg(self, egg_root, target):
     """Build an egg containing the files at egg_root for the specified target.
 There must be an egg_root/setup.py file."""
     # TODO(Brian Wickman): Do a sanity check somewhere to ensure that
     # setuptools is on the path?
     args = [
         sys.executable, 'setup.py', 'bdist_egg', '--dist-dir=dist',
         '--bdist-dir=build.%s' % target.name
     ]
     with pushd(egg_root):
         with environment_as(PYTHONPATH=':'.join(sys.path)):
             po = subprocess.Popen(args,
                                   stderr=subprocess.PIPE,
                                   stdout=subprocess.PIPE)
             rv = po.wait()
         eggs = os.path.abspath(os.path.join('dist', '*.egg'))
         eggs = glob.glob(eggs)
         if rv != 0 or len(eggs) != 1:
             comm = po.communicate()
             print('egg generation failed (return value=%d, num eggs=%d)' %
                   (rv, len(eggs)),
                   file=sys.stderr)
             print('STDOUT', file=sys.stderr)
             print(comm[0], file=sys.stderr)
             print('STDERR', file=sys.stderr)
             print(comm[1], file=sys.stderr)
             raise EggBuilder.EggBuildingException(
                 'Generation of eggs failed for target = %s' % target)
         egg_path = eggs[0]
     return egg_path
Example #7
0
  def setUpClass(cls):
    cls.origin = safe_mkdtemp()
    with pushd(cls.origin):
      subprocess.check_call(['git', 'init', '--bare'])

    cls.gitdir = safe_mkdtemp()
    cls.worktree = safe_mkdtemp()

    cls.readme_file = os.path.join(cls.worktree, 'README')

    with environment_as(GIT_DIR=cls.gitdir, GIT_WORK_TREE=cls.worktree):
      cls.init_repo('depot', cls.origin)

      touch(cls.readme_file)
      subprocess.check_call(['git', 'add', 'README'])
      subprocess.check_call(['git', 'commit', '-am', 'initial commit with decode -> \x81b'])
      subprocess.check_call(['git', 'tag', 'first'])
      subprocess.check_call(['git', 'push', '--tags', 'depot', 'master'])
      subprocess.check_call(['git', 'branch', '--set-upstream', 'master', 'depot/master'])

      with safe_open(cls.readme_file, 'w') as readme:
        readme.write('Hello World.')
      subprocess.check_call(['git', 'commit', '-am', 'Update README.'])

    cls.clone2 = safe_mkdtemp()
    with pushd(cls.clone2):
      cls.init_repo('origin', cls.origin)
      subprocess.check_call(['git', 'pull', '--tags', 'origin', 'master:master'])

      with safe_open(os.path.realpath('README'), 'a') as readme:
        readme.write('--')
      subprocess.check_call(['git', 'commit', '-am', 'Update README 2.'])
      subprocess.check_call(['git', 'push', '--tags', 'origin', 'master'])

    cls.git = Git(gitdir=cls.gitdir, worktree=cls.worktree)
Example #8
0
 def build_egg(self, egg_root, target):
   """Build an egg containing the files at egg_root for the specified target.
   There must be an egg_root/setup.py file."""
   # TODO(Brian Wickman): Do a sanity check somewhere to ensure that
   # setuptools is on the path?
   args = [
     sys.executable,
     'setup.py', 'bdist_egg',
     '--dist-dir=dist',
     '--bdist-dir=build.%s' % target.name]
   with pushd(egg_root):
     print 'EggBuilder executing: %s' % ' '.join(args)
     with environment_as(PYTHONPATH = ':'.join(sys.path)):
       po = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
       rv = po.wait()
     eggs = os.path.abspath(os.path.join('dist', '*.egg'))
     eggs = glob.glob(eggs)
     if rv != 0 or len(eggs) != 1:
       comm = po.communicate()
       print >> sys.stderr, 'egg generation failed (return value=%d, num eggs=%d)' % (
         rv, len(eggs))
       print >> sys.stderr, 'STDOUT'
       print >> sys.stderr, comm[0]
       print >> sys.stderr, 'STDERR'
       print >> sys.stderr, comm[1]
       raise EggBuilder.EggBuildingException(
         'Generation of eggs failed for target = %s' % target)
     egg_path = eggs[0]
   return egg_path
Example #9
0
    def test(self):
        self.assertEqual(set(), self.git.changed_files())
        self.assertEqual(set(['README']),
                         self.git.changed_files(from_commit='HEAD^'))

        tip_sha = self.git.commit_id
        self.assertTrue(tip_sha)

        self.assertTrue(tip_sha in self.git.changelog())

        self.assertTrue(self.git.tag_name.startswith('first-'))
        self.assertEqual('master', self.git.branch_name)

        def edit_readme():
            with open(self.readme_file, 'a') as readme:
                readme.write('More data.')

        edit_readme()
        with open(os.path.join(self.worktree, 'INSTALL'), 'w') as untracked:
            untracked.write('make install')
        self.assertEqual(set(['README']), self.git.changed_files())
        self.assertEqual(set(['README', 'INSTALL']),
                         self.git.changed_files(include_untracked=True))

        try:
            # These changes should be rejected because our branch point from origin is 1 commit behind
            # the changes pushed there in clone 2.
            self.git.commit('API Changes.')
        except Scm.RemoteException:
            with environment_as(GIT_DIR=self.gitdir,
                                GIT_WORK_TREE=self.worktree):
                subprocess.check_call(
                    ['git', 'reset', '--hard', 'depot/master'])
            self.git.refresh()
            edit_readme()

        self.git.commit('''API '"' " Changes.''')
        self.git.tag('second', message='''Tagged ' " Changes''')

        with temporary_dir() as clone:
            with pushd(clone):
                self.init_repo('origin', self.origin)
                subprocess.check_call(
                    ['git', 'pull', '--tags', 'origin', 'master:master'])

                with open(os.path.realpath('README')) as readme:
                    self.assertEqual('--More data.', readme.read())

                git = Git()

                # Check that we can pick up committed and uncommitted changes.
                with safe_open(os.path.realpath('CHANGES'), 'w') as changes:
                    changes.write('none')
                subprocess.check_call(['git', 'add', 'CHANGES'])
                self.assertEqual(set(['README', 'CHANGES']),
                                 git.changed_files(from_commit='first'))

                self.assertEqual('master', git.branch_name)
                self.assertEqual('second', git.tag_name)
Example #10
0
    def run(self):
        self._run_count += 1
        atexit.register(self.cleanup)

        if self.script_filename:
            os.unlink(self.script_filename)

        with temporary_file(cleanup=False) as fp:
            self.script_filename = fp.name
            fp.write(
                self.RUN_JOB_SCRIPT % {
                    'filename': self.job_filename,
                    'sandbox': self.sandbox,
                    'root': self.tempdir,
                    'task_id': self.task_id,
                    'state_filename': self.state_filename,
                    'portmap': repr(self.portmap),
                    'success_rate': self.success_rate,
                    'random_seed': self.random_seed + self._run_count,
                })

        with environment_as(PYTHONPATH=os.pathsep.join(sys.path)):
            self.po = subprocess.Popen([sys.executable, self.script_filename],
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
            try:
                so, se = self.po.communicate()
            except OSError as e:
                if e.errno == errno.ECHILD:
                    so = se = 'Killed'
                else:
                    raise

        rc = self.po.returncode
        if rc != 0:
            if os.path.exists(self.job_filename):
                config = open(self.job_filename).read()
            else:
                config = 'Nonexistent!'
            if 'THERMOS_DEBUG' in os.environ:
                print(
                    "Runner failed!\n\n\nconfig:%s\n\n\nstdout:%s\n\n\nstderr:%s\n\n\n"
                    % (config, so, se))

        try:
            with open(self.state_filename, 'r') as fp:
                self.state = thrift_deserialize(RunnerState(), fp.read())
        except Exception as e:
            if 'THERMOS_DEBUG' in os.environ:
                print('Failed to load Runner state: %s' % e, file=sys.stderr)
            self.state = RunnerState()

        try:
            self.reconstructed_state = CheckpointDispatcher.from_file(
                self.pathspec.getpath('runner_checkpoint'))
        except:
            self.reconstructed_state = None
        self.initialized = True
        return rc
Example #11
0
  def run(self):
    self._run_count += 1
    atexit.register(self.cleanup)

    if self.script_filename:
      os.unlink(self.script_filename)

    with temporary_file(cleanup=False) as fp:
      self.script_filename = fp.name
      fp.write(self.RUN_JOB_SCRIPT % {
        'filename': self.job_filename,
        'sandbox': self.sandbox,
        'root': self.tempdir,
        'task_id': self.task_id,
        'state_filename': self.state_filename,
        'success_rate': self.success_rate,
        'random_seed': self.random_seed + self._run_count,
        'extra_task_runner_args': self.extra_task_runner_args,
      })

    with environment_as(PYTHONPATH=os.pathsep.join(sys.path)):
      self.po = subprocess.Popen([sys.executable, self.script_filename],
        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
      try:
        so, se = self.po.communicate()
      except OSError as e:
        if e.errno == errno.ECHILD:
          so = se = 'Killed'
        else:
          raise

    rc = self.po.returncode
    if rc != 0:
      if os.path.exists(self.job_filename):
        with open(self.job_filename) as fp:
          config = fp.read()
      else:
        config = 'Nonexistent!'
      if 'THERMOS_DEBUG' in os.environ:
        print("Runner failed!\n\n\nconfig:%s\n\n\nstdout:%s\n\n\nstderr:%s\n\n\n" % (
            config, so, se))

    try:
      with open(self.state_filename, 'r') as fp:
        self.state = thrift_deserialize(RunnerState(), fp.read())
    except Exception as e:
      if 'THERMOS_DEBUG' in os.environ:
        print('Failed to load Runner state: %s' % e, file=sys.stderr)
      self.state = RunnerState()

    try:
      self.reconstructed_state = CheckpointDispatcher.from_file(
          self.pathspec.getpath('runner_checkpoint'))
    except Exception as e:
      print('Failed to replay checkpoint: %s' % e, file=sys.stderr)
      self.reconstructed_state = None
    self.initialized = True
    return rc
Example #12
0
def test_pex_python():
  py2_path_interpreter = ensure_python_interpreter('2.7.10')
  py3_path_interpreter = ensure_python_interpreter('3.6.3')
  path = ':'.join([os.path.dirname(py2_path_interpreter), os.path.dirname(py3_path_interpreter)])
  with environment_as(PATH=path):
    with temporary_dir() as td:
      pexrc_path = os.path.join(td, '.pexrc')
      with open(pexrc_path, 'w') as pexrc:
        pex_python = ensure_python_interpreter('3.6.3')
        pexrc.write("PEX_PYTHON=%s" % pex_python)

      # test PEX_PYTHON with valid constraints
      pex_out_path = os.path.join(td, 'pex.pex')
      res = run_pex_command(['--disable-cache',
        '--rcfile=%s' % pexrc_path,
        '--interpreter-constraint=>3',
        '--interpreter-constraint=<3.8',
        '-o', pex_out_path])
      res.assert_success()

      stdin_payload = b'import sys; print(sys.executable); sys.exit(0)'
      stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload)
      assert rc == 0
      correct_interpreter_path = pex_python.encode()
      assert correct_interpreter_path in stdout

      # test PEX_PYTHON with incompatible constraints
      pexrc_path = os.path.join(td, '.pexrc')
      with open(pexrc_path, 'w') as pexrc:
        pex_python = ensure_python_interpreter('2.7.10')
        pexrc.write("PEX_PYTHON=%s" % pex_python)

      pex_out_path = os.path.join(td, 'pex2.pex')
      res = run_pex_command(['--disable-cache',
        '--rcfile=%s' % pexrc_path,
        '--interpreter-constraint=>3',
        '--interpreter-constraint=<3.8',
        '-o', pex_out_path])
      res.assert_success()

      stdin_payload = b'import sys; print(sys.executable); sys.exit(0)'
      stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload)
      assert rc == 1
      fail_str = 'not compatible with specified interpreter constraints'.encode()
      assert fail_str in stdout

      # test PEX_PYTHON with no constraints
      pex_out_path = os.path.join(td, 'pex3.pex')
      res = run_pex_command(['--disable-cache',
        '--rcfile=%s' % pexrc_path,
        '-o', pex_out_path])
      res.assert_success()

      stdin_payload = b'import sys; print(sys.executable); sys.exit(0)'
      stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload)
      assert rc == 0
      correct_interpreter_path = pex_python.encode()
      assert correct_interpreter_path in stdout
Example #13
0
 def _maybe_scrubbed_classpath(self):
     if self._scrub_classpath:
         classpath = os.getenv('CLASSPATH')
         if classpath:
             log.warn('Scrubbing CLASSPATH=%s' % classpath)
         with environment_as(CLASSPATH=None):
             yield
     else:
         yield
Example #14
0
 def _maybe_scrubbed_classpath(self):
   if self._scrub_classpath:
     classpath = os.getenv('CLASSPATH')
     if classpath:
       log.warn('Scrubbing CLASSPATH=%s' % classpath)
     with environment_as(CLASSPATH=None):
       yield
   else:
     yield
Example #15
0
 def config(self, overrides=''):
   """Returns a config valid for the test build root."""
   if overrides:
     with temporary_file() as fp:
       fp.write(overrides)
       fp.close()
       with environment_as(PANTS_CONFIG_OVERRIDE=fp.name):
         return Config.load()
   else:
     return Config.load()
Example #16
0
 def config(self, overrides=''):
   """Returns a config valid for the test build root."""
   if overrides:
     with temporary_file() as fp:
       fp.write(overrides)
       fp.close()
       with environment_as(PANTS_CONFIG_OVERRIDE=fp.name):
         return Config.load()
   else:
     return Config.load()
Example #17
0
def test_setup_interpreter_constraint():
    interpreter = ensure_python_interpreter(PY27)
    with temporary_dir() as out:
        pex = os.path.join(out, 'pex.pex')
        with environment_as(PATH=os.path.dirname(interpreter)):
            results = run_pex_command([
                'jsonschema==2.6.0', '--disable-cache',
                '--interpreter-constraint=CPython=={}'.format(PY27), '-o', pex
            ])
            results.assert_success()
            subprocess.check_call([pex, '-c', 'import jsonschema'])
Example #18
0
  def test(self):
    self.assertEqual(set(), self.git.changed_files())
    self.assertEqual(set(['README']), self.git.changed_files(from_commit='HEAD^'))

    tip_sha = self.git.commit_id
    self.assertTrue(tip_sha)

    self.assertTrue(tip_sha in self.git.changelog())

    self.assertTrue(self.git.tag_name.startswith('first-'))
    self.assertEqual('master', self.git.branch_name)

    def edit_readme():
      with open(self.readme_file, 'a') as readme:
        readme.write('More data.')

    edit_readme()
    with open(os.path.join(self.worktree, 'INSTALL'), 'w') as untracked:
      untracked.write('make install')
    self.assertEqual(set(['README']), self.git.changed_files())
    self.assertEqual(set(['README', 'INSTALL']), self.git.changed_files(include_untracked=True))

    try:
      # These changes should be rejected because our branch point from origin is 1 commit behind
      # the changes pushed there in clone 2.
      self.git.commit('API Changes.')
    except Scm.RemoteException:
      with environment_as(GIT_DIR=self.gitdir, GIT_WORK_TREE=self.worktree):
        subprocess.check_call(['git', 'reset', '--hard', 'depot/master'])
      self.git.refresh()
      edit_readme()

    self.git.commit('''API '"' " Changes.''')
    self.git.tag('second', message='''Tagged ' " Changes''')

    with temporary_dir() as clone:
      with pushd(clone):
        subprocess.check_call(['git', 'init'])
        subprocess.check_call(['git', 'remote', 'add', 'origin', self.origin])
        subprocess.check_call(['git', 'pull', '--tags', 'origin', 'master:master'])

        with open(os.path.realpath('README')) as readme:
          self.assertEqual('--More data.', readme.read())

        git = Git()

        # Check that we can pick up committed and uncommitted changes.
        with safe_open(os.path.realpath('CHANGES'), 'w') as changes:
          changes.write('none')
        subprocess.check_call(['git', 'add', 'CHANGES'])
        self.assertEqual(set(['README', 'CHANGES']), git.changed_files(from_commit='first'))

        self.assertEqual('master', git.branch_name)
        self.assertEqual('second', git.tag_name)
Example #19
0
def safe_classpath(logger=None):
    """
    Yields to a block in an environment with no CLASSPATH.  This is useful to ensure hermetic java
    invocations.
  """
    classpath = os.getenv('CLASSPATH')
    if classpath:
        logger = logger or log.warn
        logger('Scrubbing CLASSPATH=%s' % classpath)
    with environment_as(CLASSPATH=None):
        yield
Example #20
0
def safe_classpath(logger=None):
  """
    Yields to a block in an environment with no CLASSPATH.  This is useful to ensure hermetic java
    invocations.
  """
  classpath = os.getenv('CLASSPATH')
  if classpath:
    logger = logger or log.warn
    logger('Scrubbing CLASSPATH=%s' % classpath)
  with environment_as(CLASSPATH=None):
    yield
Example #21
0
 def do_test_jre_env_var(self, env_var, env_value, scrubbed=True):
   with self.jre(env_var=env_var) as jre:
     executor = SubprocessExecutor(Distribution(bin_path=jre))
     with environment_as(**{env_var: env_value}):
       self.assertEqual(env_value, os.getenv(env_var))
       process = executor.spawn(classpath=['dummy/classpath'],
                                main='dummy.main',
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
       stdout, _ = process.communicate()
       self.assertEqual(0, process.returncode)
       self.assertEqual('' if scrubbed else env_value, stdout.strip())
Example #22
0
def test_pex_python_symlink():
  with temporary_dir() as td:
    with environment_as(HOME=td):
      symlink_path = os.path.join(td, 'python-symlink')
      os.symlink(sys.executable, symlink_path)
      pexrc_path = os.path.join(td, '.pexrc')
      with open(pexrc_path, 'w') as pexrc:
        pexrc.write("PEX_PYTHON=%s" % symlink_path)

      body = "print('Hello')"
      _, rc = run_simple_pex_test(body, coverage=True)
      assert rc == 0
Example #23
0
def test_interpreter_constraints_to_pex_info_py3():
  py3_interpreter = ensure_python_interpreter('3.6.3')
  with environment_as(PATH=os.path.dirname(py3_interpreter)):
    with temporary_dir() as output_dir:
      # target python 3
      pex_out_path = os.path.join(output_dir, 'pex_py3.pex')
      res = run_pex_command(['--disable-cache',
        '--interpreter-constraint=>3',
        '-o', pex_out_path])
      res.assert_success()
      pex_info = get_pex_info(pex_out_path)
      assert ['>3'] == pex_info.interpreter_constraints
Example #24
0
def test_setup_python_multiple_direct_markers():
    py36_interpreter = ensure_python_interpreter(PY36)
    py27_interpreter = ensure_python_interpreter(PY27)
    with temporary_dir() as out:
        pex = os.path.join(out, 'pex.pex')
        results = run_pex_command([
            'subprocess32==3.2.7; python_version<"3"', '--disable-cache',
            '--python-shebang=#!/usr/bin/env python',
            '--python={}'.format(py36_interpreter),
            '--python={}'.format(py27_interpreter), '-o', pex
        ])
        results.assert_success()

        py2_only_program = [pex, '-c', 'import subprocess32']

        with environment_as(PATH=os.path.dirname(py36_interpreter)):
            with pytest.raises(subprocess.CalledProcessError):
                subprocess.check_call(py2_only_program)

        with environment_as(PATH=os.path.dirname(py27_interpreter)):
            subprocess.check_call(py2_only_program)
Example #25
0
def test_pex_python_symlink():
  with temporary_dir() as td:
    with environment_as(HOME=td):
      symlink_path = os.path.join(td, 'python-symlink')
      os.symlink(sys.executable, symlink_path)
      pexrc_path = os.path.join(td, '.pexrc')
      with open(pexrc_path, 'w') as pexrc:
        pexrc.write("PEX_PYTHON=%s" % symlink_path)

      body = "print('Hello')"
      _, rc = run_simple_pex_test(body, coverage=True)
      assert rc == 0
Example #26
0
 def _call(self, cmd, *args, **kwargs):
   """Runs hadoop fs command  with the given command and args.
   Checks the result of the call by default but this can be disabled with check=False.
   """
   cmd = ['hadoop', '--config', self._config, 'dfs', cmd] + list(args)
   heapsize = str(int(self._heap_limit.as_(Data.MB)))
   with environment_as(HADOOP_HEAPSIZE=heapsize):
     if kwargs.get('check'):
       return self._cmd_class.check_call(cmd)
     elif kwargs.get('return_output'):
       return self._cmd_class.execute_and_get_output(cmd)
     elif kwargs.get('supress_output'):
       return self._cmd_class.execute_suppress_stdout(cmd)
     else:
       return self._cmd_class.execute(cmd)
Example #27
0
def test_pex_root():
  with temporary_dir() as tmp_home:
    with environment_as(HOME=tmp_home):
      with temporary_dir() as td:
        with temporary_dir() as output_dir:
          env = os.environ.copy()
          env['PEX_INTERPRETER'] = '1'

          output_path = os.path.join(output_dir, 'pex.pex')
          args = ['pex', '-o', output_path, '--not-zip-safe', '--pex-root={0}'.format(td)]
          results = run_pex_command(args=args, env=env)
          results.assert_success()
          assert ['pex.pex'] == os.listdir(output_dir), 'Expected built pex file.'
          assert [] == os.listdir(tmp_home), 'Expected empty temp home dir.'
          assert 'build' in os.listdir(td), 'Expected build directory in tmp pex root.'
Example #28
0
 def _call(self, cmd, *args, **kwargs):
     """Runs hadoop fs command  with the given command and args.
 Checks the result of the call by default but this can be disabled with check=False.
 """
     cmd = ['hadoop', '--config', self._config, 'dfs', cmd] + list(args)
     heapsize = str(int(self._heap_limit.as_(Data.MB)))
     with environment_as(HADOOP_HEAPSIZE=heapsize):
         if kwargs.get('check'):
             return self._cmd_class.check_call(cmd)
         elif kwargs.get('return_output'):
             return self._cmd_class.execute_and_get_output(cmd)
         elif kwargs.get('supress_output'):
             return self._cmd_class.execute_suppress_stdout(cmd)
         else:
             return self._cmd_class.execute(cmd)
Example #29
0
def test_override_single_variable():
  with temporary_file() as output:
    # test that the override takes place
    with environment_as(HORK = 'BORK'):
      subprocess.Popen([sys.executable, '-c', 'import os; print os.environ["HORK"]'],
        stdout=output).wait()
      output.seek(0)
      assert output.read() == 'BORK\n'

    # test that the variable is cleared
    with temporary_file() as new_output:
      subprocess.Popen([sys.executable, '-c', 'import os; print os.environ.has_key("HORK")'],
        stdout=new_output).wait()
      new_output.seek(0)
      assert new_output.read() == 'False\n'
Example #30
0
def test_pex_root():
  with temporary_dir() as tmp_home:
    with environment_as(HOME=tmp_home):
      with temporary_dir() as td:
        with temporary_dir() as output_dir:
          env = os.environ.copy()
          env['PEX_INTERPRETER'] = '1'

          output_path = os.path.join(output_dir, 'pex.pex')
          args = ['pex', '-o', output_path, '--not-zip-safe', '--pex-root={0}'.format(td)]
          results = run_pex_command(args=args, env=env)
          results.assert_success()
          assert ['pex.pex'] == os.listdir(output_dir), 'Expected built pex file.'
          assert [] == os.listdir(tmp_home), 'Expected empty temp home dir.'
          assert 'build' in os.listdir(td), 'Expected build directory in tmp pex root.'
Example #31
0
def test_override_single_variable():
    with temporary_file() as output:
        # test that the override takes place
        with environment_as(HORK='BORK'):
            subprocess.Popen(
                [sys.executable, '-c', 'import os; print(os.environ["HORK"])'],
                stdout=output).wait()
            output.seek(0)
            assert output.read() == 'BORK\n'

        # test that the variable is cleared
        with temporary_file() as new_output:
            subprocess.Popen([
                sys.executable, '-c', 'import os; print("HORK" in os.environ)'
            ],
                             stdout=new_output).wait()
            new_output.seek(0)
            assert new_output.read() == 'False\n'
Example #32
0
def write_and_run_simple_pex(inheriting=False):
  """Write a pex file that contains an executable entry point

  :param inheriting: whether this pex should inherit site-packages paths
  :type inheriting: bool
  """
  with temporary_dir() as td:
    pex_path = os.path.join(td, 'show_path.pex')
    with open(os.path.join(td, 'exe.py'), 'w') as fp:
      fp.write('')  # No contents, we just want the startup messages

    pb = PEXBuilder(path=td, preamble=None)
    pb.info.inherit_path = inheriting
    pb.set_executable(os.path.join(td, 'exe.py'))
    pb.freeze()
    pb.build(pex_path)
    with environment_as(PEX_VERBOSE='1'):
      yield run_simple_pex(pex_path)[0]
Example #33
0
    def execute(self):
        def is_python_test(target):
            # Note that we ignore PythonTestSuite, because we'll see the PythonTests targets
            # it depends on anyway,so if we don't we'll end up running the tests twice.
            # TODO(benjy): Once we're off the 'build' command we can get rid of python_test_suite,
            # or make it an alias of dependencies().
            return isinstance(target, PythonTests)

        test_targets = list(filter(is_python_test, self.context.targets()))
        if test_targets:
            self.context.lock.release()

            # TODO(benjy): Only color on terminals that support it.
            args = ['--color', 'yes']
            # TODO(benjy): A less hacky way to find the log level.
            if self.context.options.log_level == 'debug':
                args.append(
                    '-s'
                )  # Make pytest emit all stdout/stderr, even for successful tests.
            if self.context.options.pytest_run_options:
                for options in self.context.options.pytest_run_options:
                    args.extend(shlex.split(options))
            test_builder = PythonTestBuilder(
                targets=test_targets,
                args=args,
                interpreter=self.interpreter,
                conn_timeout=self.conn_timeout,
                fast=self.context.options.pytest_run_fast)
            with self.context.new_workunit(
                    name='run', labels=[WorkUnit.TOOL,
                                        WorkUnit.TEST]) as workunit:
                # pytest uses py.io.terminalwriter for output. That class detects the terminal
                # width and attempts to use all of it. However we capture and indent the console
                # output, leading to weird-looking line wraps. So we trick the detection code
                # into thinking the terminal window is narrower than it is.
                cols = os.environ.get('COLUMNS', 80)
                with environment_as(COLUMNS=str(int(cols) - 30)):
                    stdout = workunit.output('stdout') if workunit else None
                    stderr = workunit.output('stderr') if workunit else None
                    if test_builder.run(stdout=stdout, stderr=stderr):
                        raise TaskError()
Example #34
0
    def run(self, binary=None, interpreter_args=[], args=[], extra_deps=[], with_chroot=False, kill_orphans=False):
        """
      Run the PythonEnvironment in an interpreter in a subprocess.
    """
        cmdline = self.cmdline(binary, interpreter_args, args)
        path = self.path(extras=extra_deps)

        with pushd(self._dir.path() if with_chroot else os.getcwd()):
            with environment_as(PYTHONPATH=":".join(path)):
                PythonLauncher.debug("With PYTHONPATH=%s, executing %s" % (":".join(path), " ".join(cmdline)))
                # Spawn in a new session so we can cleanup any orphans
                po = subprocess.Popen(cmdline, preexec_fn=os.setsid)

                rv = -1
                try:
                    rv = po.wait()
                finally:
                    if kill_orphans and rv:
                        self._reap_orphans(po.pid)

        return rv
Example #35
0
 def build_egg(self, egg_root, target):
     """Build an egg containing the files at egg_root for the specified target.
 There must be an egg_root/setup.py file."""
     # TODO(Brian Wickman): Do a sanity check somewhere to ensure that
     # setuptools is on the path?
     args = [sys.executable, "setup.py", "bdist_egg", "--dist-dir=dist", "--bdist-dir=build.%s" % target.name]
     with pushd(egg_root):
         with environment_as(PYTHONPATH=":".join(sys.path)):
             po = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
             rv = po.wait()
         eggs = os.path.abspath(os.path.join("dist", "*.egg"))
         eggs = glob.glob(eggs)
         if rv != 0 or len(eggs) != 1:
             comm = po.communicate()
             print("egg generation failed (return value=%d, num eggs=%d)" % (rv, len(eggs)), file=sys.stderr)
             print("STDOUT", file=sys.stderr)
             print(comm[0], file=sys.stderr)
             print("STDERR", file=sys.stderr)
             print(comm[1], file=sys.stderr)
             raise EggBuilder.EggBuildingException("Generation of eggs failed for target = %s" % target)
         egg_path = eggs[0]
     return egg_path
Example #36
0
def test_cache_disable():
    with temporary_dir() as tmp_home:
        with environment_as(HOME=tmp_home):
            with temporary_dir() as td:
                with temporary_dir() as output_dir:
                    env = make_env(PEX_INTERPRETER=1)

                    output_path = os.path.join(output_dir, 'pex.pex')
                    args = [
                        'pex',
                        '-o',
                        output_path,
                        '--not-zip-safe',
                        '--disable-cache',
                        '--pex-root={0}'.format(td),
                    ]
                    results = run_pex_command(args=args, env=env)
                    results.assert_success()
                    assert [
                        'pex.pex'
                    ] == os.listdir(output_dir), 'Expected built pex file.'
                    assert [] == os.listdir(
                        tmp_home), 'Expected empty temp home dir.'
Example #37
0
  def execute(self):
    def is_python_test(target):
      # Note that we ignore PythonTestSuite, because we'll see the PythonTests targets
      # it depends on anyway,so if we don't we'll end up running the tests twice.
      # TODO(benjy): Once we're off the 'build' command we can get rid of python_test_suite,
      # or make it an alias of dependencies().
      return isinstance(target, PythonTests)

    test_targets = list(filter(is_python_test, self.context.targets()))
    if test_targets:
      self.context.lock.release()

      # TODO(benjy): Only color on terminals that support it.
      args = ['--color', 'yes']
      # TODO(benjy): A less hacky way to find the log level.
      if self.context.options.log_level == 'debug':
        args.append('-s')  # Make pytest emit all stdout/stderr, even for successful tests.
      if self.context.options.pytest_run_options:
        for options in self.context.options.pytest_run_options:
          args.extend(shlex.split(options))
      test_builder = PythonTestBuilder(targets=test_targets,
                                       args=args,
                                       interpreter=self.interpreter,
                                       conn_timeout=self.conn_timeout,
                                       fast=self.context.options.pytest_run_fast)
      with self.context.new_workunit(name='run',
                                     labels=[WorkUnit.TOOL, WorkUnit.TEST]) as workunit:
        # pytest uses py.io.terminalwriter for output. That class detects the terminal
        # width and attempts to use all of it. However we capture and indent the console
        # output, leading to weird-looking line wraps. So we trick the detection code
        # into thinking the terminal window is narrower than it is.
        cols = os.environ.get('COLUMNS', 80)
        with environment_as(COLUMNS=str(int(cols) - 30)):
          stdout = workunit.output('stdout') if workunit else None
          stderr = workunit.output('stderr') if workunit else None
          if test_builder.run(stdout=stdout, stderr=stderr):
            raise TaskError()
Example #38
0
    def _bootstrap_pex(self, pex_path: pathlib.PosixPath) -> None:
        """Bootstraps a pex with widget UI display."""
        title = f"[Bootstrap] {pex_path.name}"
        with self._accordion_widget(title) as (expand, collapse,
                                               set_output_glyph):
            try:
                with environment_as(PEX_VERBOSE="2"):
                    # Scrub the environment.

                    self._display_line(
                        "Scrubbing sys.path and sys.modules in preparation for pex bootstrap\n"
                    )
                    self._display_line(
                        f"sys.path contains {len(sys.path)} items, "
                        f"sys.modules contains {len(sys.modules)} keys\n")
                    for path in self._pex_manager.unmount():
                        self._display_line(f"scrubbed sys.path entry {path}\n")
                    self._display_line(
                        f"sys.path now contains {len(sys.path)} items, "
                        f"sys.modules now contains {len(sys.modules)} keys\n")

                    # Bootstrap pex.
                    for path in self._pex_manager.mount(pex_path):
                        self._display_line(f"added sys.path entry {path}\n")
            except Exception:
                try:
                    set_output_glyph(FAIL_GLYPH)
                    expand()
                finally:
                    raise
            else:
                self._display_line(
                    f"Successfully bootstrapped pex environment from {pex_path}\n"
                )
                set_output_glyph(SUCCESS_GLYPH)
                collapse()
Example #39
0
 def wrapped_app(*args, **kwargs):
     with environment_as(KRB5_KTNAME=self._keytab):
         return self.authorize(request, app, args, kwargs)
Example #40
0
 def env(**kwargs):
   environment = dict(ANDROID_HOME=None, ANDROID_SDK_HOME=None, ANDROID_SDK=None)
   environment.update(**kwargs)
   with environment_as(**environment):
     yield
Example #41
0
 def env(**kwargs):
     environment = dict(JDK_HOME=None, JAVA_HOME=None, PATH=None)
     environment.update(**kwargs)
     with environment_as(**environment):
         yield
Example #42
0
def test_empty_environment():
    with environment_as():
        pass
Example #43
0
 def wrapped_app(*args, **kwargs):
   with environment_as(KRB5_KTNAME=self._keytab):
     return self.authorize(request, app, args, kwargs)
Example #44
0
 def test_via_env(self):
   with environment_as(PANTS_BUILD_ROOT=self.new_root):
     self.assertEqual(self.new_root, BuildRoot().path)
Example #45
0
def test_empty_environment():
  with environment_as():
    pass
Example #46
0
 def test_via_env(self):
     with environment_as(PANTS_BUILD_ROOT=self.new_root):
         self.assertEqual(self.new_root, BuildRoot().path)
Example #47
0
 def env(**kwargs):
   environment = dict(JDK_HOME=None, JAVA_HOME=None, PATH=None)
   environment.update(**kwargs)
   with environment_as(**environment):
     yield