def setUp(self): super(TaskTestBase, self).setUp() self._testing_task_type, self.options_scope = self.synthesize_task_subtype(self.task_type()) # We locate the workdir below the pants_workdir, which BaseTest locates within # the BuildRoot. self._tmpdir = tempfile.mkdtemp(dir=self.pants_workdir) self._test_workdir = os.path.join(self._tmpdir, 'workdir') os.mkdir(self._test_workdir) Bootstrapper.reset_instance() IvySubsystem.reset_global_instance()
def dumped_chroot(self, targets): # TODO(benjy): We shouldn't need to mention DistributionLocator here, as IvySubsystem # declares it as a dependency. However if we don't then test_antlr() below fails on # uninitialized options for that subsystem. Hopefully my pending (as of 9/2016) change # to clean up how we initialize and create instances of subsystems in tests will make # this problem go away. self.context(for_subsystems=[PythonRepos, PythonSetup, IvySubsystem, DistributionLocator, ThriftBinary.Factory, BinaryUtil.Factory]) python_repos = PythonRepos.global_instance() ivy_bootstrapper = Bootstrapper(ivy_subsystem=IvySubsystem.global_instance()) thrift_binary_factory = ThriftBinary.Factory.global_instance().create interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos) interpreter_cache.setup() interpreters = list(interpreter_cache.matched_interpreters( self.python_setup.interpreter_constraints)) self.assertGreater(len(interpreters), 0) interpreter = interpreters[0] with temporary_dir() as chroot: pex_builder = PEXBuilder(path=chroot, interpreter=interpreter) python_chroot = PythonChroot(python_setup=self.python_setup, python_repos=python_repos, ivy_bootstrapper=ivy_bootstrapper, thrift_binary_factory=thrift_binary_factory, interpreter=interpreter, builder=pex_builder, targets=targets, platforms=['current']) try: python_chroot.dump() yield pex_builder, python_chroot finally: python_chroot.delete()
def ivy_cache_dir(self): """The path of the ivy cache dir used for resolves. :rtype: string """ # TODO(John Sirois): Fixup the IvySubsystem to encapsulate its properties. return IvySubsystem.global_instance().get_options().cache_dir
def _unpack(self, unpacked_archives): """Extracts files from the downloaded jar files and places them in a work directory. :param UnpackedArchives unpacked_archives: target referencing jar_libraries to unpack. """ self.context.log.info('Unpacking {}'.format( unpacked_archives.address.spec)) unpack_dir = unpacked_archives.destination safe_mkdir(unpack_dir, clean=True) unpack_filter = self.get_unpack_filter(unpacked_archives) classpath_products = ClasspathProducts( self.get_options().pants_workdir) resolve_hashes = self.resolve(None, unpacked_archives.dependencies, classpath_products) ivy_cache_dir = os.path.expanduser( IvySubsystem.global_instance().get_options().cache_dir) def to_m2(jar): return M2Coordinate(org=jar.org, name=jar.name, rev=jar.rev, classifier=jar.classifier, ext=jar.ext) libraries = self.context.build_graph.transitive_subgraph_of_addresses( [unpacked_archives.address]) libraries = [t for t in libraries if isinstance(t, JarLibrary)] coords = set() for library in libraries: coords.update(to_m2(jar) for jar in library.payload.jars) for resolve_hash in resolve_hashes: path = IvyUtils.xml_report_path(ivy_cache_dir, resolve_hash, 'default') info = IvyUtils.parse_xml_report('default', path) refs_for_libraries = set() for ref in info.modules_by_ref.keys(): if to_m2(ref) in coords: refs_for_libraries.add(ref) memo = {} for ref in tuple(refs_for_libraries): info.traverse_dependency_graph(ref, refs_for_libraries.add, memo) for ref in sorted(refs_for_libraries): module = info.modules_by_ref[ref] artifact_path = module.artifact self.context.log.debug('Extracting {} to {}.'.format( to_m2(ref), unpack_dir)) if artifact_path.endswith('.zip') or artifact_path.endswith( '.jar'): ZIP.extract(artifact_path, unpack_dir, filter_func=unpack_filter) else: self._extract_tar(artifact_path, unpack_dir, filter_func=unpack_filter)
def execute(self): targets = self.context.targets() jars, global_excludes = IvyUtils.calculate_classpath(targets) filtered_jars = [jar for jar in jars if self._is_update_coordinate(jar.coordinate)] sorted_jars = sorted((jar for jar in filtered_jars), key=lambda x: (x.org, x.name, x.rev, x.classifier)) ivyxml = os.path.join(self.workdir, 'ivy.xml') IvyUtils.generate_ivy(targets, jars=sorted_jars, excludes=global_excludes, ivyxml=ivyxml, confs=['default']) args = [ '-settings', IvySubsystem.global_instance().get_options().ivy_settings, '-ivy', ivyxml, '-confs', ','.join(self.get_options().confs) ] result = self.runjava(classpath=self.tool_classpath('dependency-update-checker'), main=self._IVY_DEPENDENCY_UPDATE_MAIN, jvm_options=self.get_options().jvm_options, args=args, workunit_name='dependency-update-checker', workunit_labels=[WorkUnitLabel.LINT]) self.context.log.debug('java {main} ... exited with result ({result})'.format( main=self._IVY_DEPENDENCY_UPDATE_MAIN, result=result)) return result
def dumped_chroot(self, targets): # TODO(benjy): We shouldn't need to mention DistributionLocator here, as IvySubsystem # declares it as a dependency. However if we don't then test_antlr() below fails on # uninitialized options for that subsystem. Hopefully my pending (as of 9/2016) change # to clean up how we initialize and create instances of subsystems in tests will make # this problem go away. self.context(for_subsystems=[PythonRepos, PythonSetup, IvySubsystem, DistributionLocator, ThriftBinary.Factory, BinaryUtil.Factory]) python_repos = PythonRepos.global_instance() ivy_bootstrapper = Bootstrapper(ivy_subsystem=IvySubsystem.global_instance()) thrift_binary_factory = ThriftBinary.Factory.global_instance().create interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos) interpreter = interpreter_cache.select_interpreter_for_targets(targets) self.assertIsNotNone(interpreter) with temporary_dir() as chroot: pex_builder = PEXBuilder(path=chroot, interpreter=interpreter) python_chroot = PythonChroot(python_setup=self.python_setup, python_repos=python_repos, ivy_bootstrapper=ivy_bootstrapper, thrift_binary_factory=thrift_binary_factory, interpreter=interpreter, builder=pex_builder, targets=targets, platforms=['current']) try: python_chroot.dump() yield pex_builder, python_chroot finally: python_chroot.delete()
def xml_report_path(cls, resolve_hash_name, conf): """The path to the xml report ivy creates after a retrieve. :param string resolve_hash_name: Hash from the Cache key from the VersionedTargetSet used for resolution. :param string conf: the ivy conf name (e.g. "default") """ cachedir = IvySubsystem.global_instance().get_options().cache_dir return os.path.join(cachedir, '{}-{}-{}.xml'.format(IvyUtils.INTERNAL_ORG_NAME, resolve_hash_name, conf))
def test_simple(self): ivy_subsystem = IvySubsystem.global_instance() bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem) ivy = bootstrapper.ivy() self.assertIsNotNone(ivy.ivy_resolution_cache_dir) self.assertIsNone(ivy.ivy_settings) bootstrap_jar_path = os.path.join(ivy_subsystem.get_options().pants_bootstrapdir, 'tools', 'jvm', 'ivy', 'bootstrap.jar') self.assertTrue(os.path.exists(bootstrap_jar_path))
def test_simple(self): ivy_subsystem = IvySubsystem.global_instance() bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem) ivy = bootstrapper.ivy() self.assertIsNotNone(ivy.ivy_cache_dir) self.assertIsNone(ivy.ivy_settings) bootstrap_jar_path = os.path.join( ivy_subsystem.get_options().pants_bootstrapdir, "tools", "jvm", "ivy", "bootstrap.jar" ) self.assertTrue(os.path.exists(bootstrap_jar_path))
def test_parse_proxy_string(self): ivy_subsystem =IvySubsystem.global_instance() self.assertEquals(('example.com', 1234), ivy_subsystem._parse_proxy_string('http://example.com:1234')) self.assertEquals(('secure-example.com', 999), ivy_subsystem._parse_proxy_string('http://secure-example.com:999')) # trailing slash is ok self.assertEquals(('example.com', 1234), ivy_subsystem._parse_proxy_string('http://example.com:1234/'))
def __init__(self, *args, **kwargs): super(IvyResolve, self).__init__(*args, **kwargs) self._cachedir = IvySubsystem.global_instance().get_options().cache_dir self._classpath_dir = os.path.join(self.workdir, 'mapped') self._outdir = self.get_options().outdir or os.path.join(self.workdir, 'reports') self._open = self.get_options().open self._report = self._open or self.get_options().report self._confs = None self._args = [] for arg in self.get_options().args: self._args.extend(safe_shlex_split(arg))
def execute(self): deprecated_conditional( lambda: True, removal_version="1.31.0.dev0", entity_description="The `outdated` goal", hint_message= "Contact the Pants team on Slack or [email protected] " "if you need this functionality.", ) targets = self.context.targets() jars, global_excludes = IvyUtils.calculate_classpath(targets) filtered_jars = [ jar for jar in jars if self._is_update_coordinate(jar.coordinate) ] sorted_jars = sorted((jar for jar in filtered_jars), key=lambda x: (x.org, x.name, x.rev, x.classifier)) ivyxml = os.path.join(self.workdir, "ivy.xml") IvyUtils.generate_ivy(targets, jars=sorted_jars, excludes=global_excludes, ivyxml=ivyxml, confs=["default"]) args = [ "-settings", IvySubsystem.global_instance().get_options().ivy_settings, "-ivy", ivyxml, "-confs", ",".join(self.get_options().confs), ] result = self.runjava( classpath=self.tool_classpath("dependency-update-checker"), main=self._IVY_DEPENDENCY_UPDATE_MAIN, jvm_options=self.get_options().jvm_options, args=args, workunit_name="dependency-update-checker", workunit_labels=[WorkUnitLabel.LINT], ) self.context.log.debug( "java {main} ... exited with result ({result})".format( main=self._IVY_DEPENDENCY_UPDATE_MAIN, result=result)) return result
def __init__(self, *args, **kwargs): super(IvyResolve, self).__init__(*args, **kwargs) self._cachedir = IvySubsystem.global_instance().get_options().cache_dir self._classpath_dir = os.path.join(self.workdir, 'mapped') self._outdir = self.get_options().outdir or os.path.join(self.workdir, 'reports') self._open = self.get_options().open self._report = self._open or self.get_options().report self._confs = None self._args = [] for arg in self.get_options().args: self._args.extend(safe_shlex_split(arg)) # Typically this should be a local cache only, since classpaths aren't portable. self.setup_artifact_cache()
def test_proxy_from_env(self): ivy_subsystem = IvySubsystem.global_instance() self.assertIsNone(ivy_subsystem.http_proxy()) self.assertIsNone(ivy_subsystem.https_proxy()) with environment_as(HTTP_PROXY='http://proxy.example.com:456', HTTPS_PROXY='https://secure-proxy.example.com:789'): self.assertEquals('http://proxy.example.com:456', ivy_subsystem.http_proxy()) self.assertEquals('https://secure-proxy.example.com:789', ivy_subsystem.https_proxy()) self.assertEquals([ '-Dhttp.proxyHost=proxy.example.com', '-Dhttp.proxyPort=456', '-Dhttps.proxyHost=secure-proxy.example.com', '-Dhttps.proxyPort=789', ], ivy_subsystem.extra_jvm_options())
def _unpack(self, unpacked_archives): """Extracts files from the downloaded jar files and places them in a work directory. :param UnpackedArchives unpacked_archives: target referencing jar_libraries to unpack. """ self.context.log.info('Unpacking {}'.format(unpacked_archives.address.spec)) unpack_dir = unpacked_archives.destination safe_mkdir(unpack_dir, clean=True) unpack_filter = self.get_unpack_filter(unpacked_archives) classpath_products = ClasspathProducts(self.get_options().pants_workdir) resolve_hashes = self.resolve(None, unpacked_archives.dependencies, classpath_products) ivy_cache_dir = os.path.expanduser(IvySubsystem.global_instance().get_options().cache_dir) def to_m2(jar): return M2Coordinate(org=jar.org, name=jar.name, rev=jar.rev, classifier=jar.classifier, ext=jar.ext) libraries = self.context.build_graph.transitive_subgraph_of_addresses([unpacked_archives.address]) libraries = [t for t in libraries if isinstance(t, JarLibrary)] coords = set() for library in libraries: coords.update(to_m2(jar) for jar in library.payload.jars) for resolve_hash in resolve_hashes: path = IvyUtils.xml_report_path(ivy_cache_dir, resolve_hash, 'default') info = IvyUtils.parse_xml_report('default', path) refs_for_libraries = set() for ref in info.modules_by_ref.keys(): if to_m2(ref) in coords: refs_for_libraries.add(ref) memo = {} for ref in tuple(refs_for_libraries): info.traverse_dependency_graph(ref, refs_for_libraries.add, memo) for ref in sorted(refs_for_libraries): module = info.modules_by_ref[ref] artifact_path = module.artifact self.context.log.debug('Extracting {} to {}.'.format(to_m2(ref), unpack_dir)) if artifact_path.endswith('.zip') or artifact_path.endswith('.jar'): ZIP.extract(artifact_path, unpack_dir, filter_func=unpack_filter) else: self._extract_tar(artifact_path, unpack_dir, filter_func=unpack_filter)
def test_simple(self): ivy_subsystem = IvySubsystem.global_instance() bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem) ivy = bootstrapper.ivy() self.assertIsNotNone(ivy.ivy_resolution_cache_dir) self.assertIsNone(ivy.ivy_settings)
def subsystem_dependencies(cls): return super(IvyOutdated, cls).subsystem_dependencies() + (IvySubsystem.scoped(cls),)
def subsystem_dependencies(cls): return super().subsystem_dependencies() + (IvySubsystem.scoped(cls),)
def ivy_resolve(self, targets, executor=None, silent=False, workunit_name=None, confs=None, custom_args=None): """Executes an ivy resolve for the relevant subset of the given targets. Returns the resulting classpath, and the set of relevant targets. Also populates the 'ivy_resolve_symlink_map' product for jars resulting from the resolve.""" if not targets: return ([], set()) # NOTE: Always pass all the targets to exec_ivy, as they're used to calculate the name of # the generated module, which in turn determines the location of the XML report file # ivy generates. We recompute this name from targets later in order to find that file. # TODO: This is fragile. Refactor so that we're not computing the name twice. ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit) ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy') fingerprint_strategy = IvyResolveFingerprintStrategy(confs) with self.invalidated(targets, invalidate_dependents=False, silent=silent, fingerprint_strategy=fingerprint_strategy) as invalidation_check: if not invalidation_check.all_vts: return ([], set()) global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts) # If a report file is not present, we need to exec ivy, even if all the individual # targets up to date... See https://rbcommons.com/s/twitter/r/2015 report_missing = False report_confs = confs or ['default'] report_paths = [] for conf in report_confs: report_path = IvyUtils.xml_report_path(global_vts.targets, conf) if not os.path.exists(report_path): report_missing = True break else: report_paths.append(report_path) target_workdir = os.path.join(ivy_workdir, global_vts.cache_key.hash) target_classpath_file = os.path.join(target_workdir, 'classpath') raw_target_classpath_file = target_classpath_file + '.raw' raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp' # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars # in artifact-cached analysis files are consistent across systems. # Note that we have one global, well-known symlink dir, again so that paths are # consistent across builds. symlink_dir = os.path.join(ivy_workdir, 'jars') # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously built a superset of targets. if report_missing or invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file): args = ['-cachepath', raw_target_classpath_file_tmp] + (custom_args if custom_args else []) self.exec_ivy( target_workdir=target_workdir, targets=global_vts.targets, args=args, executor=executor, ivy=ivy, workunit_name=workunit_name, confs=confs, use_soft_excludes=self.get_options().soft_excludes) if not os.path.exists(raw_target_classpath_file_tmp): raise TaskError('Ivy failed to create classpath file at {}' .format(raw_target_classpath_file_tmp)) shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file) logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file)) if self.artifact_cache_writes_enabled(): self.update_artifact_cache([(global_vts, [raw_target_classpath_file])]) else: logger.debug("Using previously resolved reports: {}".format(report_paths)) # Make our actual classpath be symlinks, so that the paths are uniform across systems. # Note that we must do this even if we read the raw_target_classpath_file from the artifact # cache. If we cache the target_classpath_file we won't know how to create the symlinks. with IvyTaskMixin.symlink_map_lock: products = self.context.products existing_symlinks_map = products.get_data('ivy_resolve_symlink_map', lambda: dict()) symlink_map = IvyUtils.symlink_cachepath( IvySubsystem.global_instance().get_options().cache_dir, raw_target_classpath_file, symlink_dir, target_classpath_file, existing_symlinks_map) existing_symlinks_map.update(symlink_map) with IvyUtils.cachepath(target_classpath_file) as classpath: stripped_classpath = [path.strip() for path in classpath] return (stripped_classpath, global_vts.targets)
def ivy_resolve(self, targets, executor=None, silent=False, workunit_name=None, confs=None, custom_args=None): """Executes an ivy resolve for the relevant subset of the given targets. :returns: the resulting classpath, and the unique part of the name used for the resolution report (a hash). Also populates the 'ivy_resolve_symlink_map' product for jars resulting from the resolve.""" if not targets: return ([], None) ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit) ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy') fingerprint_strategy = IvyResolveFingerprintStrategy(confs) with self.invalidated(targets, invalidate_dependents=False, silent=silent, fingerprint_strategy=fingerprint_strategy) as invalidation_check: if not invalidation_check.all_vts: return ([], None) global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts) # If a report file is not present, we need to exec ivy, even if all the individual # targets up to date... See https://rbcommons.com/s/twitter/r/2015 report_missing = False report_confs = confs or ['default'] report_paths = [] resolve_hash_name = global_vts.cache_key.hash for conf in report_confs: report_path = IvyUtils.xml_report_path(resolve_hash_name, conf) if not os.path.exists(report_path): report_missing = True break else: report_paths.append(report_path) target_workdir = os.path.join(ivy_workdir, resolve_hash_name) target_classpath_file = os.path.join(target_workdir, 'classpath') raw_target_classpath_file = target_classpath_file + '.raw' raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp' # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars # in artifact-cached analysis files are consistent across systems. # Note that we have one global, well-known symlink dir, again so that paths are # consistent across builds. symlink_dir = os.path.join(ivy_workdir, 'jars') # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously built a superset of targets. if report_missing or invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file): args = ['-cachepath', raw_target_classpath_file_tmp] + (custom_args if custom_args else []) self.exec_ivy( target_workdir=target_workdir, targets=global_vts.targets, args=args, executor=executor, ivy=ivy, workunit_name=workunit_name, confs=confs, use_soft_excludes=self.get_options().soft_excludes, resolve_hash_name=resolve_hash_name) if not os.path.exists(raw_target_classpath_file_tmp): raise TaskError('Ivy failed to create classpath file at {}' .format(raw_target_classpath_file_tmp)) shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file) logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file)) if self.artifact_cache_writes_enabled(): self.update_artifact_cache([(global_vts, [raw_target_classpath_file])]) else: logger.debug("Using previously resolved reports: {}".format(report_paths)) # Make our actual classpath be symlinks, so that the paths are uniform across systems. # Note that we must do this even if we read the raw_target_classpath_file from the artifact # cache. If we cache the target_classpath_file we won't know how to create the symlinks. with IvyTaskMixin.symlink_map_lock: products = self.context.products existing_symlinks_map = products.get_data('ivy_resolve_symlink_map', lambda: dict()) symlink_map = IvyUtils.symlink_cachepath( IvySubsystem.global_instance().get_options().cache_dir, raw_target_classpath_file, symlink_dir, target_classpath_file, existing_symlinks_map) existing_symlinks_map.update(symlink_map) with IvyUtils.cachepath(target_classpath_file) as classpath: stripped_classpath = [path.strip() for path in classpath] return (stripped_classpath, resolve_hash_name)
def __init__(self, ivy_subsystem=None): """Creates an ivy bootstrapper.""" self._ivy_subsystem = ivy_subsystem or IvySubsystem.global_instance() self._version_or_ivyxml = self._ivy_subsystem.get_options().ivy_profile self._classpath = None
def ivy_bootstrapper(self): return Bootstrapper(ivy_subsystem=IvySubsystem.global_instance())
def xml_report_path(cls, targets, conf): """The path to the xml report ivy creates after a retrieve.""" org, name = cls.identify(targets) cachedir = IvySubsystem.global_instance().get_options().cache_dir return os.path.join(cachedir, '{}-{}-{}.xml'.format(org, name, conf))
def ivy_repository_cache_dir(self): return IvySubsystem.global_instance().repository_cache_dir()
def ivy_resolution_cache_dir(self): return IvySubsystem.global_instance().resolution_cache_dir()