def execute( self, jvm_options=None, args=None, executor=None, workunit_factory=None, workunit_name=None, workunit_labels=None, ): """Executes the ivy commandline client with the given args. Raises Ivy.Error if the command fails for any reason. :param executor: Java executor to run ivy with. """ # NB(gmalmquist): It should be OK that we can't declare a subsystem_dependency in this file # (because it's just a plain old object), because Ivy is only constructed by Bootstrapper, which # makes an explicit call to IvySubsystem.global_instance() in its constructor, which in turn has # a declared dependency on DistributionLocator. executor = executor or SubprocessExecutor(DistributionLocator.cached()) runner = self.runner(jvm_options=jvm_options, args=args, executor=executor) try: with self.resolution_lock(): result = util.execute_runner(runner, workunit_factory, workunit_name, workunit_labels) if result != 0: raise self.Error( "Ivy command failed with exit code {}{}".format( result, ": " + " ".join(args) if args else "")) except executor.Error as e: raise self.Error("Problem executing ivy: {}".format(e))
def exec_ivy(cls, ivy, confs, ivyxml, args, jvm_options, executor, workunit_name, workunit_factory): ivy = ivy or Bootstrapper.default_ivy() ivy_args = ['-ivy', ivyxml] ivy_args.append('-confs') ivy_args.extend(confs) ivy_args.extend(args) ivy_jvm_options = list(jvm_options) # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner(runner, workunit_factory=workunit_factory, workunit_name=workunit_name) if result != 0: raise IvyUtils.IvyError( 'Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise IvyUtils.IvyError(e)
def shade_jar(self, shading_rules, jar_path): """Shades a jar using the shading rules from the given jvm_binary. This *overwrites* the existing jar file at ``jar_path``. :param shading_rules: predefined rules for shading :param jar_path: The filepath to the jar that should be shaded. """ self.context.log.debug('Shading {}.'.format(jar_path)) with temporary_dir() as tempdir: output_jar = os.path.join(tempdir, os.path.basename(jar_path)) with self.shader.binary_shader_for_rules( output_jar, jar_path, shading_rules) as shade_runner: result = execute_runner( shade_runner, workunit_factory=self.context.new_workunit, workunit_name='jarjar') if result != 0: raise TaskError( 'Shading tool failed to shade {0} (error code {1})'. format(jar_path, result)) if not os.path.exists(output_jar): raise TaskError( 'Shading tool returned success for {0}, but ' 'the output jar was not found at {1}'.format( jar_path, output_jar)) atomic_copy(output_jar, jar_path) return jar_path
def execute( self, jvm_options=None, args=None, executor=None, workunit_factory=None, workunit_name=None, workunit_labels=None, ): """Executes the ivy commandline client with the given args. Raises Ivy.Error if the command fails for any reason. :param executor: Java executor to run ivy with. """ # NB(gmalmquist): It should be OK that we can't declare a subsystem_dependency in this file # (because it's just a plain old object), because Ivy is only constructed by Bootstrapper, which # makes an explicit call to IvySubsystem.global_instance() in its constructor, which in turn has # a declared dependency on DistributionLocator. executor = executor or SubprocessExecutor(DistributionLocator.cached()) runner = self.runner(jvm_options=jvm_options, args=args, executor=executor) try: with self.resolution_lock: result = util.execute_runner(runner, workunit_factory, workunit_name, workunit_labels) if result != 0: raise self.Error( "Ivy command failed with exit code {}{}".format(result, ": " + " ".join(args) if args else "") ) except executor.Error as e: raise self.Error("Problem executing ivy: {}".format(e))
def exec_ivy(cls, ivy, confs, ivyxml, args, jvm_options, executor, workunit_name, workunit_factory): """ :API: public """ ivy = ivy or Bootstrapper.default_ivy() ivy_args = ['-ivy', ivyxml] ivy_args.append('-confs') ivy_args.extend(confs) ivy_args.extend(args) ivy_jvm_options = list(jvm_options) # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: with ivy.resolution_lock: result = execute_runner(runner, workunit_factory=workunit_factory, workunit_name=workunit_name) if result != 0: raise IvyUtils.IvyError('Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise IvyUtils.IvyError(e)
def exec_ivy(self, target_workdir, targets, args, confs=None, ivy=None, workunit_name='ivy', workunit_factory=None, symlink_ivyxml=False, jars=None): ivy = ivy or Bootstrapper.default_ivy() if not isinstance(ivy, Ivy): raise ValueError('The ivy argument supplied must be an Ivy instance, given %s of type %s' % (ivy, type(ivy))) ivyxml = os.path.join(target_workdir, 'ivy.xml') if not jars: jars, excludes = self._calculate_classpath(targets) else: excludes = set() ivy_args = ['-ivy', ivyxml] confs_to_resolve = confs or ['default'] ivy_args.append('-confs') ivy_args.extend(confs_to_resolve) ivy_args.extend(args) if not self._transitive: ivy_args.append('-notransitive') ivy_args.extend(self._args) def safe_link(src, dest): try: os.unlink(dest) except OSError as e: if e.errno != errno.ENOENT: raise os.symlink(src, dest) with IvyUtils.ivy_lock: self._generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve) runner = ivy.runner(jvm_options=self._jvm_options, args=ivy_args) try: result = util.execute_runner(runner, workunit_factory=workunit_factory, workunit_name=workunit_name) # Symlink to the current ivy.xml file (useful for IDEs that read it). if symlink_ivyxml: ivyxml_symlink = os.path.join(self._workdir, 'ivy.xml') safe_link(ivyxml, ivyxml_symlink) if result != 0: raise TaskError('Ivy returned %d' % result) except runner.executor.Error as e: raise TaskError(e)
def _bootstrap_shaded_jvm_tool(self, key, scope, tools, main, custom_rules=None): shaded_jar = os.path.join(self._tool_cache_path, 'shaded_jars', scope, key, '{}.jar'.format(main)) targets = list(self._resolve_tool_targets(tools, key, scope)) fingerprint_strategy = ShadedToolFingerprintStrategy(key, scope, main, custom_rules=custom_rules) with self.invalidated(targets, # We're the only dependent in reality since we shade. invalidate_dependents=False, fingerprint_strategy=fingerprint_strategy) as invalidation_check: if not invalidation_check.invalid_vts and os.path.exists(shaded_jar): return [shaded_jar] # Ensure we have a single binary jar we can shade. binary_jar = os.path.join(self._tool_cache_path, 'binary_jars', scope, key, '{}.jar'.format(main)) safe_mkdir_for(binary_jar) classpath = self._bootstrap_classpath(key, targets) if len(classpath) == 1: shutil.copy(classpath[0], binary_jar) else: with self.open_jar(binary_jar) as jar: for classpath_jar in classpath: jar.writejar(classpath_jar) jar.main(main) # Now shade the binary jar and return that single jar as the safe tool classpath. safe_mkdir_for(shaded_jar) with self.shader.binary_shader(shaded_jar, main, binary_jar, custom_rules=custom_rules, jvm_options=self.get_options().jvm_options) as shader: try: result = util.execute_runner(shader, workunit_factory=self.context.new_workunit, workunit_name='shade-{}'.format(key)) if result != 0: raise TaskError("Shading of tool '{key}' with main class {main} for {scope} failed " "with exit code {result}, command run was:\n\t{cmd}" .format(key=key, main=main, scope=scope, result=result, cmd=shader.cmd)) except Executor.Error as e: raise TaskError("Shading of tool '{key}' with main class {main} for {scope} failed " "with: {exception}".format(key=key, main=main, scope=scope, exception=e)) if self.artifact_cache_writes_enabled(): tool_vts = self.tool_vts(invalidation_check) self.update_artifact_cache([(tool_vts, [shaded_jar])]) return [shaded_jar]
def execute(self, jvm_options=None, args=None, executor=None, workunit_factory=None, workunit_name=None, workunit_labels=None): """Executes the ivy commandline client with the given args. Raises Ivy.Error if the command fails for any reason. """ runner = self.runner(jvm_options=jvm_options, args=args, executor=executor) try: result = util.execute_runner(runner, workunit_factory, workunit_name, workunit_labels) if result != 0: raise self.Error('Ivy command failed with exit code %d%s' % (result, ': ' + ' '.join(args) if args else '')) except self._java.Error as e: raise self.Error('Problem executing ivy: %s' % e)
def _exec_ivy(self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name='ivy', use_soft_excludes=False, resolve_hash_name=None): ivy_jvm_options = self.get_options().jvm_options[:] # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, 'ivy.xml') ivy_args = ['-ivy', ivyxml] confs_to_resolve = confs or ('default', ) ivy_args.append('-confs') ivy_args.extend(confs_to_resolve) ivy_args.extend(args) # TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better # diagnostics can be had in `IvyUtils.generate_ivy` if this is done. # See: https://github.com/pantsbuild/pants/issues/2239 try: jars, excludes = IvyUtils.calculate_classpath( targets, gather_excludes=not use_soft_excludes) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner( runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name) if result != 0: raise self.Error( 'Ivy returned {result}. cmd={cmd}'.format( result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise self.Error(e) except IvyUtils.IvyError as e: raise self.Error('Failed to prepare ivy resolve: {}'.format(e))
def _call_coursier(self, cmd_args, coursier_jar, output_fn, executor): runner = executor.runner( classpath=[coursier_jar], main='coursier.cli.Coursier', jvm_options=self.get_options().jvm_options, args=cmd_args) labels = [WorkUnitLabel.COMPILER] if self.get_options().report else [WorkUnitLabel.TOOL] return_code = util.execute_runner(runner, self.context.new_workunit, 'coursier', labels) if return_code: raise TaskError('The coursier process exited non-zero: {0}'.format(return_code)) with open(output_fn, 'r') as f: return json.loads(f.read())
def exec_ivy(self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name='ivy', jars=None, use_soft_excludes=False, resolve_hash_name=None): ivy_jvm_options = copy.copy(self.get_options().jvm_options) # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, 'ivy.xml') if not jars: jars, excludes = IvyUtils.calculate_classpath( targets, gather_excludes=not use_soft_excludes) else: excludes = set() ivy_args = ['-ivy', ivyxml] confs_to_resolve = confs or ['default'] ivy_args.append('-confs') ivy_args.extend(confs_to_resolve) ivy_args.extend(args) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner( runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name) if result != 0: raise TaskError('Ivy returned {result}. cmd={cmd}'.format( result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise TaskError(e)
def _exec_ivy( self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name="ivy", use_soft_excludes=False, resolve_hash_name=None, ): ivy_jvm_options = self.get_options().jvm_options[:] # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append("-Dsun.io.useCanonCaches=false") ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, "ivy.xml") ivy_args = ["-ivy", ivyxml] confs_to_resolve = confs or ("default",) ivy_args.append("-confs") ivy_args.extend(confs_to_resolve) ivy_args.extend(args) # TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better # diagnostics can be had in `IvyUtils.generate_ivy` if this is done. # See: https://github.com/pantsbuild/pants/issues/2239 try: jars, excludes = IvyUtils.calculate_classpath(targets, gather_excludes=not use_soft_excludes) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner( runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name ) if result != 0: raise self.Error("Ivy returned {result}. cmd={cmd}".format(result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise self.Error(e) except IvyUtils.IvyError as e: raise self.Error("Failed to prepare ivy resolve: {}".format(e))
def exec_ivy(self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name='ivy', jars=None, use_soft_excludes=False, resolve_hash_name=None): ivy_jvm_options = copy.copy(self.get_options().jvm_options) # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, 'ivy.xml') if not jars: automatic_excludes = self.get_options().automatic_excludes jars, excludes = IvyUtils.calculate_classpath(targets, gather_excludes=not use_soft_excludes, automatic_excludes=automatic_excludes) else: excludes = set() ivy_args = ['-ivy', ivyxml] confs_to_resolve = confs or ['default'] ivy_args.append('-confs') ivy_args.extend(confs_to_resolve) ivy_args.extend(args) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner(runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name) if result != 0: raise TaskError('Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise TaskError(e)
def shade_jar(self, shading_rules, jar_path): """Shades a jar using the shading rules from the given jvm_binary. This *overwrites* the existing jar file at ``jar_path``. :param shading_rules: predefined rules for shading :param jar_path: The filepath to the jar that should be shaded. """ self.context.log.debug('Shading {}.'.format(jar_path)) with temporary_dir() as tempdir: output_jar = os.path.join(tempdir, os.path.basename(jar_path)) with self.shader.binary_shader_for_rules(output_jar, jar_path, shading_rules) as shade_runner: result = execute_runner(shade_runner, workunit_factory=self.context.new_workunit, workunit_name='jarjar') if result != 0: raise TaskError('Shading tool failed to shade {0} (error code {1})'.format(jar_path, result)) if not os.path.exists(output_jar): raise TaskError('Shading tool returned success for {0}, but ' 'the output jar was not found at {1}'.format(jar_path, output_jar)) atomic_copy(output_jar, jar_path) return jar_path
def _bootstrap_shaded_jvm_tool(self, jvm_tool, targets): fingerprint_strategy = ShadedToolFingerprintStrategy( jvm_tool.main, custom_rules=jvm_tool.custom_rules) with self.invalidated( targets, # We're the only dependent in reality since we shade. invalidate_dependents=False, fingerprint_strategy=fingerprint_strategy ) as invalidation_check: # If there are no vts, then there are no resolvable targets, so we exit early with an empty # classpath. This supports the optional tool classpath case. if not invalidation_check.all_vts: return [] tool_vts = self.tool_vts(invalidation_check) jar_name = '{main}-{hash}.jar'.format(main=jvm_tool.main, hash=tool_vts.cache_key.hash) shaded_jar = os.path.join(self._tool_cache_path, 'shaded_jars', jar_name) if not invalidation_check.invalid_vts and os.path.exists( shaded_jar): return [shaded_jar] # Ensure we have a single binary jar we can shade. binary_jar = os.path.join(self._tool_cache_path, 'binary_jars', jar_name) safe_mkdir_for(binary_jar) classpath = self._bootstrap_classpath(jvm_tool, targets) if len(classpath) == 1: shutil.copy(classpath[0], binary_jar) else: with self.open_jar(binary_jar) as jar: for classpath_jar in classpath: jar.writejar(classpath_jar) jar.main(jvm_tool.main) # Now shade the binary jar and return that single jar as the safe tool classpath. safe_mkdir_for(shaded_jar) with self.shader.binary_shader( shaded_jar, jvm_tool.main, binary_jar, custom_rules=jvm_tool.custom_rules, jvm_options=self.get_options().jvm_options) as shader: try: result = util.execute_runner( shader, workunit_factory=self.context.new_workunit, workunit_name='shade-{}'.format(jvm_tool.key)) if result != 0: raise TaskError( "Shading of tool '{key}' with main class {main} for {scope} failed " "with exit code {result}, command run was:\n\t{cmd}" .format(key=jvm_tool.key, main=jvm_tool.main, scope=jvm_tool.scope, result=result, cmd=shader.cmd)) except Executor.Error as e: raise TaskError( "Shading of tool '{key}' with main class {main} for {scope} failed " "with: {exception}".format(key=jvm_tool.key, main=jvm_tool.main, scope=jvm_tool.scope, exception=e)) if self.artifact_cache_writes_enabled(): self.update_artifact_cache([(tool_vts, [shaded_jar])]) return [shaded_jar]
def _bootstrap_shaded_jvm_tool(self, jvm_tool, targets): fingerprint_strategy = ShadedToolFingerprintStrategy(jvm_tool.main, custom_rules=jvm_tool.custom_rules) with self.invalidated(targets, # We're the only dependent in reality since we shade. invalidate_dependents=False, fingerprint_strategy=fingerprint_strategy) as invalidation_check: # If there are no vts, then there are no resolvable targets, so we exit early with an empty # classpath. This supports the optional tool classpath case. if not invalidation_check.all_vts: return [] tool_vts = self.tool_vts(invalidation_check) jar_name = '{main}-{hash}.jar'.format(main=jvm_tool.main, hash=tool_vts.cache_key.hash) shaded_jar = os.path.join(self._tool_cache_path, 'shaded_jars', jar_name) if not invalidation_check.invalid_vts and os.path.exists(shaded_jar): return [shaded_jar] # Ensure we have a single binary jar we can shade. binary_jar = os.path.join(self._tool_cache_path, 'binary_jars', jar_name) safe_mkdir_for(binary_jar) classpath = self._bootstrap_classpath(jvm_tool, targets) if len(classpath) == 1: shutil.copy(classpath[0], binary_jar) else: with self.open_jar(binary_jar) as jar: for classpath_jar in classpath: jar.writejar(classpath_jar) jar.main(jvm_tool.main) # Now shade the binary jar and return that single jar as the safe tool classpath. safe_mkdir_for(shaded_jar) with self.shader.binary_shader(shaded_jar, jvm_tool.main, binary_jar, custom_rules=jvm_tool.custom_rules, jvm_options=self.get_options().jvm_options) as shader: try: result = util.execute_runner(shader, workunit_factory=self.context.new_workunit, workunit_name='shade-{}'.format(jvm_tool.key)) if result != 0: raise TaskError("Shading of tool '{key}' with main class {main} for {scope} failed " "with exit code {result}, command run was:\n\t{cmd}" .format(key=jvm_tool.key, main=jvm_tool.main, scope=jvm_tool.scope, result=result, cmd=shader.cmd)) except Executor.Error as e: raise TaskError("Shading of tool '{key}' with main class {main} for {scope} failed " "with: {exception}".format(key=jvm_tool.key, main=jvm_tool.main, scope=jvm_tool.scope, exception=e)) if self.artifact_cache_writes_enabled(): self.update_artifact_cache([(tool_vts, [shaded_jar])]) return [shaded_jar]