def test_blows_up_on_invalid_args(self):
try:
self._default_args_execute_process_request()
except ValueError:
self.assertTrue(False, "should be able to construct without error")
with self.assertRaises(ValueError):
self._default_args_execute_process_request(argv=['1'])
with self.assertRaises(ValueError):
self._default_args_execute_process_request(argv=('1', ), env=[])
# TODO(cosmicexplorer): we should probably check that the digest info in
# ExecuteProcessRequest is valid, beyond just checking if it's a string.
with self.assertRaises(ValueError):
ExecuteProcessRequest(argv=('1', ),
env=tuple(),
input_files_digest='',
digest_length='')
with self.assertRaises(ValueError):
ExecuteProcessRequest(argv=('1', ),
env=tuple(),
input_files_digest=3,
digest_length=0)
with self.assertRaises(ValueError):
ExecuteProcessRequest(argv=('1', ),
env=tuple(),
input_files_digest='',
digest_length=-1)
def test_blows_up_on_invalid_args(self):
try:
self._default_args_execute_process_request()
except ValueError:
self.assertTrue(False, "should be able to construct without error")
with self.assertRaises(TypeCheckError):
self._default_args_execute_process_request(argv=['1'])
with self.assertRaises(TypeCheckError):
self._default_args_execute_process_request(argv=('1', ),
env=['foo', 'bar'])
# TODO(cosmicexplorer): we should probably check that the digest info in
# ExecuteProcessRequest is valid, beyond just checking if it's a string.
with self.assertRaises(TypeCheckError):
ExecuteProcessRequest(argv=('1', ),
env=dict(),
input_files='',
output_files=())
with self.assertRaises(TypeCheckError):
ExecuteProcessRequest(argv=('1', ),
env=dict(),
input_files=3,
output_files=())
with self.assertRaises(TypeCheckError):
ExecuteProcessRequest(argv=('1', ),
env=tuple(),
input_files=EMPTY_DIRECTORY_DIGEST,
output_files=["blah"])
def test_blows_up_on_invalid_args(self):
try:
self._default_args_execute_process_request()
except ValueError:
self.assertTrue(False, "should be able to construct without error")
with self.assertRaises(TypeCheckError):
self._default_args_execute_process_request(argv=['1'])
with self.assertRaises(TypeCheckError):
self._default_args_execute_process_request(argv=('1',), env=['foo', 'bar'])
with self.assertRaisesRegexp(TypeCheckError, "env"):
ExecuteProcessRequest(
argv=('1',),
env=(),
input_files='',
output_files=(),
output_directories=(),
timeout_seconds=0.1,
description=''
)
with self.assertRaisesRegexp(TypeCheckError, "input_files"):
ExecuteProcessRequest(argv=('1',),
env=dict(),
input_files=3,
output_files=(),
output_directories=(),
timeout_seconds=0.1,
description=''
)
with self.assertRaisesRegexp(TypeCheckError, "output_files"):
ExecuteProcessRequest(
argv=('1',),
env=dict(),
input_files=EMPTY_DIRECTORY_DIGEST,
output_files=("blah"),
output_directories=(),
timeout_seconds=0.1,
description=''
)
with self.assertRaisesRegexp(TypeCheckError, "timeout"):
ExecuteProcessRequest(
argv=('1',),
env=dict(),
input_files=EMPTY_DIRECTORY_DIGEST,
output_files=("blah"),
output_directories=(),
timeout_seconds=None,
description=''
)
def _execute_hermetic_compile(self, cmd, ctx):
# For now, executing a compile remotely only works for targets that
# do not have any dependencies or inner classes
input_snapshot = ctx.target.sources_snapshot(scheduler=self.context._scheduler)
output_files = tuple(
# Assume no extra .class files to grab. We'll fix up that case soon.
# Drop the source_root from the file path.
# Assumes `-d .` has been put in the command.
os.path.relpath(f.path.replace('.java', '.class'), ctx.target.target_base)
for f in input_snapshot.files if f.path.endswith('.java')
)
exec_process_request = ExecuteProcessRequest(
argv=tuple(cmd),
input_files=input_snapshot.directory_digest,
output_files=output_files,
description='Compiling {} with javac'.format(ctx.target.address.spec),
)
exec_result = self.context.execute_process_synchronously(
exec_process_request,
'javac',
(WorkUnitLabel.TASK, WorkUnitLabel.JVM),
)
# Dump the output to the .pants.d directory where it's expected by downstream tasks.
classes_directory = ctx.classes_dir
self.context._scheduler.materialize_directories((
DirectoryToMaterialize(text_type(classes_directory), exec_result.output_directory_digest),
))
def process_request_java_args_from_java_sources(binary, sources_snapshot, out_dir):
env = []
return ExecuteProcessRequest(
argv=binary.prefix_of_command() + tuple(f.path for f in sources_snapshot.files),
input_files_digest=str(sources_snapshot.fingerprint),
digest_length=sources_snapshot.digest_length,
env=env)
def process_request_from_java_sources(binary):
env = []
return ExecuteProcessRequest(
argv=binary.prefix_of_command() + ('-version',),
env=env,
input_files_digest="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
digest_length=0)
def _run_bootstrapper(self, bridge_jar, context):
bootstrapper = self._relative_to_buildroot(
self._zinc_factory._compiler_bootstrapper(self._products), )
bootstrapper_args = [
'--out',
self._relative_to_buildroot(bridge_jar),
'--compiler-interface',
self._relative_to_buildroot(self.compiler_interface),
'--compiler-bridge-src',
self._relative_to_buildroot(self.compiler_bridge),
'--scala-compiler',
self._relative_to_buildroot(self.scala_compiler),
'--scala-library',
self._relative_to_buildroot(self.scala_library),
'--scala-reflect',
self._relative_to_buildroot(self.scala_reflect),
]
input_jar_snapshots = context._scheduler.capture_snapshots(
(PathGlobsAndRoot(
PathGlobs(tuple([bootstrapper] + bootstrapper_args[1::2])),
text_type(get_buildroot()),
), ))
argv = tuple(['.jdk/bin/java'] +
['-cp', bootstrapper, Zinc.ZINC_BOOTSTRAPER_MAIN] +
bootstrapper_args)
req = ExecuteProcessRequest(
argv=argv,
input_files=input_jar_snapshots[0].directory_digest,
output_files=(self._relative_to_buildroot(bridge_jar), ),
description='bootstrap compiler bridge.',
# Since this is always hermetic, we need to use `underlying_dist`
jdk_home=self.underlying_dist.home,
)
return context.execute_process_synchronously_or_raise(
req, 'zinc-subsystem', [WorkUnitLabel.COMPILER])
async def javac_compile_process_result(
javac_compile_req: JavacCompileRequest) -> JavacCompileResult:
java_files = javac_compile_req.javac_sources.java_files
for java_file in java_files:
if not java_file.endswith(".java"):
raise ValueError(
f"Can only compile .java files but got {java_file}")
sources_snapshot = await Get(Snapshot, PathGlobs,
PathGlobs(java_files, ()))
output_dirs = tuple(
{os.path.dirname(java_file)
for java_file in java_files})
process_request = ExecuteProcessRequest(
argv=javac_compile_req.argv_from_source_snapshot(sources_snapshot),
input_files=sources_snapshot.directory_digest,
output_directories=output_dirs,
description='javac compilation')
javac_proc_result = await Get(ExecuteProcessResult, ExecuteProcessRequest,
process_request)
return JavacCompileResult(
javac_proc_result.stdout.decode(),
javac_proc_result.stderr.decode(),
javac_proc_result.output_directory_digest,
)
def test_write_file(self):
request = ExecuteProcessRequest(
argv=("/bin/bash", "-c", "echo -n 'European Burmese' > roland"),
description="echo roland",
output_files=("roland", ),
input_files=EMPTY_DIRECTORY_DIGEST,
)
execute_process_result = self.request_single_product(
ExecuteProcessResult, request)
self.assertEqual(
execute_process_result.output_directory_digest,
Digest(
fingerprint=
"63949aa823baf765eff07b946050d76ec0033144c785a94d3ebd82baa931cd16",
serialized_bytes_length=80,
))
files_content_result = self.request_single_product(
FilesContent,
execute_process_result.output_directory_digest,
)
self.assertEqual(files_content_result.dependencies,
(FileContent("roland", b"European Burmese", False), ))
def test_generic_pex_creation(self) -> None:
input_files_content = InputFilesContent((
FileContent(path='main.py', content=b'print("from main")'),
FileContent(path='subdir/sub.py', content=b'print("from sub")'),
))
input_files = self.request_single_product(Digest, input_files_content)
pex_output = self.create_pex_and_get_all_data(entry_point='main',
input_files=input_files)
pex_files = pex_output['files']
self.assertTrue('pex' not in pex_files)
self.assertTrue('main.py' in pex_files)
self.assertTrue('subdir/sub.py' in pex_files)
python_setup = PythonSetup.global_instance()
env = {
"PATH": create_path_env_var(python_setup.interpreter_search_paths)
}
pex = pex_output['pex']
req = ExecuteProcessRequest(
argv=('python', 'test.pex'),
env=env,
input_files=pex.directory_digest,
description="Run the pex and make sure it works")
result = self.request_single_product(ExecuteProcessResult, req)
self.assertEqual(result.stdout, b"from main\n")
def test_pex_execution(self) -> None:
input_files_content = InputFilesContent(
(
FileContent(path="main.py", content=b'print("from main")'),
FileContent(path="subdir/sub.py", content=b'print("from sub")'),
)
)
input_files = self.request_single_product(Digest, input_files_content)
pex_output = self.create_pex_and_get_all_data(entry_point="main", input_files=input_files)
pex_files = pex_output["files"]
self.assertTrue("pex" not in pex_files)
self.assertTrue("main.py" in pex_files)
self.assertTrue("subdir/sub.py" in pex_files)
init_subsystem(PythonSetup)
python_setup = PythonSetup.global_instance()
env = {"PATH": create_path_env_var(python_setup.interpreter_search_paths)}
req = ExecuteProcessRequest(
argv=("python", "test.pex"),
env=env,
input_files=pex_output["pex"].directory_digest,
description="Run the pex and make sure it works",
)
result = self.request_single_product(ExecuteProcessResult, req)
self.assertEqual(result.stdout, b"from main\n")
def javac_compile_process_result(javac_compile_req):
java_files = javac_compile_req.javac_sources.java_files
for java_file in java_files:
if not java_file.endswith(".java"):
raise ValueError(
"Can only compile .java files but got {}".format(java_file))
sources_snapshot = yield Get(Snapshot, PathGlobs,
PathGlobs(java_files, ()))
output_dirs = tuple(
{os.path.dirname(java_file)
for java_file in java_files})
process_request = ExecuteProcessRequest(
argv=javac_compile_req.argv_from_source_snapshot(sources_snapshot),
input_files=sources_snapshot.directory_digest,
output_directories=output_dirs,
description='javac compilation')
javac_proc_result = yield Get(ExecuteProcessResult, ExecuteProcessRequest,
process_request)
stdout = javac_proc_result.stdout
stderr = javac_proc_result.stderr
yield JavacCompileResult(
text_type(stdout),
text_type(stderr),
javac_proc_result.output_directory_digest,
)
def console_output(self, targets):
if not self.get_options().transitive:
targets = self.context.target_roots
input_snapshots = tuple(
target.sources_snapshot(scheduler=self.context._scheduler)
for target in targets)
input_files = set(f.path for snapshot in input_snapshots
for f in snapshot.files)
# TODO: Work out a nice library-like utility for writing an argfile, as this will be common.
with temporary_dir() as tmpdir:
list_file = os.path.join(tmpdir, 'input_files_list')
with open(list_file, 'w') as list_file_out:
for input_file in sorted(input_files):
list_file_out.write(input_file)
list_file_out.write(b'\n')
list_file_snapshot = self.context._scheduler.capture_snapshots(
(PathGlobsAndRoot(
PathGlobs(('input_files_list', )),
str(tmpdir),
), ))[0]
cloc_path, cloc_snapshot = ClocBinary.global_instance(
).hackily_snapshot(self.context)
directory_digest = self.context._scheduler.merge_directories(
tuple(s.directory_digest for s in input_snapshots + (
cloc_snapshot,
list_file_snapshot,
)))
cmd = (
'/usr/bin/perl',
cloc_path,
'--skip-uniqueness',
'--ignored=ignored',
'--list-file=input_files_list',
'--report-file=report',
)
# The cloc script reaches into $PATH to look up perl. Let's assume it's in /usr/bin.
req = ExecuteProcessRequest(cmd, (), directory_digest,
('ignored', 'report'), (), 15 * 60, 'cloc')
exec_result = self.context.execute_process_synchronously(
req, 'cloc', (WorkUnitLabel.TOOL, ))
files_content_tuple = self.context._scheduler.product_request(
FilesContent,
[exec_result.output_directory_digest])[0].dependencies
files_content = {fc.path: fc.content for fc in files_content_tuple}
for line in files_content['report'].split('\n'):
yield line
if self.get_options().ignored:
yield 'Ignored the following files:'
for line in files_content['ignored'].split('\n'):
yield line
def test_create_from_snapshot_with_env(self):
req = ExecuteProcessRequest(
argv=('foo', ),
description="Some process",
env={'VAR': 'VAL'},
input_files=EMPTY_DIRECTORY_DIGEST,
)
self.assertEqual(req.env, ('VAR', 'VAL'))
def test_create_from_snapshot_with_env(self):
req = ExecuteProcessRequest(
argv=("foo",),
description="Some process",
env={"VAR": "VAL"},
input_files=EMPTY_DIRECTORY_DIGEST,
)
self.assertEqual(req.env, ("VAR", "VAL"))
def _compile_hermetic(self, jvm_options, ctx, classes_dir, zinc_args,
compiler_bridge_classpath_entry, dependency_classpath,
scalac_classpath_entries):
zinc_relpath = fast_relpath(self._zinc.zinc, get_buildroot())
snapshots = [
self._zinc.snapshot(self.context._scheduler),
ctx.target.sources_snapshot(self.context._scheduler),
]
relevant_classpath_entries = dependency_classpath + [compiler_bridge_classpath_entry]
directory_digests = tuple(
entry.directory_digest for entry in relevant_classpath_entries if entry.directory_digest
)
if len(directory_digests) != len(relevant_classpath_entries):
for dep in relevant_classpath_entries:
if dep.directory_digest is None:
logger.warning(
"ClasspathEntry {} didn't have a Digest, so won't be present for hermetic "
"execution".format(dep)
)
snapshots.extend(
classpath_entry.directory_digest for classpath_entry in scalac_classpath_entries
)
# TODO: Extract something common from Executor._create_command to make the command line
# TODO: Lean on distribution for the bin/java appending here
merged_input_digest = self.context._scheduler.merge_directories(
tuple(s.directory_digest for s in snapshots) + directory_digests
)
argv = ['.jdk/bin/java'] + jvm_options + [
'-cp', zinc_relpath,
Zinc.ZINC_COMPILE_MAIN
] + zinc_args
# TODO(#6071): Our ExecuteProcessRequest expects a specific string type for arguments,
# which py2 doesn't default to. This can be removed when we drop python 2.
argv = [text_type(arg) for arg in argv]
req = ExecuteProcessRequest(
argv=tuple(argv),
input_files=merged_input_digest,
output_directories=(classes_dir,),
description="zinc compile for {}".format(ctx.target.address.spec),
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying_dist`
jdk_home=text_type(self._zinc.underlying_dist.home),
)
res = self.context.execute_process_synchronously_or_raise(
req, self.name(), [WorkUnitLabel.COMPILER])
# TODO: Materialize as a batch in do_compile or somewhere
self.context._scheduler.materialize_directories((
DirectoryToMaterialize(get_buildroot(), res.output_directory_digest),
))
# TODO: This should probably return a ClasspathEntry rather than a Digest
return res.output_directory_digest
def _runtool_hermetic(self, main, tool_name, args, distribution, tgt=None, input_files=tuple(), input_digest=None, output_dir=None):
tool_classpath_abs = self.tool_classpath(tool_name)
tool_classpath = fast_relpath_collection(tool_classpath_abs)
classpath_for_cmd = os.pathsep.join(tool_classpath)
cmd = [
distribution.java,
]
cmd.extend(self.get_options().jvm_options)
cmd.extend(['-cp', classpath_for_cmd])
cmd.extend([main])
cmd.extend(args)
pathglobs = list(tool_classpath)
pathglobs.extend(f if os.path.isfile(f) else '{}/**'.format(f) for f in input_files)
if pathglobs:
root = PathGlobsAndRoot(
PathGlobs(tuple(pathglobs)),
text_type(get_buildroot()))
# dont capture snapshot, if pathglobs is empty
path_globs_input_digest = self.context._scheduler.capture_snapshots((root,))[0].directory_digest
if path_globs_input_digest and input_digest:
epr_input_files = self.context._scheduler.merge_directories(
(path_globs_input_digest, input_digest))
else:
epr_input_files = path_globs_input_digest or input_digest
epr = ExecuteProcessRequest(
argv=tuple(cmd),
input_files=epr_input_files,
output_files=tuple(),
output_directories=(output_dir,),
timeout_seconds=15*60,
description='run {} for {}'.format(tool_name, tgt),
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying_dist`
jdk_home=text_type(self._zinc.underlying_dist.home),
)
res = self.context.execute_process_synchronously_without_raising(
epr,
self.name(),
[WorkUnitLabel.TOOL])
if res.exit_code != 0:
raise TaskError(res.stderr)
if output_dir:
dump_digest(output_dir, res.output_directory_digest)
self.context._scheduler.materialize_directories((
DirectoryToMaterialize(
# NB the first element here is the root to materialize into, not the dir to snapshot
text_type(get_buildroot()),
res.output_directory_digest),
))
# TODO drop a file containing the digest, named maybe output_dir.digest
return res
def _runtool_hermetic(self, main, tool_name, args, distribution, tgt=None, input_files=tuple(), input_digest=None, output_dir=None):
tool_classpath_abs = self.tool_classpath(tool_name)
tool_classpath = fast_relpath_collection(tool_classpath_abs)
# TODO(#6071): Our ExecuteProcessRequest expects a specific string type for arguments,
# which py2 doesn't default to. This can be removed when we drop python 2.
str_jvm_options = [text_type(opt) for opt in self.get_options().jvm_options]
cmd = [
distribution.java,
] + str_jvm_options + [
'-cp', os.pathsep.join(tool_classpath),
main,
] + args
pathglobs = list(tool_classpath)
pathglobs.extend(f if os.path.isfile(f) else '{}/**'.format(f) for f in input_files)
if pathglobs:
root = PathGlobsAndRoot(
PathGlobs(tuple(pathglobs)),
text_type(get_buildroot()))
# dont capture snapshot, if pathglobs is empty
path_globs_input_digest = self.context._scheduler.capture_snapshots((root,))[0].directory_digest
epr_input_files = self.context._scheduler.merge_directories(
((path_globs_input_digest,) if path_globs_input_digest else ())
+ ((input_digest,) if input_digest else ()))
epr = ExecuteProcessRequest(
argv=tuple(cmd),
input_files=epr_input_files,
output_files=tuple(),
output_directories=(output_dir,),
timeout_seconds=15*60,
description='run {} for {}'.format(tool_name, tgt),
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying.home` because
# ExecuteProcessRequest requires an existing, local jdk location.
jdk_home=text_type(distribution.underlying_home),
)
res = self.context.execute_process_synchronously_without_raising(
epr,
self.name(),
[WorkUnitLabel.TOOL])
if res.exit_code != 0:
raise TaskError(res.stderr, exit_code=res.exit_code)
if output_dir:
res.output_directory_digest.dump(output_dir)
self.context._scheduler.materialize_directories((
DirectoryToMaterialize(
# NB the first element here is the root to materialize into, not the dir to snapshot
text_type(get_buildroot()),
res.output_directory_digest),
))
# TODO drop a file containing the digest, named maybe output_dir.digest
return res
def create_requirements_pex(request, pex_bin, python_setup,
pex_build_environment, platform):
"""Returns a PEX with the given requirements, optional entry point, and optional
interpreter constraints."""
interpreter_search_paths = create_path_env_var(
python_setup.interpreter_search_paths)
env = {
"PATH": interpreter_search_paths,
**pex_build_environment.invocation_environment_dict
}
interpreter_constraint_args = []
for constraint in request.interpreter_constraints:
interpreter_constraint_args.extend(
["--interpreter-constraint", constraint])
# NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
# `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
# execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
# somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
# necessarily the interpreter that PEX will use to execute the generated .pex file.
# TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target
# platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735.
argv = [
"python", f"./{pex_bin.executable}", "--output-file",
request.output_filename
]
if request.entry_point is not None:
argv.extend(["--entry-point", request.entry_point])
argv.extend(interpreter_constraint_args + list(request.requirements))
# NOTE
# PEX outputs are platform dependent so in order to get a PEX that we can use locally, without cross-building
# we specify that out PEX command be run on the current local platform. When we support cross-building
# through CLI flags we can configure requests that build a PEX for out local platform that are
# able to execute on a different platform, but for now in order to guarantee correct build we need
# to restrict this command to execute on the same platform type that the output is intended for.
# The correct way to interpret the keys (execution_platform_constraint, target_platform_constraint)
# of this dictionary is "The output of this command is intended for `target_platform_constraint` iff
# it is run on `execution_platform_constraint`".
execute_process_request = MultiPlatformExecuteProcessRequest(
{
(PlatformConstraint(platform.value),
PlatformConstraint(platform.value)):
ExecuteProcessRequest(
argv=tuple(argv),
env=env,
input_files=pex_bin.directory_digest,
description=
f"Create a requirements PEX: {', '.join(request.requirements)}",
output_files=(request.output_filename, ))
})
result = yield Get(ExecuteProcessResult,
MultiPlatformExecuteProcessRequest,
execute_process_request)
yield RequirementsPex(directory_digest=result.output_directory_digest)
def _default_args_execute_process_request(self, argv=tuple(), env=None):
env = env or dict()
return ExecuteProcessRequest(
argv=argv,
description='',
env=env,
input_files=EMPTY_DIRECTORY_DIGEST,
output_files=(),
)
def console_output(self, targets):
if not self.get_options().transitive:
targets = self.context.target_roots
buildroot = get_buildroot()
with temporary_dir() as tmpdir:
# Write the paths of all files we want cloc to process to the so-called 'list file'.
# TODO: 1) list_file, report_file and ignored_file should be relative files within the
# execution "chroot", 2) list_file should be part of an input files Snapshot, and
# 3) report_file and ignored_file should be part of an output files Snapshot, when we have
# that capability.
list_file = os.path.join(tmpdir, 'list_file')
with open(list_file, 'w') as list_file_out:
for target in targets:
for source in target.sources_relative_to_buildroot():
list_file_out.write(os.path.join(buildroot, source))
list_file_out.write(b'\n')
report_file = os.path.join(tmpdir, 'report_file')
ignored_file = os.path.join(tmpdir, 'ignored')
# TODO: Look at how to make BinaryUtil support Snapshots - such as adding an instrinsic to do
# network fetch directly into a Snapshot.
# See http://cloc.sourceforge.net/#options for cloc cmd-line options.
cmd = (self._get_cloc_script(), '--skip-uniqueness',
'--ignored={}'.format(ignored_file),
'--list-file={}'.format(list_file),
'--report-file={}'.format(report_file))
if self.context._scheduler is None:
with self.context.new_workunit(name='cloc',
labels=[WorkUnitLabel.TOOL],
cmd=' '.join(cmd)) as workunit:
result = subprocess.call(cmd,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'))
else:
# TODO: Longer term we need to figure out what to put on $PATH in a remote execution env.
# Currently, we are adding everything within $PATH to the request.
env_path = ['PATH', os.environ.get('PATH')]
req = ExecuteProcessRequest(cmd, env_path)
execute_process_result, = self.context._scheduler.product_request(
ExecuteProcessResult, [req])
exit_code = execute_process_result.exit_code
if exit_code != 0:
raise TaskError('{} ... exited non-zero ({}).'.format(
' '.join(cmd), result))
with open(report_file, 'r') as report_file_in:
for line in report_file_in.read().split('\n'):
yield line
if self.get_options().ignored:
yield 'Ignored the following files:'
with open(ignored_file, 'r') as ignored_file_in:
for line in ignored_file_in.read().split('\n'):
yield line
def test_fallible_failing_command_returns_exited_result(self):
request = ExecuteProcessRequest(
argv=("/bin/bash", "-c", "exit 1"),
description='one-cat',
input_files=EMPTY_DIRECTORY_DIGEST,
)
result = self.scheduler.product_request(FallibleExecuteProcessResult, [request])[0]
self.assertEqual(result.exit_code, 1)
def cat_files_process_result_concatted(cat_exe_req):
cat_bin = cat_exe_req.shell_cat
cat_files_snapshot = yield Get(Snapshot, PathGlobs, cat_exe_req.path_globs)
process_request = ExecuteProcessRequest(
argv=cat_bin.argv_from_snapshot(cat_files_snapshot),
input_files=cat_files_snapshot.directory_digest,
description='cat some files',
)
cat_process_result = yield Get(ExecuteProcessResult, ExecuteProcessRequest, process_request)
yield Concatted(cat_process_result.stdout.decode('utf-8'))
def get_javac_version_output(javac_version_command):
javac_version_proc_req = ExecuteProcessRequest(
argv=javac_version_command.gen_argv(),
description=javac_version_command.description,
input_files=EMPTY_DIRECTORY_DIGEST,
)
javac_version_proc_result = yield Get(
ExecuteProcessResult, ExecuteProcessRequest, javac_version_proc_req)
yield JavacVersionOutput(text_type(javac_version_proc_result.stderr))
def test_fallible_failing_command_returns_exited_result(self):
request = ExecuteProcessRequest(
argv=("/bin/bash", "-c", "exit 1"),
description="one-cat",
input_files=EMPTY_DIRECTORY_DIGEST,
)
result = self.request_single_product(FallibleExecuteProcessResult, request)
self.assertEqual(result.exit_code, 1)
async def cat_files_process_result_concatted(cat_exe_req: CatExecutionRequest) -> Concatted:
cat_bin = cat_exe_req.shell_cat
cat_files_snapshot = await Get[Snapshot](PathGlobs, cat_exe_req.path_globs)
process_request = ExecuteProcessRequest(
argv=cat_bin.argv_from_snapshot(cat_files_snapshot),
input_files=cat_files_snapshot.directory_digest,
description="cat some files",
)
cat_process_result = await Get[ExecuteProcessResult](ExecuteProcessRequest, process_request)
return Concatted(cat_process_result.stdout.decode())
def test_non_fallible_failing_command_raises(self):
request = ExecuteProcessRequest(
argv=("/bin/bash", "-c", "exit 1"),
description="one-cat",
input_files=EMPTY_DIRECTORY_DIGEST,
)
with self.assertRaises(ExecutionError) as cm:
self.request_single_product(ExecuteProcessResult, request)
self.assertIn("process 'one-cat' failed with exit code 1.", str(cm.exception))
def test_timeout(self):
request = ExecuteProcessRequest(
argv=("/bin/bash", "-c", "/bin/sleep 0.2; /bin/echo -n 'European Burmese'"),
timeout_seconds=0.1,
description="sleepy-cat",
input_files=EMPTY_DIRECTORY_DIGEST,
)
result = self.request_single_product(FallibleExecuteProcessResult, request)
self.assertNotEqual(result.exit_code, 0)
self.assertIn(b"Exceeded timeout", result.stdout)
self.assertIn(b"sleepy-cat", result.stdout)
def test_jdk(self):
with temporary_dir() as temp_dir:
with open(os.path.join(temp_dir, 'roland'), 'w') as f:
f.write('European Burmese')
request = ExecuteProcessRequest(
argv=('/bin/cat', '.jdk/roland'),
input_files=EMPTY_DIRECTORY_DIGEST,
description='cat JDK roland',
jdk_home=temp_dir,
)
result = self.request_single_product(ExecuteProcessResult, request)
self.assertEqual(result.stdout, b'European Burmese')
def test_jdk(self):
with temporary_dir() as temp_dir:
with open(os.path.join(temp_dir, "roland"), "w") as f:
f.write("European Burmese")
request = ExecuteProcessRequest(
argv=("/bin/cat", ".jdk/roland"),
input_files=EMPTY_DIRECTORY_DIGEST,
description="cat JDK roland",
jdk_home=temp_dir,
)
result = self.request_single_product(ExecuteProcessResult, request)
self.assertEqual(result.stdout, b"European Burmese")
