def test_imports(tempdir, sc, build_store, cfg): # Make dependencies doc = { "name": "foosoft", "version": "na", "build": {"commands": []}, } foo_id, foo_path = build_store.ensure_present(doc, cfg) doc = { "name": "barsoft", "version": "na", "build": {"commands": []}, } bar_id, bar_path = build_store.ensure_present(doc, cfg) virtuals = {'virtual:bar' : bar_id} # Dependee LD_LIBRARY_PATH = os.environ.get("LD_LIBRARY_PATH", "") doc = { "import": [{"ref": "FOOSOFT", "id": foo_id}, {"ref": "BARSOFT", "id": "virtual:bar"}], "commands": [ {"cmd": env_to_stderr + ["FOOSOFT_DIR"]}, {"cmd": env_to_stderr + ["FOOSOFT_ID"]}, {"cmd": env_to_stderr + ["BARSOFT_DIR"]}, {"cmd": env_to_stderr + ["BARSOFT_ID"]}, ] } with log_capture('build') as logger: ret_env = run_job.run_job(logger, build_store, doc, {}, '<no-artifact>', virtuals, tempdir, cfg) eq_(["FOOSOFT_DIR=%r" % foo_path, "FOOSOFT_ID=%r" % foo_id, "BARSOFT_DIR=%r" % bar_path, "BARSOFT_ID=%r" % bar_id], filter_out(logger.lines))
def test_inputs(tempdir, sc, build_store, cfg): job_spec = { "commands": [ { "env": {"LD_LIBRARY_PATH": os.environ.get("LD_LIBRARY_PATH", "")}, "cmd": [sys.executable, "$in0", "$in1"], "inputs": [ {"text": ["import sys", "import json", "with open(sys.argv[1]) as f:" " print json.load(f)['foo']"]}, {"json": {"foo": "Hello1"}} ] }, { "env": {"LD_LIBRARY_PATH": os.environ.get("LD_LIBRARY_PATH", "")}, "cmd": [sys.executable, "$in0"], "inputs": [{"string": "import sys\nprint 'Hello2'"}] }, ] } with log_capture('build') as logger: ret_env = run_job.run_job(logger, build_store, job_spec, {"BAZ": "BAZ"}, '<no-artifact>', {"virtual:bash": "bash/ljnq7g35h6h4qtb456h5r35ku3dq25nl"}, tempdir, cfg) logger.assertLogged('INFO:Hello1') logger.assertLogged('INFO:Hello2')
def test_imports(tempdir, sc, build_store, cfg): # Make dependencies doc = { "name": "foosoft", "version": "na", "build": { "commands": [] }, } foo_id, foo_path = build_store.ensure_present(doc, cfg) doc = { "name": "barsoft", "version": "na", "build": { "commands": [] }, } bar_id, bar_path = build_store.ensure_present(doc, cfg) virtuals = {'virtual:bar': bar_id} # Dependee LD_LIBRARY_PATH = os.environ.get("LD_LIBRARY_PATH", "") doc = { "import": [{ "ref": "FOOSOFT", "id": foo_id }, { "ref": "BARSOFT", "id": "virtual:bar" }], "commands": [ { "cmd": env_to_stderr + ["FOOSOFT_DIR"] }, { "cmd": env_to_stderr + ["FOOSOFT_ID"] }, { "cmd": env_to_stderr + ["BARSOFT_DIR"] }, { "cmd": env_to_stderr + ["BARSOFT_ID"] }, ] } with log_capture('build') as logger: ret_env = run_job.run_job(logger, build_store, doc, {}, '<no-artifact>', virtuals, tempdir, cfg) eq_([ "FOOSOFT_DIR=%r" % foo_path, "FOOSOFT_ID=%r" % foo_id, "BARSOFT_DIR=%r" % bar_path, "BARSOFT_ID=%r" % bar_id ], filter_out(logger.lines))
def test_trap_tarball_attack(): logger = logging.getLogger() with temp_source_cache(logger) as sc: for tb in mock_dangerous_tarballs: with log_capture() as logger: key = sc.fetch_archive('file:' + tb) with temp_dir() as d: with assert_raises(SecurityError): sc.unpack(key, d) logger.assertLogged('^ERROR:.*attempted to break out')
def test_env_control(tempdir, sc, build_store, cfg): LD_LIBRARY_PATH = os.environ.get("LD_LIBRARY_PATH", "") job_spec = { "commands": [ { "set": "LD_LIBRARY_PATH", "value": LD_LIBRARY_PATH }, { "set": "FOO", "value": "foo" }, { "set": "FOO", "value": "bar" }, { "append_flag": "CFLAGS", "value": "-O3" }, { "prepend_flag": "CFLAGS", "value": "-O2" }, { "prepend_flag": "CFLAGS", "value": "-O1" }, { "append_path": "PATH", "value": "/bar/bin" }, { "prepend_path": "PATH", "value": "/foo/bin" }, { "cmd": env_to_stderr + ["FOO"] }, { "cmd": env_to_stderr + ["CFLAGS"] }, { "cmd": env_to_stderr + ["PATH"] }, ] } with log_capture('build') as logger: ret_env = run_job.run_job(logger, build_store, job_spec, {}, '<no-artifact>', {}, tempdir, cfg) eq_(["FOO='bar'", "CFLAGS='-O1 -O2 -O3'", "PATH='/foo/bin:/bar/bin'"], filter_out(logger.lines))
def test_attach_log(tempdir, sc, build_store, cfg): if 'linux' not in sys.platform: raise SkipTest('Linux only') with file(pjoin(tempdir, 'hello'), 'w') as f: f.write('hello from pipe') job_spec = { "commands": [ {"hit": ["logpipe", "mylog", "WARNING"], "to_var": "LOG"}, {"cmd": ["/bin/dd", "if=hello", "of=$LOG"]}, ]} with log_capture('build') as logger: run_job.run_job(logger, build_store, job_spec, {}, '<no-artifact>', {}, tempdir, cfg) logger.assertLogged('^WARNING:mylog:hello from pipe$')
def test_run_job_environment(tempdir, sc, build_store, cfg): # tests that the environment gets correctly set up and that the local scope feature # works LD_LIBRARY_PATH = os.environ.get("LD_LIBRARY_PATH", "") job_spec = { "commands": [ {"set": "LD_LIBRARY_PATH", "value": LD_LIBRARY_PATH}, {"set": "FOO", "value": "foo"}, {"set": "BAR", "nohash_value": "bar"}, { "commands": [ {"set": "BAR", "value": "${FOO}x"}, {"set": "HI", "nohash_value": "hi"}, {"cmd": env_to_stderr + ["FOO"]}, {"cmd": env_to_stderr + ["BAR"]}, {"cmd": env_to_stderr + ["HI"]}, ], }, {"cmd": env_to_stderr + ["FOO"]}, {"cmd": env_to_stderr + ["BAR"]}, {"cmd": env_to_stderr + ["HI"]}, ]} with log_capture('build') as logger: ret_env = run_job.run_job(logger, build_store, job_spec, {"BAZ": "BAZ"}, '<no-artifact>', {"virtual:bash": "bash/ljnq7g35h6h4qtb456h5r35ku3dq25nl"}, tempdir, cfg) assert 'HDIST_CONFIG' in ret_env del ret_env['HDIST_CONFIG'] del ret_env['PWD'] expected = { 'ARTIFACT': '<no-artifact>', 'BAR': 'bar', 'BAZ': 'BAZ', 'FOO': 'foo', 'HDIST_IMPORT': '', 'HDIST_IMPORT_PATHS': '', 'HDIST_VIRTUALS': 'virtual:bash=bash/ljnq7g35h6h4qtb456h5r35ku3dq25nl', 'LD_LIBRARY_PATH': LD_LIBRARY_PATH, 'PATH': '' } eq_(expected, ret_env) lines = filter_out(logger.lines) eq_(["FOO='foo'", "BAR='foox'", "HI='hi'", "FOO='foo'", "BAR='bar'", 'HI=None'], lines)
def test_attach_log(tempdir, sc, build_store, cfg): if 'linux' not in sys.platform: raise SkipTest('Linux only') with file(pjoin(tempdir, 'hello'), 'w') as f: f.write('hello from pipe') job_spec = { "commands": [ { "hit": ["logpipe", "mylog", "WARNING"], "to_var": "LOG" }, { "cmd": ["/bin/dd", "if=hello", "of=$LOG"] }, ] } with log_capture('build') as logger: run_job.run_job(logger, build_store, job_spec, {}, '<no-artifact>', {}, tempdir, cfg) logger.assertLogged('^WARNING:mylog:hello from pipe$')
def test_env_control(tempdir, sc, build_store, cfg): LD_LIBRARY_PATH = os.environ.get("LD_LIBRARY_PATH", "") job_spec = { "commands": [ {"set": "LD_LIBRARY_PATH", "value": LD_LIBRARY_PATH}, {"set": "FOO", "value": "foo"}, {"set": "FOO", "value": "bar"}, {"append_flag": "CFLAGS", "value": "-O3"}, {"prepend_flag": "CFLAGS", "value": "-O2"}, {"prepend_flag": "CFLAGS", "value": "-O1"}, {"append_path": "PATH", "value": "/bar/bin"}, {"prepend_path": "PATH", "value": "/foo/bin"}, {"cmd": env_to_stderr + ["FOO"]}, {"cmd": env_to_stderr + ["CFLAGS"]}, {"cmd": env_to_stderr + ["PATH"]}, ]} with log_capture('build') as logger: ret_env = run_job.run_job(logger, build_store, job_spec, {}, '<no-artifact>', {}, tempdir, cfg) eq_(["FOO='bar'", "CFLAGS='-O1 -O2 -O3'", "PATH='/foo/bin:/bar/bin'"], filter_out(logger.lines))
def test_inputs(tempdir, sc, build_store, cfg): job_spec = { "commands": [ { "env": { "LD_LIBRARY_PATH": os.environ.get("LD_LIBRARY_PATH", "") }, "cmd": [sys.executable, "$in0", "$in1"], "inputs": [{ "text": [ "import sys", "import json", "with open(sys.argv[1]) as f:" " print json.load(f)['foo']" ] }, { "json": { "foo": "Hello1" } }] }, { "env": { "LD_LIBRARY_PATH": os.environ.get("LD_LIBRARY_PATH", "") }, "cmd": [sys.executable, "$in0"], "inputs": [{ "string": "import sys\nprint 'Hello2'" }] }, ] } with log_capture('build') as logger: ret_env = run_job.run_job( logger, build_store, job_spec, {"BAZ": "BAZ"}, '<no-artifact>', {"virtual:bash": "bash/ljnq7g35h6h4qtb456h5r35ku3dq25nl"}, tempdir, cfg) logger.assertLogged('INFO:Hello1') logger.assertLogged('INFO:Hello2')
def test_log_pipe_stress(tempdir, sc, build_store, cfg): if 'linux' not in sys.platform: raise SkipTest('Linux only') # Stress-test the log piping a bit, since the combination of Unix FIFO # pipes and poll() is a bit tricky to get right. # We want to launch many clients who each concurrently send many messages, # then check that they all get through to log_capture. We do this by # writing out two Python scripts and executing them... NJOBS = 5 NMSGS = 300 # must divide 2 with open(pjoin(tempdir, 'client.py'), 'w') as f: f.write( dedent('''\ import os, sys msg = sys.argv[1] * (256 // 4) # less than PIPE_BUF, more than what we set BUFSIZE to for i in range(int(sys.argv[2]) // 2): with open(os.environ["LOG"], "a") as f: f.write("%s\\n" % msg) f.write("%s\\n" % msg) # hit stdout too sys.stdout.write("stdout:%s\\nstdout:%s\\n" % (sys.argv[1], sys.argv[1])) sys.stdout.flush() sys.stderr.write("stderr:%s\\nstderr:%s\\n" % (sys.argv[1], sys.argv[1])) sys.stderr.flush() ''')) with open(pjoin(tempdir, 'launcher.py'), 'w') as f: f.write( dedent('''\ import sys import subprocess procs = [subprocess.Popen([sys.executable, sys.argv[1], '%4d' % i, sys.argv[3]]) for i in range(int(sys.argv[2]))] for p in procs: if not p.wait() == 0: raise AssertionError("process failed: %d" % p.pid) ''')) job_spec = { "commands": [ { "hit": ["logpipe", "mylog", "WARNING"], "to_var": "LOG" }, { "set": "LD_LIBRARY_PATH", "value": os.environ.get("LD_LIBRARY_PATH", "") }, { "cmd": [ sys.executable, pjoin(tempdir, 'launcher.py'), pjoin(tempdir, 'client.py'), str(NJOBS), str(NMSGS) ] }, ] } old = run_job.LOG_PIPE_BUFSIZE try: run_job.LOG_PIPE_BUFSIZE = 50 with log_capture('build') as logger: run_job.run_job(logger, build_store, job_spec, {}, '<no-artifact>', {}, tempdir, cfg) finally: run_job.LOG_PIPE_BUFSIZE = old log_bins = [0] * NJOBS stdout_bins = [0] * NJOBS stderr_bins = [0] * NJOBS for line in logger.lines: parts = line.split(':') if len(parts) != 3: continue level, log, msg = parts if log == 'mylog': assert level == 'WARNING' assert msg == msg[:4] * (256 // 4) idx = int(msg[:4]) log_bins[idx] += 1 elif log == 'stdout': assert level == 'INFO' stdout_bins[int(msg)] += 1 elif log == 'stderr': assert level == 'INFO' stderr_bins[int(msg)] += 1 assert all(x == NMSGS for x in log_bins) assert all(x == NMSGS for x in stdout_bins) assert all(x == NMSGS for x in stderr_bins)
def doit(): with log_capture() as logger: run_job.run_job(logger, build_store, job_spec, {"echo": "/bin/echo"}, '<no-artifact>', {}, tempdir, cfg) eq_(["HI='a b'"], filter_out(logger.lines))
def test_run_job_environment(tempdir, sc, build_store, cfg): # tests that the environment gets correctly set up and that the local scope feature # works LD_LIBRARY_PATH = os.environ.get("LD_LIBRARY_PATH", "") job_spec = { "commands": [ { "set": "LD_LIBRARY_PATH", "value": LD_LIBRARY_PATH }, { "set": "FOO", "value": "foo" }, { "set": "BAR", "nohash_value": "bar" }, { "commands": [ { "set": "BAR", "value": "${FOO}x" }, { "set": "HI", "nohash_value": "hi" }, { "cmd": env_to_stderr + ["FOO"] }, { "cmd": env_to_stderr + ["BAR"] }, { "cmd": env_to_stderr + ["HI"] }, ], }, { "cmd": env_to_stderr + ["FOO"] }, { "cmd": env_to_stderr + ["BAR"] }, { "cmd": env_to_stderr + ["HI"] }, ] } with log_capture('build') as logger: ret_env = run_job.run_job( logger, build_store, job_spec, {"BAZ": "BAZ"}, '<no-artifact>', {"virtual:bash": "bash/ljnq7g35h6h4qtb456h5r35ku3dq25nl"}, tempdir, cfg) assert 'HDIST_CONFIG' in ret_env del ret_env['HDIST_CONFIG'] del ret_env['PWD'] expected = { 'ARTIFACT': '<no-artifact>', 'BAR': 'bar', 'BAZ': 'BAZ', 'FOO': 'foo', 'HDIST_IMPORT': '', 'HDIST_IMPORT_PATHS': '', 'HDIST_VIRTUALS': 'virtual:bash=bash/ljnq7g35h6h4qtb456h5r35ku3dq25nl', 'LD_LIBRARY_PATH': LD_LIBRARY_PATH, 'PATH': '' } eq_(expected, ret_env) lines = filter_out(logger.lines) eq_([ "FOO='foo'", "BAR='foox'", "HI='hi'", "FOO='foo'", "BAR='bar'", 'HI=None' ], lines)
def test_log_pipe_stress(tempdir, sc, build_store, cfg): if 'linux' not in sys.platform: raise SkipTest('Linux only') # Stress-test the log piping a bit, since the combination of Unix FIFO # pipes and poll() is a bit tricky to get right. # We want to launch many clients who each concurrently send many messages, # then check that they all get through to log_capture. We do this by # writing out two Python scripts and executing them... NJOBS = 5 NMSGS = 300 # must divide 2 with open(pjoin(tempdir, 'client.py'), 'w') as f: f.write(dedent('''\ import os, sys msg = sys.argv[1] * (256 // 4) # less than PIPE_BUF, more than what we set BUFSIZE to for i in range(int(sys.argv[2]) // 2): with open(os.environ["LOG"], "a") as f: f.write("%s\\n" % msg) f.write("%s\\n" % msg) # hit stdout too sys.stdout.write("stdout:%s\\nstdout:%s\\n" % (sys.argv[1], sys.argv[1])) sys.stdout.flush() sys.stderr.write("stderr:%s\\nstderr:%s\\n" % (sys.argv[1], sys.argv[1])) sys.stderr.flush() ''')) with open(pjoin(tempdir, 'launcher.py'), 'w') as f: f.write(dedent('''\ import sys import subprocess procs = [subprocess.Popen([sys.executable, sys.argv[1], '%4d' % i, sys.argv[3]]) for i in range(int(sys.argv[2]))] for p in procs: if not p.wait() == 0: raise AssertionError("process failed: %d" % p.pid) ''')) job_spec = { "commands": [ {"hit": ["logpipe", "mylog", "WARNING"], "to_var": "LOG"}, {"set": "LD_LIBRARY_PATH", "value": os.environ.get("LD_LIBRARY_PATH", "")}, {"cmd": [sys.executable, pjoin(tempdir, 'launcher.py'), pjoin(tempdir, 'client.py'), str(NJOBS), str(NMSGS)]}, ]} old = run_job.LOG_PIPE_BUFSIZE try: run_job.LOG_PIPE_BUFSIZE = 50 with log_capture('build') as logger: run_job.run_job(logger, build_store, job_spec, {}, '<no-artifact>', {}, tempdir, cfg) finally: run_job.LOG_PIPE_BUFSIZE = old log_bins = [0] * NJOBS stdout_bins = [0] * NJOBS stderr_bins = [0] * NJOBS for line in logger.lines: parts = line.split(':') if len(parts) != 3: continue level, log, msg = parts if log == 'mylog': assert level == 'WARNING' assert msg == msg[:4] * (256 // 4) idx = int(msg[:4]) log_bins[idx] += 1 elif log == 'stdout': assert level == 'INFO' stdout_bins[int(msg)] += 1 elif log == 'stderr': assert level == 'INFO' stderr_bins[int(msg)] += 1 assert all(x == NMSGS for x in log_bins) assert all(x == NMSGS for x in stdout_bins) assert all(x == NMSGS for x in stderr_bins)