def test_skip_None(self):
self.assertEqual(
combine_envs(None, {'USER': '******'}, None, {'TERM': 'xterm'},
None), {
'USER': '******',
'TERM': 'xterm'
})
def test_should_exit_when_invoked_as_script(self):
args = [sys.executable, inspect.getsourcefile(MRJobLauncher), "--quux", "baz"]
# add . to PYTHONPATH (in case mrjob isn't actually installed)
env = combine_envs(os.environ, {"PYTHONPATH": os.path.abspath(".")})
proc = Popen(args, stderr=PIPE, stdout=PIPE, env=env)
proc.communicate()
self.assertEqual(proc.returncode, 2)
def run_job(self, args=()):
args = [sys.executable, MRTwoStepJob.mr_job_script()] + list(args) + ["--no-conf"]
# add . to PYTHONPATH (in case mrjob isn't actually installed)
env = combine_envs(os.environ, {"PYTHONPATH": os.path.abspath(".")})
proc = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env)
stdout, stderr = proc.communicate(input="foo\nbar\nbar\n")
return stdout, stderr, proc.returncode
def test_should_exit_when_invoked_as_script(self):
args = [sys.executable, MRJob.mr_job_script(), '--quux', 'baz']
# add . to PYTHONPATH (in case mrjob isn't actually installed)
env = combine_envs(os.environ, {'PYTHONPATH': os.path.abspath('.')})
proc = Popen(args, stderr=PIPE, stdout=PIPE, env=env)
proc.communicate()
assert_equal(proc.returncode, 2)
def test_should_exit_when_invoked_as_script(self):
args = [sys.executable, MRJob.mr_job_script(), '--quux', 'baz']
# add . to PYTHONPATH (in case mrjob isn't actually installed)
env = combine_envs(os.environ, {'PYTHONPATH': os.path.abspath('.')})
proc = Popen(args, stderr=PIPE, stdout=PIPE, env=env)
proc.communicate()
self.assertEqual(proc.returncode, 2)
def run_job(self, args=()):
args = ([sys.executable, MRTwoStepJob.mr_job_script()] + list(args) +
['--no-conf'])
# add . to PYTHONPATH (in case mrjob isn't actually installed)
env = combine_envs(os.environ, {'PYTHONPATH': os.path.abspath('.')})
proc = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env)
stdout, stderr = proc.communicate(input='foo\nbar\nbar\n')
return stdout, stderr, proc.returncode
def test_later_values_take_precedence(self):
self.assertEqual(
combine_envs({
'TMPDIR': '/tmp',
'HOME': '/home/dave'
}, {'TMPDIR': '/var/tmp'}), {
'TMPDIR': '/var/tmp',
'HOME': '/home/dave'
})
def test_should_exit_when_invoked_as_script(self):
args = [sys.executable, inspect.getsourcefile(MRJobLauncher),
'--quux', 'baz']
# add . to PYTHONPATH (in case mrjob isn't actually installed)
env = combine_envs(os.environ,
{'PYTHONPATH': mrjob_pythonpath()})
proc = Popen(args, stderr=PIPE, stdout=PIPE, env=env)
_, err = proc.communicate()
self.assertEqual(proc.returncode, 2, err)
def test_paths(self):
assert_equal(combine_envs(
{'PATH': '/bin:/usr/bin',
'PYTHONPATH': '/usr/lib/python/site-packages',
'PS1': '> '},
{'PATH': '/home/dave/bin',
'PYTHONPATH': '/home/dave/python',
'CLASSPATH': '/home/dave/java',
'PS1': '\w> '}),
{'PATH': '/home/dave/bin:/bin:/usr/bin',
'PYTHONPATH': '/home/dave/python:/usr/lib/python/site-packages',
'CLASSPATH': '/home/dave/java',
'PS1': '\w> '})
def test_clear_paths(self):
self.assertEqual(
combine_envs(
{"PATH": "/bin:/usr/bin", "PYTHONPATH": "/usr/lib/python/site-packages", "PS1": "> "},
{
"PATH": ClearedValue("/home/dave/bin"),
"PYTHONPATH": ClearedValue(None),
"CLASSPATH": "/home/dave/java",
"PS1": "\w> ",
},
),
{"PATH": "/home/dave/bin", "CLASSPATH": "/home/dave/java", "PS1": "\w> "},
)
def test_paths(self):
self.assertEqual(combine_envs(
{'PATH': '/bin:/usr/bin',
'PYTHONPATH': '/usr/lib/python/site-packages',
'PS1': '> '},
{'PATH': '/home/dave/bin',
'PYTHONPATH': '/home/dave/python',
'CLASSPATH': '/home/dave/java',
'PS1': '\w> '}),
{'PATH': '/home/dave/bin:/bin:/usr/bin',
'PYTHONPATH': '/home/dave/python:/usr/lib/python/site-packages',
'CLASSPATH': '/home/dave/java',
'PS1': '\w> '})
def test_clear_paths(self):
self.assertEqual(
combine_envs(
{'PATH': '/bin:/usr/bin',
'PYTHONPATH': '/usr/lib/python/site-packages',
'PS1': '> '},
{'PATH': ClearedValue('/home/dave/bin'),
'PYTHONPATH': ClearedValue(None),
'CLASSPATH': '/home/dave/java',
'PS1': '\w> '}),
{'PATH': '/home/dave/bin',
'CLASSPATH': '/home/dave/java',
'PS1': '\w> '})
def test_later_values_take_precedence(self):
self.assertEqual(
combine_envs({"TMPDIR": "/tmp", "HOME": "/home/dave"}, {"TMPDIR": "/var/tmp"}),
{"TMPDIR": "/var/tmp", "HOME": "/home/dave"},
)
def test_empty(self):
self.assertEqual(combine_envs(), {})
def test_skip_None(self):
assert_equal(combine_envs(None, {'USER': '******'}, None,
{'TERM': 'xterm'}, None),
{'USER': '******', 'TERM': 'xterm'})
def test_later_values_take_precedence(self):
assert_equal(
combine_envs({'TMPDIR': '/tmp', 'HOME': '/home/dave'},
{'TMPDIR': '/var/tmp'}),
{'TMPDIR': '/var/tmp', 'HOME': '/home/dave'})
def test_empty(self):
assert_equal(combine_envs(), {})
def test_empty(self):
self.assertEqual(combine_envs(), {})
def _add_python_archive(self, path):
file_dict = self._add_archive_for_upload(path)
log.debug('adding %s to PYTHONPATH' % file_dict['name'])
self._cmdenv = combine_envs(
self._cmdenv, {'PYTHONPATH': file_dict['name']})
self._python_archives.append(file_dict)
def _invoke_step(self, args, outfile_name, env=None):
"""Run the given command, outputting into outfile, and reading
from the previous outfile (or, for the first step, from our
original output files).
outfile is a path relative to our local tmp dir. commands are run
inside self._working_dir
We'll intelligently handle stderr from the process.
"""
# keep the current environment because we need PATH to find binaries
# and make PYTHONPATH work
env = combine_envs(
{'PYTHONPATH': os.getcwd()},
os.environ,
self._cmdenv,
env or {})
# decide where to get input
if self._prev_outfile is not None:
input_paths = [self._prev_outfile]
else:
input_paths = []
for path in self._input_paths:
if path == '-':
input_paths.append(self._dump_stdin_to_local_file())
else:
input_paths.append(path)
# add input to the command line
for path in input_paths:
args.append(os.path.abspath(path))
log.info('> %s' % cmd_line(args))
# set up outfile
outfile = os.path.join(self._get_local_tmp_dir(), outfile_name)
log.info('writing to %s' % outfile)
log.debug('')
self._prev_outfile = outfile
write_to = open(outfile, 'w')
# run the process
proc = Popen(args, stdout=write_to, stderr=PIPE,
cwd=self._working_dir, env=env)
# handle counters, status msgs, and other stuff on stderr
stderr_lines = self._process_stderr_from_script(proc.stderr)
tb_lines = find_python_traceback(stderr_lines)
self._print_counters()
returncode = proc.wait()
if returncode != 0:
# try to throw a useful exception
if tb_lines:
raise Exception(
'Command %r returned non-zero exit status %d:\n%s' %
(args, returncode, ''.join(tb_lines)))
else:
raise Exception(
'Command %r returned non-zero exit status %d: %s' %
(args, returncode))
# flush file descriptors
write_to.flush()
def test_skip_None(self):
self.assertEqual(
combine_envs(None, {"USER": "******"}, None, {"TERM": "xterm"}, None), {"USER": "******", "TERM": "xterm"}
)
def test_empty(self):
assert_equal(combine_envs(), {})
