def test_cleanup_options(self):
stderr = StringIO()
with no_handlers_for_logger('mrjob.runner'):
log_to_stream('mrjob.runner', stderr)
opts = RunnerOptionStore(
'inline',
dict(cleanup=['LOCAL_SCRATCH', 'REMOTE_SCRATCH'],
cleanup_on_failure=['JOB_FLOW', 'SCRATCH']),
[])
self.assertEqual(opts['cleanup'], ['LOCAL_TMP', 'CLOUD_TMP'])
self.assertIn(
'Deprecated cleanup option LOCAL_SCRATCH has been renamed'
' to LOCAL_TMP', stderr.getvalue())
self.assertIn(
'Deprecated cleanup option REMOTE_SCRATCH has been renamed'
' to CLOUD_TMP', stderr.getvalue())
self.assertEqual(opts['cleanup_on_failure'], ['CLUSTER', 'TMP'])
self.assertIn(
'Deprecated cleanup_on_failure option JOB_FLOW has been'
' renamed to CLUSTER', stderr.getvalue())
self.assertIn(
'Deprecated cleanup_on_failure option SCRATCH has been renamed'
' to TMP', stderr.getvalue())
def test_extra_kwargs_passed_in_directly_okay(self):
with logger_disabled('mrjob.runner'):
opts = RunnerOptionStore('inline', {
'local_tmp_dir': '/var/tmp',
'foo': 'bar'
}, [])
self.assertEqual(opts['local_tmp_dir'], '/var/tmp')
self.assertNotIn('bar', opts)
def test_empty_runner_error(self):
conf = dict(runner=dict(local=dict(local_tmp_dir='/tmp')))
path = self.save_conf('basic', conf)
stderr = StringIO()
with no_handlers_for_logger():
log_to_stream('mrjob.runner', stderr)
RunnerOptionStore('inline', {}, [path])
self.assertEqual("No configs specified for inline runner\n",
stderr.getvalue())
def test_runner_option_store(self):
stderr = StringIO()
with no_handlers_for_logger('mrjob.conf'):
log_to_stream('mrjob.conf', stderr)
opts = RunnerOptionStore(
'inline', dict(base_tmp_dir='/scratch'), [])
self.assertEqual(opts['local_tmp_dir'], '/scratch')
self.assertNotIn('base_tmp_dir', opts)
self.assertIn('Deprecated option base_tmp_dir has been renamed'
' to local_tmp_dir', stderr.getvalue())
def test_recurse(self):
path = os.path.join(self.tmp_dir, 'LOL.conf')
recurse_conf = dict(include=path)
with open(path, 'w') as f:
dump_mrjob_conf(recurse_conf, f)
stderr = StringIO()
with no_handlers_for_logger():
log_to_stream('mrjob.conf', stderr)
RunnerOptionStore('inline', {}, [path])
self.assertIn('%s tries to recursively include %s!' % (path, path),
stderr.getvalue())
def test_conf_contain_only_include_file(self):
"""If a config file only include other configuration files
no warnings are thrown as long as the included files are
not empty.
"""
# dummy configuration for include file 1
conf = {
'runners': {
'inline': {
'local_tmp_dir': "include_file1_local_tmp_dir"
}
}
}
include_file_1 = self.save_conf('include_file_1', conf)
# dummy configuration for include file 2
conf = {
'runners': {
'inline': {
'local_tmp_dir': "include_file2_local_tmp_dir"
}
}
}
include_file_2 = self.save_conf('include_file_2', conf)
# test configuration
conf = {
'include': [include_file_1, include_file_2]
}
path = self.save_conf('twoincludefiles', conf)
stderr = StringIO()
with no_handlers_for_logger():
log_to_stream('mrjob.runner', stderr)
RunnerOptionStore('inline', {}, [path])
self.assertEqual(
"",
stderr.getvalue())
def test_cleanup_options(self):
stderr = StringIO()
with no_handlers_for_logger('mrjob.runner'):
log_to_stream('mrjob.runner', stderr)
opts = RunnerOptionStore(
'inline',
dict(cleanup=['LOCAL_SCRATCH', 'REMOTE_SCRATCH'],
cleanup_on_failure=['SCRATCH']), [])
self.assertEqual(opts['cleanup'], ['LOCAL_TMP', 'REMOTE_TMP'])
self.assertIn(
'Deprecated cleanup option LOCAL_SCRATCH has been renamed'
' to LOCAL_TMP', stderr.getvalue())
self.assertIn(
'Deprecated cleanup option REMOTE_SCRATCH has been renamed'
' to REMOTE_TMP', stderr.getvalue())
# should quietly convert string to list
self.assertEqual(opts['cleanup_on_failure'], ['TMP'])
self.assertIn(
'Deprecated cleanup_on_failure option SCRATCH has been renamed'
' to TMP', stderr.getvalue())
def opts_for_conf(self, name, conf):
conf_path = self.save_conf(name, conf)
return RunnerOptionStore('inline', {}, [conf_path])
def test_extra_kwargs_in_mrjob_conf_okay(self):
with logger_disabled('mrjob.runner'):
opts = RunnerOptionStore('inline', {}, [self.path])
self.assertEqual(opts['setup_cmds'], ['echo foo'])
self.assertNotIn('qux', opts)
def setUp(self):
super(ClearTagTestCase, self).setUp()
self.base_conf_path = self.save_conf('base.conf', self.BASE_CONF)
self.base_opts = RunnerOptionStore('inline', {}, [self.base_conf_path])
def test_multiple_configs_via_runner_args(self):
path_left = self.save_conf('left.conf', self.BASE_CONFIG_LEFT)
path_right = self.save_conf('right.conf', self.BASE_CONFIG_RIGHT)
opts = RunnerOptionStore('inline', {}, [path_left, path_right])
self.assertEqual(opts['jobconf'],
dict(from_left=1, from_both=2, from_right=2))
def _assert_interp(self, val, **kwargs):
opts = RunnerOptionStore('inline', kwargs, [])
self.assertEqual(opts['interpreter'], val)
