def test_list(self):
self.assertEqual(_load_yaml_with_clear_tag('- foo\n- bar'),
['foo', 'bar'])
self.assertEqual(_load_yaml_with_clear_tag('!clear\n- foo\n- bar'),
ClearedValue(['foo', 'bar']))
self.assertEqual(_load_yaml_with_clear_tag('- foo\n- !clear bar'),
['foo', ClearedValue('bar')])
self.assertEqual(
_load_yaml_with_clear_tag('!clear\n- !clear foo\n- !clear bar'),
ClearedValue([ClearedValue('foo'), ClearedValue('bar')]))
def test_cleared_opt_values(self):
self.assertEqual(
combine_opts(dict(foo=combine_lists), {'foo': ['bar']},
{'foo': ClearedValue(['baz'])}),
# ClearedValue(['baz']) overrides bar
{'foo': ['baz']})
self.assertEqual(
combine_opts(dict(foo=combine_lists), {'foo': ['bar']},
{'foo': ClearedValue(None)}),
# not None!
{'foo': []})
def test_clear_paths(self):
self.assertEqual(
combine_envs(
{'PATH': '/bin:/usr/bin',
'PYTHONPATH': '/usr/lib/python/site-packages',
'PS1': '> '},
{'PATH': ClearedValue('/home/dave/bin'),
'PYTHONPATH': ClearedValue(None),
'CLASSPATH': '/home/dave/java',
'PS1': '\w> '}),
{'PATH': '/home/dave/bin',
'CLASSPATH': '/home/dave/java',
'PS1': '\w> '})
def test_cant_clear_entire_opt_dicts(self):
self.assertRaises(
TypeError,
combine_opts,
dict(foo=combine_lists),
{'foo': ['bar']},
ClearedValue({'foo': ['baz']}))
def test_no_support_for_clear_tags(self):
conf = {'runners': {'foo': {'qux': ClearedValue('quux')}}}
conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')
with open(conf_path, 'w') as f:
self.assertRaises(TypeError,
dump_mrjob_conf, conf, f)
def test_round_trip_with_clear_tag(self):
self._test_round_trip(
{'runners': {
'foo': {
'qux': ClearedValue('quux')
}
}})
def test_cleared_value(self):
self.assertEqual(
combine_jobconfs(
dict(foo='bar', bar='baz'),
dict(bar=ClearedValue('qux')),
),
dict(foo='bar', bar='qux'),
)
def test_deleted_value(self):
self.assertEqual(
combine_jobconfs(
dict(foo='bar', bar='baz'),
dict(bar=ClearedValue(None)),
),
dict(foo='bar'),
)
def test_nesting(self):
self.assertEqual(
_load_yaml_with_clear_tag('foo:\n - bar\n - baz: qux'),
{'foo': ['bar', {'baz': 'qux'}]})
self.assertEqual(
_load_yaml_with_clear_tag(
'!clear\n foo:\n - bar\n - baz: qux'),
ClearedValue({'foo': ['bar', {'baz': 'qux'}]}))
self.assertEqual(
_load_yaml_with_clear_tag('!clear foo:\n - bar\n - baz: qux'),
{ClearedValue('foo'): ['bar', {'baz': 'qux'}]})
self.assertEqual(
_load_yaml_with_clear_tag('foo: !clear\n - bar\n - baz: qux'),
{'foo': ClearedValue(['bar', {'baz': 'qux'}])})
self.assertEqual(
_load_yaml_with_clear_tag('foo:\n - !clear bar\n - baz: qux'),
{'foo': [ClearedValue('bar'), {'baz': 'qux'}]})
self.assertEqual(
_load_yaml_with_clear_tag(
'foo:\n - bar\n - !clear\n baz: qux'),
{'foo': ['bar', ClearedValue({'baz': 'qux'})]})
self.assertEqual(
_load_yaml_with_clear_tag('foo:\n - bar\n - !clear baz: qux'),
{'foo': ['bar', {ClearedValue('baz'): 'qux'}]})
self.assertEqual(
_load_yaml_with_clear_tag('foo:\n - bar\n - baz: !clear qux'),
{'foo': ['bar', {'baz': ClearedValue('qux')}]})
def test_cleared_value(self):
self.assertEqual(
combine_dicts({
'USER': '******',
'TERM': 'xterm'
}, {'USER': ClearedValue('caleb')}), {
'TERM': 'xterm',
'USER': '******'
})
def test_clear_setup(self):
opts = self.opts_for_conf('extend.conf', {
'include': self.base_conf_path,
'runners': {
'inline': {
'setup': ClearedValue(['instead do this'])
}
}
})
self.assertEqual(opts['cmdenv'], self.base_opts['cmdenv'])
self.assertEqual(opts['jobconf'], self.base_opts['jobconf'])
self.assertEqual(opts['setup'], ['instead do this'])
def test_clear_jobconf(self):
opts = self.opts_for_conf('extend.conf', {
'include': self.base_conf_path,
'runners': {
'inline': {
'jobconf': ClearedValue(None)
}
}
})
self.assertEqual(opts['cmdenv'], self.base_opts['cmdenv'])
self.assertEqual(opts['jobconf'], {})
self.assertEqual(opts['setup'], self.base_opts['setup'])
def test_clear_cmdenv_path(self):
opts = self.opts_for_conf('extend.conf', {
'include': self.base_conf_path,
'runners': {
'inline': {
'cmdenv': {
'PATH': ClearedValue('/some/even/better/dir')
}
}
}
})
self.assertEqual(opts['cmdenv'], {'PATH': '/some/even/better/dir'})
self.assertEqual(opts['jobconf'], self.base_opts['jobconf'])
self.assertEqual(opts['setup'], self.base_opts['setup'])
def test_list(self):
self.assertEqual(_fix_clear_tags(['foo', 'bar']), ['foo', 'bar'])
self.assertEqual(_fix_clear_tags(ClearedValue(['foo', 'bar'])),
ClearedValue(['foo', 'bar']))
self.assertEqual(_fix_clear_tags(['foo', ClearedValue('bar')]),
['foo', 'bar'])
self.assertEqual(
_fix_clear_tags(
ClearedValue([ClearedValue('foo'),
ClearedValue('bar')])),
ClearedValue(['foo', 'bar']))
def test_dict(self):
self.assertEqual(_load_yaml_with_clear_tag('foo: bar'),
{'foo': 'bar'})
self.assertEqual(_load_yaml_with_clear_tag('!clear\nfoo: bar'),
ClearedValue({'foo': 'bar'}))
self.assertEqual(_load_yaml_with_clear_tag('!clear foo: bar'),
{ClearedValue('foo'): 'bar'})
self.assertEqual(_load_yaml_with_clear_tag('foo: !clear bar'),
{'foo': ClearedValue('bar')})
self.assertEqual(
_load_yaml_with_clear_tag('!clear\n!clear foo: !clear bar'),
ClearedValue({ClearedValue('foo'): ClearedValue('bar')}))
self.assertEqual(
_load_yaml_with_clear_tag('!clear foo: bar\nfoo: baz'),
{ClearedValue('foo'): 'bar', 'foo': 'baz'})
def test_string(self):
self.assertEqual(_load_yaml_with_clear_tag('foo'), 'foo')
self.assertEqual(_load_yaml_with_clear_tag('!clear foo'),
ClearedValue('foo'))
def test_nesting(self):
self.assertEqual(_fix_clear_tags({'foo': ['bar', {
'baz': 'qux'
}]}), {'foo': ['bar', {
'baz': 'qux'
}]})
self.assertEqual(
_fix_clear_tags(ClearedValue({'foo': ['bar', {
'baz': 'qux'
}]})), ClearedValue({'foo': ['bar', {
'baz': 'qux'
}]}))
self.assertEqual(
_fix_clear_tags({ClearedValue('foo'): ['bar', {
'baz': 'qux'
}]}),
{'foo': ClearedValue(['bar', {
'baz': 'qux'
}])})
self.assertEqual(
_fix_clear_tags({'foo': ClearedValue(['bar', {
'baz': 'qux'
}])}), {'foo': ClearedValue(['bar', {
'baz': 'qux'
}])})
self.assertEqual(
_fix_clear_tags({'foo': [ClearedValue('bar'), {
'baz': 'qux'
}]}), {'foo': ['bar', {
'baz': 'qux'
}]})
self.assertEqual(
_fix_clear_tags({'foo': ['bar',
ClearedValue({'baz': 'qux'})]}),
{'foo': ['bar', {
'baz': 'qux'
}]})
self.assertEqual(
_fix_clear_tags({'foo': ['bar', {
ClearedValue('baz'): 'qux'
}]}), {'foo': ['bar', {
'baz': ClearedValue('qux')
}]})
self.assertEqual(
_fix_clear_tags({'foo': ['bar', {
'baz': ClearedValue('qux')
}]}), {'foo': ['bar', {
'baz': ClearedValue('qux')
}]})
def test_dict(self):
self.assertEqual(_fix_clear_tags({'foo': 'bar'}), {'foo': 'bar'})
self.assertEqual(_fix_clear_tags(ClearedValue({'foo': 'bar'})),
ClearedValue({'foo': 'bar'}))
self.assertEqual(_fix_clear_tags({ClearedValue('foo'): 'bar'}),
{'foo': ClearedValue('bar')})
self.assertEqual(_fix_clear_tags({'foo': ClearedValue('bar')}),
{'foo': ClearedValue('bar')})
self.assertEqual(
_fix_clear_tags(
ClearedValue({ClearedValue('foo'): ClearedValue('bar')})),
ClearedValue({'foo': ClearedValue('bar')}))
# ClearedValue('foo') key overrides 'foo' key
self.assertEqual(
_fix_clear_tags({
ClearedValue('foo'): 'bar',
'foo': 'baz'
}), {'foo': ClearedValue('bar')})
def test_int(self):
self.assertEqual(_fix_clear_tags(18), 18)
self.assertEqual(_fix_clear_tags(ClearedValue(18)), ClearedValue(18))
def test_string(self):
self.assertEqual(_fix_clear_tags('foo'), 'foo')
self.assertEqual(_fix_clear_tags(ClearedValue('foo')),
ClearedValue('foo'))
def test_none(self):
self.assertEqual(_fix_clear_tags(None), None)
self.assertEqual(_fix_clear_tags(ClearedValue(None)),
ClearedValue(None))
def test_int(self):
self.assertEqual(_load_yaml_with_clear_tag('18'), 18)
self.assertEqual(_load_yaml_with_clear_tag('!clear 18'),
ClearedValue(18))
def test_dont_accept_wrapped_dicts(self):
self.assertRaises(AttributeError, combine_jobconfs,
ClearedValue(dict(foo='bar')))
def test_null(self):
self.assertEqual(_load_yaml_with_clear_tag('null'), None)
self.assertEqual(_load_yaml_with_clear_tag('!clear null'),
ClearedValue(None))
def test_empty(self):
self.assertEqual(_load_yaml_with_clear_tag(''), None)
self.assertEqual(_load_yaml_with_clear_tag('!clear'),
ClearedValue(None))
def test_dont_accept_wrapped_dicts(self):
self.assertRaises(AttributeError, combine_dicts,
ClearedValue({'USER': '******'}))
def test_deleted_value(self):
self.assertEqual(
combine_dicts({
'USER': '******',
'TERM': 'xterm'
}, {'USER': ClearedValue(None)}), {'TERM': 'xterm'})
