def test_dont_overwrite(self):
# this jobconf contains two versions of the same variable
jobconf = {
'mapred.jar': 'a.jar',
'mapreduce.job.jar': 'a.jar',
}
self.assertEqual(translate_jobconf_dict(jobconf, '1.0'), jobconf)
self.assertFalse(self.log.warning.called)
def test_dont_overwrite(self):
# this jobconf contains two versions of the same variable
jobconf = {
'mapred.jar': 'a.jar',
'mapreduce.job.jar': 'a.jar',
}
self.assertEqual(translate_jobconf_dict(jobconf, '1.0'), jobconf)
self.assertFalse(self.log.warning.called)
def _jobconf_for_step(self, step_num):
"""Get the jobconf dictionary, optionally including step-specific
jobconf info.
Also translate jobconfs to the current Hadoop version, if necessary.
"""
step = self._get_step(step_num)
jobconf = combine_dicts(self._opts['jobconf'], step.get('jobconf'))
# if user is using the wrong jobconfs, add in the correct ones
version = self.get_hadoop_version()
if version:
jobconf = translate_jobconf_dict(jobconf, hadoop_version=version)
return jobconf
def test_hadoop_2(self):
self.assertEqual(
translate_jobconf_dict({
'foo.bar': 'baz',
'mapred.jar': 'a.jar',
'mapreduce.job.user.name': 'dave',
}, hadoop_version='2.0'),
{
'foo.bar': 'baz',
'mapred.jar': 'a.jar',
'mapreduce.job.jar': 'a.jar',
'mapreduce.job.user.name': 'dave',
})
self.assertTrue(self.log.warning.called)
def _jobconf_for_step(self, step_num):
"""Get the jobconf dictionary, optionally including step-specific
jobconf info.
Also translate jobconfs to the current Hadoop version, if necessary.
"""
step = self._get_step(step_num)
jobconf = combine_dicts(self._opts['jobconf'], step.get('jobconf'))
# if user is using the wrong jobconfs, add in the correct ones
version = self.get_hadoop_version()
if version:
jobconf = translate_jobconf_dict(jobconf, hadoop_version=version)
return jobconf
def test_hadoop_2(self):
self.assertEqual(
translate_jobconf_dict({
'foo.bar': 'baz',
'mapred.jar': 'a.jar',
'mapreduce.job.user.name': 'dave',
}, hadoop_version='2.0'),
{
'foo.bar': 'baz',
'mapred.jar': 'a.jar',
'mapreduce.job.jar': 'a.jar',
'mapreduce.job.user.name': 'dave',
})
self.assertTrue(self.log.warning.called)
def test_no_version(self):
self.assertEqual(
translate_jobconf_dict({
'foo.bar': 'baz', # unknown jobconf
'mapred.jar': 'a.jar', # Hadoop 1 jobconf
'mapreduce.job.user.name': 'dave', # Hadoop 2 jobconf
}),
{
'foo.bar': 'baz',
'mapred.jar': 'a.jar',
'mapreduce.job.jar': 'a.jar',
'mapreduce.job.user.name': 'dave',
'user.name': 'dave',
})
self.assertFalse(self.log.warning.called)
def test_no_version(self):
self.assertEqual(
translate_jobconf_dict({
'foo.bar': 'baz', # unknown jobconf
'mapred.jar': 'a.jar', # Hadoop 1 jobconf
'mapreduce.job.user.name': 'dave', # Hadoop 2 jobconf
}),
{
'foo.bar': 'baz',
'mapred.jar': 'a.jar',
'mapreduce.job.jar': 'a.jar',
'mapreduce.job.user.name': 'dave',
'user.name': 'dave',
})
self.assertFalse(self.log.warning.called)
def _jobconf_for_step(self, step_num):
"""Get the jobconf dictionary, optionally including step-specific
jobconf info.
Also translate jobconfs to the current Hadoop version, if necessary.
"""
step = self._get_step(step_num)
# _sort_values_jobconf() isn't relevant to Spark,
# but it doesn't do any harm either
jobconf = combine_dicts(self._sort_values_jobconf(),
self._opts['jobconf'], step.get('jobconf'))
# if user is using the wrong jobconfs, add in the correct ones
# and log a warning
hadoop_version = self.get_hadoop_version()
if hadoop_version:
jobconf = translate_jobconf_dict(jobconf, hadoop_version)
return jobconf
def _jobconf_for_step(self, step_num):
"""Get the jobconf dictionary, optionally including step-specific
jobconf info.
Also translate jobconfs to the current Hadoop version, if necessary.
"""
step = self._get_step(step_num)
# _sort_values_jobconf() isn't relevant to Spark,
# but it doesn't do any harm either
jobconf = combine_dicts(self._sort_values_jobconf(),
self._opts['jobconf'],
step.get('jobconf'))
# if user is using the wrong jobconfs, add in the correct ones
# and log a warning
hadoop_version = self.get_hadoop_version()
if hadoop_version:
jobconf = translate_jobconf_dict(jobconf, hadoop_version)
return jobconf
def test_empty(self):
self.assertEqual(translate_jobconf_dict({}), {})
self.assertFalse(self.log.warning.called)
def test_empty(self):
self.assertEqual(translate_jobconf_dict({}), {})
self.assertFalse(self.log.warning.called)
