def test_from_overlays(self):
self._check_matching(JobConfig())
jc = JobConfig(job_name='TestIngester',
preload=True,
data_directory='/tmp/a',
job_group='gg',
tracing='info')
jc.comment = 'Test configuration'
jc.target_pe_count = 2
self._check_matching(jc)
jc = JobConfig(job_name='TestIngester2')
jc.comment = 'Test configuration2'
self._check_matching(jc)
jc = JobConfig(preload=True)
jc.raw_overlay = {'a': 34}
self._check_matching(jc)
jc = JobConfig(preload=True)
jc.raw_overlay = {'x': 'fff'}
jc.submission_parameters['one'] = 1
jc.submission_parameters['two'] = 2
self._check_matching(jc)
def test_non_empty(self):
jc = JobConfig(job_name='TestIngester')
jc.comment = 'Test configuration'
jc.target_pe_count = 2
jco = self._check_overlays(jc)
self.assertIn('jobConfig', jco)
self.assertIn('jobName', jco['jobConfig'])
self.assertEqual('TestIngester', jco['jobConfig']['jobName'])
def test_non_empty(self):
jc = JobConfig(job_name='TestIngester')
jc.comment = 'Test configuration'
jc.target_pe_count = 2
jco = self._check_overlays(jc)
self.assertIn('jobConfig', jco)
self.assertIn('jobName', jco['jobConfig'])
self.assertEqual('TestIngester', jco['jobConfig']['jobName'])
def test_overwrite(self):
jc = JobConfig('Overwrite')
raw = {}
raw['jobConfig'] = {'jobName': 'myjob72', 'jobGroup': 'gg'}
raw['deploymentConfig'] = {'fusionScheme': 'dummy', 'other': 'xx'}
raw['unknown'] = {'hasdsd': 32532}
jc.raw_overlay = raw
jc.target_pe_count = 93
self.assertEqual('Overwrite', jc.job_name)
self.assertEqual(93, jc.target_pe_count)
gc = {}
jc._add_overlays(gc) #type: ignore
self.assertTrue('jobConfigOverlays' in gc)
jcol = gc['jobConfigOverlays']
self.assertIsInstance(jcol, list)
self.assertEqual(1, len(jcol))
jco = jcol[0]
self.assertIsInstance(jco, dict)
self.assertEqual(3, len(jco))
# test unknown value copied as-is
self.assertTrue('unknown' in jco)
self.assertEqual(jc.raw_overlay['unknown'], jco['unknown'])
# test overwrite/merge of job config
self.assertTrue('jobConfig' in jco)
jjc = jco['jobConfig']
self.assertIsInstance(jjc, dict)
self.assertEqual(2, len(jjc))
self.assertEqual('Overwrite', jjc.get('jobName'))
self.assertEqual('gg', jjc.get('jobGroup'))
# test overwrite/merge of deployment config
self.assertTrue('deploymentConfig' in jco)
dc = jco['deploymentConfig']
self.assertIsInstance(dc, dict)
self.assertEqual(3, len(dc))
self.assertEqual('xx', dc.get('other'))
self.assertEqual('manual', dc.get('fusionScheme'))
self.assertEqual(93, dc.get('fusionTargetPeCount'))
def test_overwrite(self):
jc = JobConfig('Overwrite')
raw = {}
raw['jobConfig'] = {'jobName': 'myjob72', 'jobGroup': 'gg'}
raw['deploymentConfig'] = {'fusionScheme': 'dummy', 'other': 'xx'}
raw['unknown'] = {'hasdsd': 32532}
jc.raw_overlay = raw
jc.target_pe_count = 93
self.assertEqual('Overwrite', jc.job_name)
self.assertEqual(93, jc.target_pe_count)
gc = {}
jc._add_overlays(gc) #type: ignore
self.assertTrue('jobConfigOverlays' in gc)
jcol = gc['jobConfigOverlays']
self.assertIsInstance(jcol, list)
self.assertEqual(1, len(jcol))
jco = jcol[0]
self.assertIsInstance(jco, dict)
self.assertEqual(3, len(jco))
# test unknown value copied as-is
self.assertTrue('unknown' in jco)
self.assertEqual(jc.raw_overlay['unknown'], jco['unknown'])
# test overwrite/merge of job config
self.assertTrue('jobConfig' in jco)
jjc = jco['jobConfig']
self.assertIsInstance(jjc, dict)
self.assertEqual(2, len(jjc))
self.assertEqual('Overwrite', jjc.get('jobName'))
self.assertEqual('gg', jjc.get('jobGroup'))
# test overwrite/merge of deployment config
self.assertTrue('deploymentConfig' in jco)
dc = jco['deploymentConfig']
self.assertIsInstance(dc, dict)
self.assertEqual(3, len(dc))
self.assertEqual('xx', dc.get('other'))
self.assertEqual('manual', dc.get('fusionScheme'))
self.assertEqual(93, dc.get('fusionTargetPeCount'))
def test_merge(self):
jc = JobConfig(job_name='Merge')
jc.target_pe_count = 7
jc.raw_overlay = {'jobConfig': {'jobGroup': 'mygroup82'}}
jc.raw_overlay['deploymentConfig'] = {'threadingModel': 'manual'}
jc.raw_overlay['other'] = {'xx': 'yyy'}
gc = {}
jc._add_overlays(gc) #type: ignore
self.assertTrue('jobConfigOverlays' in gc)
jcol = gc['jobConfigOverlays']
self.assertIsInstance(jcol, list)
self.assertEqual(1, len(jcol))
jco = jcol[0]
self.assertIsInstance(jco, dict)
self.assertEqual(3, len(jco))
# test unknown value copied as-is
self.assertTrue('other' in jco)
self.assertEqual(jc.raw_overlay['other'], jco['other'])
# test merge of job config
self.assertTrue('jobConfig' in jco)
jjc = jco['jobConfig']
self.assertIsInstance(jjc, dict)
self.assertEqual(2, len(jjc))
self.assertEqual('Merge', jjc.get('jobName'))
self.assertEqual('mygroup82', jjc.get('jobGroup'))
# test merge of deployment config
self.assertTrue('deploymentConfig' in jco)
dc = jco['deploymentConfig']
self.assertIsInstance(dc, dict)
self.assertEqual(3, len(dc))
self.assertEqual('manual', dc.get('threadingModel'))
self.assertEqual('manual', dc.get('fusionScheme'))
self.assertEqual(7, dc.get('fusionTargetPeCount'))
def test_merge(self):
jc = JobConfig(job_name='Merge')
jc.target_pe_count = 7
jc.raw_overlay = {'jobConfig': {'jobGroup':'mygroup82'}}
jc.raw_overlay['deploymentConfig'] = {'threadingModel':'manual'}
jc.raw_overlay['other'] = {'xx':'yyy'}
gc = {}
jc._add_overlays(gc) #type: ignore
self.assertTrue('jobConfigOverlays' in gc)
jcol = gc['jobConfigOverlays']
self.assertIsInstance(jcol, list)
self.assertEqual(1, len(jcol))
jco = jcol[0]
self.assertIsInstance(jco, dict)
self.assertEqual(3, len(jco))
# test unknown value copied as-is
self.assertTrue('other' in jco)
self.assertEqual(jc.raw_overlay['other'], jco['other'])
# test merge of job config
self.assertTrue('jobConfig' in jco)
jjc = jco['jobConfig']
self.assertIsInstance(jjc, dict)
self.assertEqual(2, len(jjc))
self.assertEqual('Merge', jjc.get('jobName'))
self.assertEqual('mygroup82', jjc.get('jobGroup'))
# test merge of deployment config
self.assertTrue('deploymentConfig' in jco)
dc = jco['deploymentConfig']
self.assertIsInstance(dc, dict)
self.assertEqual(3, len(dc))
self.assertEqual('manual', dc.get('threadingModel'))
self.assertEqual('manual', dc.get('fusionScheme'))
self.assertEqual(7, dc.get('fusionTargetPeCount'))
def test_from_overlays(self):
self._check_matching(JobConfig())
jc = JobConfig(job_name='TestIngester', preload=True, data_directory='/tmp/a', job_group='gg', tracing='info')
jc.comment = 'Test configuration'
jc.target_pe_count = 2
self._check_matching(jc)
jc = JobConfig(job_name='TestIngester2')
jc.comment = 'Test configuration2'
self._check_matching(jc)
jc = JobConfig(preload=True)
jc.raw_overlay = {'a': 34}
self._check_matching(jc)
jc = JobConfig(preload=True)
jc.raw_overlay = {'x': 'fff'}
jc.submission_parameters['one'] = 1
jc.submission_parameters['two'] = 2
self._check_matching(jc)
