def test_from_overlays(self): self._check_matching(JobConfig()) jc = JobConfig(job_name='TestIngester', preload=True, data_directory='/tmp/a', job_group='gg', tracing='info') jc.comment = 'Test configuration' jc.target_pe_count = 2 self._check_matching(jc) jc = JobConfig(job_name='TestIngester2') jc.comment = 'Test configuration2' self._check_matching(jc) jc = JobConfig(preload=True) jc.raw_overlay = {'a': 34} self._check_matching(jc) jc = JobConfig(preload=True) jc.raw_overlay = {'x': 'fff'} jc.submission_parameters['one'] = 1 jc.submission_parameters['two'] = 2 self._check_matching(jc)
def test_non_empty(self): jc = JobConfig(job_name='TestIngester') jc.comment = 'Test configuration' jc.target_pe_count = 2 jco = self._check_overlays(jc) self.assertIn('jobConfig', jco) self.assertIn('jobName', jco['jobConfig']) self.assertEqual('TestIngester', jco['jobConfig']['jobName'])
def test_non_empty(self): jc = JobConfig(job_name='TestIngester') jc.comment = 'Test configuration' jc.target_pe_count = 2 jco = self._check_overlays(jc) self.assertIn('jobConfig', jco) self.assertIn('jobName', jco['jobConfig']) self.assertEqual('TestIngester', jco['jobConfig']['jobName'])
def test_overwrite(self): jc = JobConfig('Overwrite') raw = {} raw['jobConfig'] = {'jobName': 'myjob72', 'jobGroup': 'gg'} raw['deploymentConfig'] = {'fusionScheme': 'dummy', 'other': 'xx'} raw['unknown'] = {'hasdsd': 32532} jc.raw_overlay = raw jc.target_pe_count = 93 self.assertEqual('Overwrite', jc.job_name) self.assertEqual(93, jc.target_pe_count) gc = {} jc._add_overlays(gc) #type: ignore self.assertTrue('jobConfigOverlays' in gc) jcol = gc['jobConfigOverlays'] self.assertIsInstance(jcol, list) self.assertEqual(1, len(jcol)) jco = jcol[0] self.assertIsInstance(jco, dict) self.assertEqual(3, len(jco)) # test unknown value copied as-is self.assertTrue('unknown' in jco) self.assertEqual(jc.raw_overlay['unknown'], jco['unknown']) # test overwrite/merge of job config self.assertTrue('jobConfig' in jco) jjc = jco['jobConfig'] self.assertIsInstance(jjc, dict) self.assertEqual(2, len(jjc)) self.assertEqual('Overwrite', jjc.get('jobName')) self.assertEqual('gg', jjc.get('jobGroup')) # test overwrite/merge of deployment config self.assertTrue('deploymentConfig' in jco) dc = jco['deploymentConfig'] self.assertIsInstance(dc, dict) self.assertEqual(3, len(dc)) self.assertEqual('xx', dc.get('other')) self.assertEqual('manual', dc.get('fusionScheme')) self.assertEqual(93, dc.get('fusionTargetPeCount'))
def test_overwrite(self): jc = JobConfig('Overwrite') raw = {} raw['jobConfig'] = {'jobName': 'myjob72', 'jobGroup': 'gg'} raw['deploymentConfig'] = {'fusionScheme': 'dummy', 'other': 'xx'} raw['unknown'] = {'hasdsd': 32532} jc.raw_overlay = raw jc.target_pe_count = 93 self.assertEqual('Overwrite', jc.job_name) self.assertEqual(93, jc.target_pe_count) gc = {} jc._add_overlays(gc) #type: ignore self.assertTrue('jobConfigOverlays' in gc) jcol = gc['jobConfigOverlays'] self.assertIsInstance(jcol, list) self.assertEqual(1, len(jcol)) jco = jcol[0] self.assertIsInstance(jco, dict) self.assertEqual(3, len(jco)) # test unknown value copied as-is self.assertTrue('unknown' in jco) self.assertEqual(jc.raw_overlay['unknown'], jco['unknown']) # test overwrite/merge of job config self.assertTrue('jobConfig' in jco) jjc = jco['jobConfig'] self.assertIsInstance(jjc, dict) self.assertEqual(2, len(jjc)) self.assertEqual('Overwrite', jjc.get('jobName')) self.assertEqual('gg', jjc.get('jobGroup')) # test overwrite/merge of deployment config self.assertTrue('deploymentConfig' in jco) dc = jco['deploymentConfig'] self.assertIsInstance(dc, dict) self.assertEqual(3, len(dc)) self.assertEqual('xx', dc.get('other')) self.assertEqual('manual', dc.get('fusionScheme')) self.assertEqual(93, dc.get('fusionTargetPeCount'))
def test_merge(self): jc = JobConfig(job_name='Merge') jc.target_pe_count = 7 jc.raw_overlay = {'jobConfig': {'jobGroup': 'mygroup82'}} jc.raw_overlay['deploymentConfig'] = {'threadingModel': 'manual'} jc.raw_overlay['other'] = {'xx': 'yyy'} gc = {} jc._add_overlays(gc) #type: ignore self.assertTrue('jobConfigOverlays' in gc) jcol = gc['jobConfigOverlays'] self.assertIsInstance(jcol, list) self.assertEqual(1, len(jcol)) jco = jcol[0] self.assertIsInstance(jco, dict) self.assertEqual(3, len(jco)) # test unknown value copied as-is self.assertTrue('other' in jco) self.assertEqual(jc.raw_overlay['other'], jco['other']) # test merge of job config self.assertTrue('jobConfig' in jco) jjc = jco['jobConfig'] self.assertIsInstance(jjc, dict) self.assertEqual(2, len(jjc)) self.assertEqual('Merge', jjc.get('jobName')) self.assertEqual('mygroup82', jjc.get('jobGroup')) # test merge of deployment config self.assertTrue('deploymentConfig' in jco) dc = jco['deploymentConfig'] self.assertIsInstance(dc, dict) self.assertEqual(3, len(dc)) self.assertEqual('manual', dc.get('threadingModel')) self.assertEqual('manual', dc.get('fusionScheme')) self.assertEqual(7, dc.get('fusionTargetPeCount'))
def test_merge(self): jc = JobConfig(job_name='Merge') jc.target_pe_count = 7 jc.raw_overlay = {'jobConfig': {'jobGroup':'mygroup82'}} jc.raw_overlay['deploymentConfig'] = {'threadingModel':'manual'} jc.raw_overlay['other'] = {'xx':'yyy'} gc = {} jc._add_overlays(gc) #type: ignore self.assertTrue('jobConfigOverlays' in gc) jcol = gc['jobConfigOverlays'] self.assertIsInstance(jcol, list) self.assertEqual(1, len(jcol)) jco = jcol[0] self.assertIsInstance(jco, dict) self.assertEqual(3, len(jco)) # test unknown value copied as-is self.assertTrue('other' in jco) self.assertEqual(jc.raw_overlay['other'], jco['other']) # test merge of job config self.assertTrue('jobConfig' in jco) jjc = jco['jobConfig'] self.assertIsInstance(jjc, dict) self.assertEqual(2, len(jjc)) self.assertEqual('Merge', jjc.get('jobName')) self.assertEqual('mygroup82', jjc.get('jobGroup')) # test merge of deployment config self.assertTrue('deploymentConfig' in jco) dc = jco['deploymentConfig'] self.assertIsInstance(dc, dict) self.assertEqual(3, len(dc)) self.assertEqual('manual', dc.get('threadingModel')) self.assertEqual('manual', dc.get('fusionScheme')) self.assertEqual(7, dc.get('fusionTargetPeCount'))
def test_from_overlays(self): self._check_matching(JobConfig()) jc = JobConfig(job_name='TestIngester', preload=True, data_directory='/tmp/a', job_group='gg', tracing='info') jc.comment = 'Test configuration' jc.target_pe_count = 2 self._check_matching(jc) jc = JobConfig(job_name='TestIngester2') jc.comment = 'Test configuration2' self._check_matching(jc) jc = JobConfig(preload=True) jc.raw_overlay = {'a': 34} self._check_matching(jc) jc = JobConfig(preload=True) jc.raw_overlay = {'x': 'fff'} jc.submission_parameters['one'] = 1 jc.submission_parameters['two'] = 2 self._check_matching(jc)