def test_from_overlays(self): self._check_matching(JobConfig()) jc = JobConfig(job_name='TestIngester', preload=True, data_directory='/tmp/a', job_group='gg', tracing='info') jc.comment = 'Test configuration' jc.target_pe_count = 2 self._check_matching(jc) jc = JobConfig(job_name='TestIngester2') jc.comment = 'Test configuration2' self._check_matching(jc) jc = JobConfig(preload=True) jc.raw_overlay = {'a': 34} self._check_matching(jc) jc = JobConfig(preload=True) jc.raw_overlay = {'x': 'fff'} jc.submission_parameters['one'] = 1 jc.submission_parameters['two'] = 2 self._check_matching(jc)
def test_property(self): jc = JobConfig() self.assertIs(None, jc.raw_overlay) raw = {} jc.raw_overlay = raw self.assertIs(raw, jc.raw_overlay) raw['jobConfig'] = {'jobName': 'myjob72'} raw['something'] = {'stuff': 3} self.assertIs(raw, jc.raw_overlay) self.assertEqual('myjob72', jc.raw_overlay['jobConfig']['jobName']) gc = {} jc._add_overlays(gc) #type: ignore self.assertTrue('jobConfigOverlays' in gc) jcol = gc['jobConfigOverlays'] self.assertIsInstance(jcol, list) self.assertEqual(1, len(jcol)) jco = jcol[0] self.assertIsInstance(jco, dict) # Copied as-is self.assertEqual(raw, jco)
def test_property(self): jc = JobConfig() self.assertIs(None, jc.raw_overlay) raw = {} jc.raw_overlay = raw self.assertIs(raw, jc.raw_overlay) raw['jobConfig'] = {'jobName': 'myjob72'} raw['something'] = {'stuff': 3} self.assertIs(raw, jc.raw_overlay) self.assertEqual('myjob72', jc.raw_overlay['jobConfig']['jobName']) gc = {} jc._add_overlays(gc) #type: ignore self.assertTrue('jobConfigOverlays' in gc) jcol = gc['jobConfigOverlays'] self.assertIsInstance(jcol, list) self.assertEqual(1, len(jcol)) jco = jcol[0] self.assertIsInstance(jco, dict) # Copied as-is self.assertEqual(raw, jco)
def _multi_injection(self, raw_overlay=None): topo = Topology() streamsx.spl.toolkit.add_toolkit(topo, TestEmInject._TK) context = _rand_path() name = _rand_path() job_name = _rand_path() s1 = endpoint.inject(topo, name=name+'N1', context=context+'C1', monitor=self._monitor) s2 = endpoint.inject(topo, name=name+'N2', context=context+'C2', monitor=self._monitor) jc = JobConfig() self._job_name = _rand_path() jc.job_name = self._job_name jc.add(self.test_config) # Force separate PEs for the inject operators if raw_overlay: jc.raw_overlay = raw_overlay else: s1.colocate(s2) self._path = '/' + context + 'C1/' + name + 'N1/ports/output/0/inject' self.tester = Tester(topo) self.tester.local_check = self._multi_inject self.tester.contents(s1, [{'seq1':i} for i in range(self.N)]) self.tester.contents(s2, [{'seq2':i} for i in range(self.N)]) self.tester.test(self.test_ctxtype, self.test_config) self._check_no_endpoint()
def test_image_name_image_tag(self): topo = Topology("test_image_name_image_tag") heartbeat = topo.source(lambda: itertools.count()) heartbeat.print() image_name = 'py-tst' image_tag = 'v1.0' cfg = {ConfigParams.SSL_VERIFY: False} jc = JobConfig() jc.raw_overlay = { 'edgeConfig': { 'imageName': image_name, 'imageTag': image_tag, 'pipPackages': ['pandas', 'numpy'], 'rpms': ['atlas-devel'] } } jc.add(cfg) try: submission_result = submit(ContextTypes.EDGE, topo.graph, cfg) print(str(submission_result)) self.assertTrue(submission_result is not None) self.assertTrue(self._is_not_blank(submission_result.image)) self.assertTrue(self._is_not_blank(submission_result.imageDigest)) self.assertTrue(image_name in submission_result.image) self.assertTrue(image_tag in submission_result.image) except RuntimeError as e: print(str(e)) self.skipTest("Skip test, CPD does not support EDGE.")
def test_colocation(self): topo = Topology() s1 = topo.source([], name='S1') s2 = topo.source([], name='S2') s1f = s1.filter(lambda x: True, name='S1F') s2f = s2.filter(lambda x: True, name='S2F') s1e = s1f.for_each(lambda x: None, name='S1E') s2e = s2f.for_each(lambda x: None, name='S2E') # S1 -> S1F -> S1E # S2 -> S2F -> S2E s2e.colocate(s1) # S1(X) -> S1F -> S1E # S2 -> S2F -> S2E(X) s2f.colocate([s2, s1f]) # S1(X) -> S1F(Y) -> S1E # S2(Y) -> S2F(Y) -> S2E(X) s1f.colocate(s1e) # S1(X) -> S1F(Y) -> S1E(Y) # S2(Y) -> S2F(Y) -> S2E(X) beacon = Source(topo, "spl.utility::Beacon", 'tuple<uint64 seq>', params={ 'period': 0.02, 'iterations': 100 }, name='BeaconColo') beacon.seq = beacon.output('IterationCount()') beacon.colocate(s2) # Now a colocation independent of the other sets s3 = topo.source([], name='S3') s3f = s3.filter(lambda x: True, name='S3F') s3e = s3f.for_each(lambda x: None, name='S3E') s3.colocate(s3e) # and then join to an existing set #print("MERGE!!!") s3.colocate(s1) self.tester = Tester(topo) self.tester.local_check = self.check_colocations jc = JobConfig() jc.raw_overlay = {'deploymentConfig': {'fusionScheme': 'legacy'}} jc.add(self.test_config) sr = self.tester.test(self.test_ctxtype, self.test_config)
def test_colocation(self): topo = Topology() s1 = topo.source([], name='S1') s2 = topo.source([], name='S2') s1f = s1.filter(lambda x : True, name='S1F') s2f = s2.filter(lambda x : True, name='S2F') s1e = s1f.for_each(lambda x : None, name='S1E') s2e = s2f.for_each(lambda x : None, name='S2E') # S1 -> S1F -> S1E # S2 -> S2F -> S2E s2e.colocate(s1) # S1(X) -> S1F -> S1E # S2 -> S2F -> S2E(X) s2f.colocate([s2,s1f]) # S1(X) -> S1F(Y) -> S1E # S2(Y) -> S2F(Y) -> S2E(X) s1f.colocate(s1e) # S1(X) -> S1F(Y) -> S1E(Y) # S2(Y) -> S2F(Y) -> S2E(X) beacon = Source(topo, "spl.utility::Beacon", 'tuple<uint64 seq>', params = {'period': 0.02, 'iterations':100}, name = 'BeaconColo') beacon.seq = beacon.output('IterationCount()') beacon.colocate(s2) # Now a colocation independent of the other sets s3 = topo.source([], name='S3') s3f = s3.filter(lambda x : True, name='S3F') s3e = s3f.for_each(lambda x : None, name='S3E') s3.colocate(s3e) # and then join to an existing set #print("MERGE!!!") s3.colocate(s1) self.tester = Tester(topo) self.tester.local_check = self.check_colocations jc = JobConfig() jc.raw_overlay = {'deploymentConfig': {'fusionScheme':'legacy'}} jc.add(self.test_config) sr = self.tester.test(self.test_ctxtype, self.test_config)
def test_from_overlays(self): self._check_matching(JobConfig()) jc = JobConfig(job_name='TestIngester', preload=True, data_directory='/tmp/a', job_group='gg', tracing='info') jc.comment = 'Test configuration' jc.target_pe_count = 2 self._check_matching(jc) jc = JobConfig(job_name='TestIngester2') jc.comment = 'Test configuration2' self._check_matching(jc) jc = JobConfig(preload=True) jc.raw_overlay = {'a': 34} self._check_matching(jc) jc = JobConfig(preload=True) jc.raw_overlay = {'x': 'fff'} jc.submission_parameters['one'] = 1 jc.submission_parameters['two'] = 2 self._check_matching(jc)
def test_overwrite(self): jc = JobConfig('Overwrite') raw = {} raw['jobConfig'] = {'jobName': 'myjob72', 'jobGroup': 'gg'} raw['deploymentConfig'] = {'fusionScheme': 'dummy', 'other': 'xx'} raw['unknown'] = {'hasdsd': 32532} jc.raw_overlay = raw jc.target_pe_count = 93 self.assertEqual('Overwrite', jc.job_name) self.assertEqual(93, jc.target_pe_count) gc = {} jc._add_overlays(gc) #type: ignore self.assertTrue('jobConfigOverlays' in gc) jcol = gc['jobConfigOverlays'] self.assertIsInstance(jcol, list) self.assertEqual(1, len(jcol)) jco = jcol[0] self.assertIsInstance(jco, dict) self.assertEqual(3, len(jco)) # test unknown value copied as-is self.assertTrue('unknown' in jco) self.assertEqual(jc.raw_overlay['unknown'], jco['unknown']) # test overwrite/merge of job config self.assertTrue('jobConfig' in jco) jjc = jco['jobConfig'] self.assertIsInstance(jjc, dict) self.assertEqual(2, len(jjc)) self.assertEqual('Overwrite', jjc.get('jobName')) self.assertEqual('gg', jjc.get('jobGroup')) # test overwrite/merge of deployment config self.assertTrue('deploymentConfig' in jco) dc = jco['deploymentConfig'] self.assertIsInstance(dc, dict) self.assertEqual(3, len(dc)) self.assertEqual('xx', dc.get('other')) self.assertEqual('manual', dc.get('fusionScheme')) self.assertEqual(93, dc.get('fusionTargetPeCount'))
def test_overwrite(self): jc = JobConfig('Overwrite') raw = {} raw['jobConfig'] = {'jobName': 'myjob72', 'jobGroup': 'gg'} raw['deploymentConfig'] = {'fusionScheme': 'dummy', 'other': 'xx'} raw['unknown'] = {'hasdsd': 32532} jc.raw_overlay = raw jc.target_pe_count = 93 self.assertEqual('Overwrite', jc.job_name) self.assertEqual(93, jc.target_pe_count) gc = {} jc._add_overlays(gc) #type: ignore self.assertTrue('jobConfigOverlays' in gc) jcol = gc['jobConfigOverlays'] self.assertIsInstance(jcol, list) self.assertEqual(1, len(jcol)) jco = jcol[0] self.assertIsInstance(jco, dict) self.assertEqual(3, len(jco)) # test unknown value copied as-is self.assertTrue('unknown' in jco) self.assertEqual(jc.raw_overlay['unknown'], jco['unknown']) # test overwrite/merge of job config self.assertTrue('jobConfig' in jco) jjc = jco['jobConfig'] self.assertIsInstance(jjc, dict) self.assertEqual(2, len(jjc)) self.assertEqual('Overwrite', jjc.get('jobName')) self.assertEqual('gg', jjc.get('jobGroup')) # test overwrite/merge of deployment config self.assertTrue('deploymentConfig' in jco) dc = jco['deploymentConfig'] self.assertIsInstance(dc, dict) self.assertEqual(3, len(dc)) self.assertEqual('xx', dc.get('other')) self.assertEqual('manual', dc.get('fusionScheme')) self.assertEqual(93, dc.get('fusionTargetPeCount'))
def test_merge(self): jc = JobConfig(job_name='Merge') jc.target_pe_count = 7 jc.raw_overlay = {'jobConfig': {'jobGroup': 'mygroup82'}} jc.raw_overlay['deploymentConfig'] = {'threadingModel': 'manual'} jc.raw_overlay['other'] = {'xx': 'yyy'} gc = {} jc._add_overlays(gc) #type: ignore self.assertTrue('jobConfigOverlays' in gc) jcol = gc['jobConfigOverlays'] self.assertIsInstance(jcol, list) self.assertEqual(1, len(jcol)) jco = jcol[0] self.assertIsInstance(jco, dict) self.assertEqual(3, len(jco)) # test unknown value copied as-is self.assertTrue('other' in jco) self.assertEqual(jc.raw_overlay['other'], jco['other']) # test merge of job config self.assertTrue('jobConfig' in jco) jjc = jco['jobConfig'] self.assertIsInstance(jjc, dict) self.assertEqual(2, len(jjc)) self.assertEqual('Merge', jjc.get('jobName')) self.assertEqual('mygroup82', jjc.get('jobGroup')) # test merge of deployment config self.assertTrue('deploymentConfig' in jco) dc = jco['deploymentConfig'] self.assertIsInstance(dc, dict) self.assertEqual(3, len(dc)) self.assertEqual('manual', dc.get('threadingModel')) self.assertEqual('manual', dc.get('fusionScheme')) self.assertEqual(7, dc.get('fusionTargetPeCount'))
def test_merge(self): jc = JobConfig(job_name='Merge') jc.target_pe_count = 7 jc.raw_overlay = {'jobConfig': {'jobGroup':'mygroup82'}} jc.raw_overlay['deploymentConfig'] = {'threadingModel':'manual'} jc.raw_overlay['other'] = {'xx':'yyy'} gc = {} jc._add_overlays(gc) #type: ignore self.assertTrue('jobConfigOverlays' in gc) jcol = gc['jobConfigOverlays'] self.assertIsInstance(jcol, list) self.assertEqual(1, len(jcol)) jco = jcol[0] self.assertIsInstance(jco, dict) self.assertEqual(3, len(jco)) # test unknown value copied as-is self.assertTrue('other' in jco) self.assertEqual(jc.raw_overlay['other'], jco['other']) # test merge of job config self.assertTrue('jobConfig' in jco) jjc = jco['jobConfig'] self.assertIsInstance(jjc, dict) self.assertEqual(2, len(jjc)) self.assertEqual('Merge', jjc.get('jobName')) self.assertEqual('mygroup82', jjc.get('jobGroup')) # test merge of deployment config self.assertTrue('deploymentConfig' in jco) dc = jco['deploymentConfig'] self.assertIsInstance(dc, dict) self.assertEqual(3, len(dc)) self.assertEqual('manual', dc.get('threadingModel')) self.assertEqual('manual', dc.get('fusionScheme')) self.assertEqual(7, dc.get('fusionTargetPeCount'))