def test_from_overlays(self):
self._check_matching(JobConfig())
jc = JobConfig(job_name='TestIngester',
preload=True,
data_directory='/tmp/a',
job_group='gg',
tracing='info')
jc.comment = 'Test configuration'
jc.target_pe_count = 2
self._check_matching(jc)
jc = JobConfig(job_name='TestIngester2')
jc.comment = 'Test configuration2'
self._check_matching(jc)
jc = JobConfig(preload=True)
jc.raw_overlay = {'a': 34}
self._check_matching(jc)
jc = JobConfig(preload=True)
jc.raw_overlay = {'x': 'fff'}
jc.submission_parameters['one'] = 1
jc.submission_parameters['two'] = 2
self._check_matching(jc)
def test_property(self):
jc = JobConfig()
self.assertIs(None, jc.raw_overlay)
raw = {}
jc.raw_overlay = raw
self.assertIs(raw, jc.raw_overlay)
raw['jobConfig'] = {'jobName': 'myjob72'}
raw['something'] = {'stuff': 3}
self.assertIs(raw, jc.raw_overlay)
self.assertEqual('myjob72', jc.raw_overlay['jobConfig']['jobName'])
gc = {}
jc._add_overlays(gc) #type: ignore
self.assertTrue('jobConfigOverlays' in gc)
jcol = gc['jobConfigOverlays']
self.assertIsInstance(jcol, list)
self.assertEqual(1, len(jcol))
jco = jcol[0]
self.assertIsInstance(jco, dict)
# Copied as-is
self.assertEqual(raw, jco)
def test_property(self):
jc = JobConfig()
self.assertIs(None, jc.raw_overlay)
raw = {}
jc.raw_overlay = raw
self.assertIs(raw, jc.raw_overlay)
raw['jobConfig'] = {'jobName': 'myjob72'}
raw['something'] = {'stuff': 3}
self.assertIs(raw, jc.raw_overlay)
self.assertEqual('myjob72', jc.raw_overlay['jobConfig']['jobName'])
gc = {}
jc._add_overlays(gc) #type: ignore
self.assertTrue('jobConfigOverlays' in gc)
jcol = gc['jobConfigOverlays']
self.assertIsInstance(jcol, list)
self.assertEqual(1, len(jcol))
jco = jcol[0]
self.assertIsInstance(jco, dict)
# Copied as-is
self.assertEqual(raw, jco)
def _multi_injection(self, raw_overlay=None):
topo = Topology()
streamsx.spl.toolkit.add_toolkit(topo, TestEmInject._TK)
context = _rand_path()
name = _rand_path()
job_name = _rand_path()
s1 = endpoint.inject(topo, name=name+'N1', context=context+'C1', monitor=self._monitor)
s2 = endpoint.inject(topo, name=name+'N2', context=context+'C2', monitor=self._monitor)
jc = JobConfig()
self._job_name = _rand_path()
jc.job_name = self._job_name
jc.add(self.test_config)
# Force separate PEs for the inject operators
if raw_overlay:
jc.raw_overlay = raw_overlay
else:
s1.colocate(s2)
self._path = '/' + context + 'C1/' + name + 'N1/ports/output/0/inject'
self.tester = Tester(topo)
self.tester.local_check = self._multi_inject
self.tester.contents(s1, [{'seq1':i} for i in range(self.N)])
self.tester.contents(s2, [{'seq2':i} for i in range(self.N)])
self.tester.test(self.test_ctxtype, self.test_config)
self._check_no_endpoint()
def test_image_name_image_tag(self):
topo = Topology("test_image_name_image_tag")
heartbeat = topo.source(lambda: itertools.count())
heartbeat.print()
image_name = 'py-tst'
image_tag = 'v1.0'
cfg = {ConfigParams.SSL_VERIFY: False}
jc = JobConfig()
jc.raw_overlay = {
'edgeConfig': {
'imageName': image_name,
'imageTag': image_tag,
'pipPackages': ['pandas', 'numpy'],
'rpms': ['atlas-devel']
}
}
jc.add(cfg)
try:
submission_result = submit(ContextTypes.EDGE, topo.graph, cfg)
print(str(submission_result))
self.assertTrue(submission_result is not None)
self.assertTrue(self._is_not_blank(submission_result.image))
self.assertTrue(self._is_not_blank(submission_result.imageDigest))
self.assertTrue(image_name in submission_result.image)
self.assertTrue(image_tag in submission_result.image)
except RuntimeError as e:
print(str(e))
self.skipTest("Skip test, CPD does not support EDGE.")
def test_colocation(self):
topo = Topology()
s1 = topo.source([], name='S1')
s2 = topo.source([], name='S2')
s1f = s1.filter(lambda x: True, name='S1F')
s2f = s2.filter(lambda x: True, name='S2F')
s1e = s1f.for_each(lambda x: None, name='S1E')
s2e = s2f.for_each(lambda x: None, name='S2E')
# S1 -> S1F -> S1E
# S2 -> S2F -> S2E
s2e.colocate(s1)
# S1(X) -> S1F -> S1E
# S2 -> S2F -> S2E(X)
s2f.colocate([s2, s1f])
# S1(X) -> S1F(Y) -> S1E
# S2(Y) -> S2F(Y) -> S2E(X)
s1f.colocate(s1e)
# S1(X) -> S1F(Y) -> S1E(Y)
# S2(Y) -> S2F(Y) -> S2E(X)
beacon = Source(topo,
"spl.utility::Beacon",
'tuple<uint64 seq>',
params={
'period': 0.02,
'iterations': 100
},
name='BeaconColo')
beacon.seq = beacon.output('IterationCount()')
beacon.colocate(s2)
# Now a colocation independent of the other sets
s3 = topo.source([], name='S3')
s3f = s3.filter(lambda x: True, name='S3F')
s3e = s3f.for_each(lambda x: None, name='S3E')
s3.colocate(s3e)
# and then join to an existing set
#print("MERGE!!!")
s3.colocate(s1)
self.tester = Tester(topo)
self.tester.local_check = self.check_colocations
jc = JobConfig()
jc.raw_overlay = {'deploymentConfig': {'fusionScheme': 'legacy'}}
jc.add(self.test_config)
sr = self.tester.test(self.test_ctxtype, self.test_config)
def test_colocation(self):
topo = Topology()
s1 = topo.source([], name='S1')
s2 = topo.source([], name='S2')
s1f = s1.filter(lambda x : True, name='S1F')
s2f = s2.filter(lambda x : True, name='S2F')
s1e = s1f.for_each(lambda x : None, name='S1E')
s2e = s2f.for_each(lambda x : None, name='S2E')
# S1 -> S1F -> S1E
# S2 -> S2F -> S2E
s2e.colocate(s1)
# S1(X) -> S1F -> S1E
# S2 -> S2F -> S2E(X)
s2f.colocate([s2,s1f])
# S1(X) -> S1F(Y) -> S1E
# S2(Y) -> S2F(Y) -> S2E(X)
s1f.colocate(s1e)
# S1(X) -> S1F(Y) -> S1E(Y)
# S2(Y) -> S2F(Y) -> S2E(X)
beacon = Source(topo, "spl.utility::Beacon",
'tuple<uint64 seq>',
params = {'period': 0.02, 'iterations':100},
name = 'BeaconColo')
beacon.seq = beacon.output('IterationCount()')
beacon.colocate(s2)
# Now a colocation independent of the other sets
s3 = topo.source([], name='S3')
s3f = s3.filter(lambda x : True, name='S3F')
s3e = s3f.for_each(lambda x : None, name='S3E')
s3.colocate(s3e)
# and then join to an existing set
#print("MERGE!!!")
s3.colocate(s1)
self.tester = Tester(topo)
self.tester.local_check = self.check_colocations
jc = JobConfig()
jc.raw_overlay = {'deploymentConfig': {'fusionScheme':'legacy'}}
jc.add(self.test_config)
sr = self.tester.test(self.test_ctxtype, self.test_config)
def test_from_overlays(self):
self._check_matching(JobConfig())
jc = JobConfig(job_name='TestIngester', preload=True, data_directory='/tmp/a', job_group='gg', tracing='info')
jc.comment = 'Test configuration'
jc.target_pe_count = 2
self._check_matching(jc)
jc = JobConfig(job_name='TestIngester2')
jc.comment = 'Test configuration2'
self._check_matching(jc)
jc = JobConfig(preload=True)
jc.raw_overlay = {'a': 34}
self._check_matching(jc)
jc = JobConfig(preload=True)
jc.raw_overlay = {'x': 'fff'}
jc.submission_parameters['one'] = 1
jc.submission_parameters['two'] = 2
self._check_matching(jc)
def test_overwrite(self):
jc = JobConfig('Overwrite')
raw = {}
raw['jobConfig'] = {'jobName': 'myjob72', 'jobGroup': 'gg'}
raw['deploymentConfig'] = {'fusionScheme': 'dummy', 'other': 'xx'}
raw['unknown'] = {'hasdsd': 32532}
jc.raw_overlay = raw
jc.target_pe_count = 93
self.assertEqual('Overwrite', jc.job_name)
self.assertEqual(93, jc.target_pe_count)
gc = {}
jc._add_overlays(gc) #type: ignore
self.assertTrue('jobConfigOverlays' in gc)
jcol = gc['jobConfigOverlays']
self.assertIsInstance(jcol, list)
self.assertEqual(1, len(jcol))
jco = jcol[0]
self.assertIsInstance(jco, dict)
self.assertEqual(3, len(jco))
# test unknown value copied as-is
self.assertTrue('unknown' in jco)
self.assertEqual(jc.raw_overlay['unknown'], jco['unknown'])
# test overwrite/merge of job config
self.assertTrue('jobConfig' in jco)
jjc = jco['jobConfig']
self.assertIsInstance(jjc, dict)
self.assertEqual(2, len(jjc))
self.assertEqual('Overwrite', jjc.get('jobName'))
self.assertEqual('gg', jjc.get('jobGroup'))
# test overwrite/merge of deployment config
self.assertTrue('deploymentConfig' in jco)
dc = jco['deploymentConfig']
self.assertIsInstance(dc, dict)
self.assertEqual(3, len(dc))
self.assertEqual('xx', dc.get('other'))
self.assertEqual('manual', dc.get('fusionScheme'))
self.assertEqual(93, dc.get('fusionTargetPeCount'))
def test_overwrite(self):
jc = JobConfig('Overwrite')
raw = {}
raw['jobConfig'] = {'jobName': 'myjob72', 'jobGroup': 'gg'}
raw['deploymentConfig'] = {'fusionScheme': 'dummy', 'other': 'xx'}
raw['unknown'] = {'hasdsd': 32532}
jc.raw_overlay = raw
jc.target_pe_count = 93
self.assertEqual('Overwrite', jc.job_name)
self.assertEqual(93, jc.target_pe_count)
gc = {}
jc._add_overlays(gc) #type: ignore
self.assertTrue('jobConfigOverlays' in gc)
jcol = gc['jobConfigOverlays']
self.assertIsInstance(jcol, list)
self.assertEqual(1, len(jcol))
jco = jcol[0]
self.assertIsInstance(jco, dict)
self.assertEqual(3, len(jco))
# test unknown value copied as-is
self.assertTrue('unknown' in jco)
self.assertEqual(jc.raw_overlay['unknown'], jco['unknown'])
# test overwrite/merge of job config
self.assertTrue('jobConfig' in jco)
jjc = jco['jobConfig']
self.assertIsInstance(jjc, dict)
self.assertEqual(2, len(jjc))
self.assertEqual('Overwrite', jjc.get('jobName'))
self.assertEqual('gg', jjc.get('jobGroup'))
# test overwrite/merge of deployment config
self.assertTrue('deploymentConfig' in jco)
dc = jco['deploymentConfig']
self.assertIsInstance(dc, dict)
self.assertEqual(3, len(dc))
self.assertEqual('xx', dc.get('other'))
self.assertEqual('manual', dc.get('fusionScheme'))
self.assertEqual(93, dc.get('fusionTargetPeCount'))
def test_merge(self):
jc = JobConfig(job_name='Merge')
jc.target_pe_count = 7
jc.raw_overlay = {'jobConfig': {'jobGroup': 'mygroup82'}}
jc.raw_overlay['deploymentConfig'] = {'threadingModel': 'manual'}
jc.raw_overlay['other'] = {'xx': 'yyy'}
gc = {}
jc._add_overlays(gc) #type: ignore
self.assertTrue('jobConfigOverlays' in gc)
jcol = gc['jobConfigOverlays']
self.assertIsInstance(jcol, list)
self.assertEqual(1, len(jcol))
jco = jcol[0]
self.assertIsInstance(jco, dict)
self.assertEqual(3, len(jco))
# test unknown value copied as-is
self.assertTrue('other' in jco)
self.assertEqual(jc.raw_overlay['other'], jco['other'])
# test merge of job config
self.assertTrue('jobConfig' in jco)
jjc = jco['jobConfig']
self.assertIsInstance(jjc, dict)
self.assertEqual(2, len(jjc))
self.assertEqual('Merge', jjc.get('jobName'))
self.assertEqual('mygroup82', jjc.get('jobGroup'))
# test merge of deployment config
self.assertTrue('deploymentConfig' in jco)
dc = jco['deploymentConfig']
self.assertIsInstance(dc, dict)
self.assertEqual(3, len(dc))
self.assertEqual('manual', dc.get('threadingModel'))
self.assertEqual('manual', dc.get('fusionScheme'))
self.assertEqual(7, dc.get('fusionTargetPeCount'))
def test_merge(self):
jc = JobConfig(job_name='Merge')
jc.target_pe_count = 7
jc.raw_overlay = {'jobConfig': {'jobGroup':'mygroup82'}}
jc.raw_overlay['deploymentConfig'] = {'threadingModel':'manual'}
jc.raw_overlay['other'] = {'xx':'yyy'}
gc = {}
jc._add_overlays(gc) #type: ignore
self.assertTrue('jobConfigOverlays' in gc)
jcol = gc['jobConfigOverlays']
self.assertIsInstance(jcol, list)
self.assertEqual(1, len(jcol))
jco = jcol[0]
self.assertIsInstance(jco, dict)
self.assertEqual(3, len(jco))
# test unknown value copied as-is
self.assertTrue('other' in jco)
self.assertEqual(jc.raw_overlay['other'], jco['other'])
# test merge of job config
self.assertTrue('jobConfig' in jco)
jjc = jco['jobConfig']
self.assertIsInstance(jjc, dict)
self.assertEqual(2, len(jjc))
self.assertEqual('Merge', jjc.get('jobName'))
self.assertEqual('mygroup82', jjc.get('jobGroup'))
# test merge of deployment config
self.assertTrue('deploymentConfig' in jco)
dc = jco['deploymentConfig']
self.assertIsInstance(dc, dict)
self.assertEqual(3, len(dc))
self.assertEqual('manual', dc.get('threadingModel'))
self.assertEqual('manual', dc.get('fusionScheme'))
self.assertEqual(7, dc.get('fusionTargetPeCount'))
