def test_gpu_pipeline(self): """ The gpu pipeline from the examples directory """ with tempfile.TemporaryDirectory() as tempdir: output_dir = os.path.join(tempdir, 'outputs') with open( os.path.join( os.path.dirname( os.path.dirname(os.path.dirname(__file__))), 'examples', 'gpu.yaml')) as r: pipeline = yaml.load(r) o = Orchestrator({ **pipeline, **{ 'backend': { 'type': 'Dummy', 'n_workers': 1, 'staging_dir': STAGING_DIR }, 'localization': { 'staging_dir': '/mnt/nfs/canine' } } }) df = o.run_pipeline(output_dir) self.assertTrue((df[('job', 'exit_code')] == '0:0').all()) for i, row in df.iterrows(): self.assertTrue(os.path.isfile(row[('outputs', 'stdout')])) self.assertTrue(os.path.isfile(row[('outputs', 'stderr')])) self.assertEqual(len(df), len(o.job_spec))
def setUpClass(cls): cls.orchestrator = Orchestrator({ 'name': 'canine-unittest', 'backend': { 'type': 'Dummy', 'n_workers': 1, 'staging_dir': STAGING_DIR }, 'inputs': { 'jobIndex': [0, 1, 2, 3, 4], 'common_file': __file__ }, 'script': [ 'echo $(hostname) $jobIndex $common_file', 'touch f1.txt f2.txt f3.txt' ], 'localization': { 'strategy': 'Batched', 'staging_dir': '/mnt/nfs/canine' }, 'outputs': { 'output-glob': ('*.txt', lambda x: len(x)) } }) cls.backend = cls.orchestrator.backend.__enter__() cls.localizer = cls.orchestrator._localizer_type( cls.orchestrator.backend, **cls.orchestrator.localizer_args).__enter__()
def test_big_pipeline(self): """ Tests a large-scale pipeline """ with tempfile.TemporaryDirectory() as tempdir: output_dir = os.path.join(tempdir, 'outputs') o = Orchestrator({ 'adapter': { 'product': True }, 'inputs': { 'i': [*range(25)], 'j': [*range(25)], }, 'script': [ 'python3 -c "[print(i,j) for i in range($i) for j in range($j)]"' ], 'backend': { 'type': 'Dummy', 'n_workers': cpu_count(), 'cpus': 1, 'staging_dir': STAGING_DIR }, 'resources': { 'cpus-per-task': 1, 'mem-per-cpu': '256M' }, 'localization': { 'staging_dir': '/mnt/nfs/canine' } }) df = o.run_pipeline(output_dir) self.assertTrue((df[('job', 'exit_code')] == '0:0').all()) for i, row in df.iterrows(): self.assertTrue(os.path.isfile(row[('outputs', 'stdout')])) self.assertTrue(os.path.isfile(row[('outputs', 'stderr')])) self.assertEqual(len(df), len(o.job_spec))