def test_env(env): # Force asserts to show the full file when failures occur. # Useful to debug errors that arise. # Must import the data into the project. with signac.TemporaryProject(name=gen.PROJECT_NAME) as p: fp = gen.get_masked_flowproject(p) fp.import_from(origin=gen.ARCHIVE_DIR) jobs = fp.find_jobs(dict(environment=_env_name(env))) if not len(jobs): raise RuntimeError( "No reference data for environment {}!".format(_env_name(env)) ) reference = [] generated = [] for job in jobs: parameters = job.sp.parameters() if 'bundle' in parameters: bundle = parameters.pop('bundle') tmp_out = io.TextIOWrapper( io.BytesIO(), sys.stdout.encoding) with open(os.devnull, 'w') as devnull: with redirect_stderr(devnull): with redirect_stdout(tmp_out): fp.submit( env=env, jobs=[job], names=bundle, pretend=True, force=True, bundle_size=len(bundle), **parameters) tmp_out.seek(0) msg = "---------- Bundled submission of job {}".format(job) generated.extend([msg] + tmp_out.read().splitlines()) with open(job.fn('script_{}.sh'.format('_'.join(bundle)))) as file: reference.extend([msg] + file.read().splitlines()) else: for op in fp.operations: if 'partition' in parameters: # Don't try to submit GPU operations to CPU partitions # and vice versa. We should be able to relax this # requirement if we make our error checking more # consistent. if operator.xor( 'gpu' in parameters['partition'].lower(), 'gpu' in op.lower()): continue tmp_out = io.TextIOWrapper( io.BytesIO(), sys.stdout.encoding) with open(os.devnull, 'w') as devnull: with redirect_stderr(devnull): with redirect_stdout(tmp_out): fp.submit( env=env, jobs=[job], names=[op], pretend=True, force=True, **parameters) tmp_out.seek(0) msg = "---------- Submission of operation {} for job {}.".format(op, job) generated.extend([msg] + tmp_out.read().splitlines()) with open(job.fn('script_{}.sh'.format(op))) as file: reference.extend([msg] + file.read().splitlines()) assert '\n'.join(reference) == '\n'.join(generated)
def test_submit_status(self): env = get_environment() sched = env.scheduler_type() sched.reset() project = self.mock_project() for job in project: list(project.classify(job)) self.assertEqual(project.next_operation(job).name, 'a_op') self.assertEqual(project.next_operation(job).job, job) with suspend_logging(): with redirect_stdout(StringIO()): project.submit(env) self.assertEqual(len(list(sched.jobs())), len(project)) for job in project: self.assertEqual( project.next_operation(job).get_status(), JobStatus.submitted) sched.step() sched.step() project._fetch_scheduler_status(file=StringIO()) for job in project: self.assertEqual( project.next_operation(job).get_status(), JobStatus.queued)
def test_submit_test_submission_script(self): env = get_environment(test=True) sscript = env.script(a=0) sscript.seek(0) tmp_out = StringIO() with redirect_stdout(tmp_out): env.submit(sscript, hold=True) tmp_out.seek(0)
def test_print_status(self): project = self.mock_project() for job in project: list(project.classify(job)) self.assertEqual(project.next_operation(job).name, 'a_op') self.assertEqual(project.next_operation(job).job, job) fd = StringIO() with redirect_stderr(StringIO()): with redirect_stdout(StringIO()): project.print_status(file=fd, err=fd)
def test_submit(self): env = get_environment() sched = env.scheduler_type() sched.reset() project = self.mock_project() self.assertEqual(len(list(sched.jobs())), 0) with suspend_logging(): with redirect_stdout(StringIO()): project.submit(env) self.assertEqual(len(list(sched.jobs())), len(project)) sched.reset()
def test_single_submit(self): env = get_environment() env.scheduler_type.reset() self.assertTrue(issubclass(env, MockEnvironment)) sscript = env.script() with suspend_logging(): with redirect_stdout(StringIO()): env.submit(sscript, _id='test') scheduler = env.get_scheduler() self.assertEqual(len(list(scheduler.jobs())), 1) for job in scheduler.jobs(): self.assertEqual(job.status(), JobStatus.submitted)
def test_environment_get_config_value(self): env = get_environment(test=True) with redirect_stdout(StringIO()): with pytest.raises(ConfigKeyError): a = env.get_config_value("a") a = env.get_config_value("a", None) assert a is None a = env.get_config_value("a", 42) assert a == 42
def test_environment_get_config_value(self): env = get_environment(test=True) with redirect_stdout(StringIO()): with self.assertRaises(ConfigKeyError): a = env.get_config_value('a') a = env.get_config_value('a', None) self.assertIsNone(a) a = env.get_config_value('a', 42) self.assertEqual(a, 42)
def test_resubmit(self): env = get_environment() sched = env.scheduler_type() sched.reset() project = self.mock_project() self.assertEqual(len(list(sched.jobs())), 0) with suspend_logging(): with redirect_stdout(StringIO()): project.submit(env) for i in range(5): # push all jobs through the queue self.assertEqual(len(list(sched.jobs())), len(project)) project.submit(env) sched.step() self.assertEqual(len(list(sched.jobs())), 0)
def main(args): # If the ARCHIVE_DIR already exists, only recreate if forced. if os.path.exists(ARCHIVE_DIR): if args.force: print("Removing existing archive '{}'.".format(ARCHIVE_DIR)) os.unlink(ARCHIVE_DIR) else: print("Archive '{}' already exists, exiting. " "Use `-f/--force` to overwrite.".format(ARCHIVE_DIR)) return # NOTE: We should replace the below line with # with signac.TemporaryProject(name=PROJECT_NAME, cls=TestProject) as fp: # once the next version of signac is released, and we can then remove # the additional FlowProject instantiation below with signac.TemporaryProject(name=PROJECT_NAME) as p: init(p) fp = get_masked_flowproject(p) for job in fp: with job: kwargs = job.statepoint() env = get_nested_attr(flow, kwargs['environment']) parameters = kwargs['parameters'] if 'bundle' in parameters: bundle = parameters.pop('bundle') fn = 'script_{}.sh'.format('_'.join(bundle)) tmp_out = io.TextIOWrapper(io.BytesIO(), sys.stdout.encoding) with redirect_stdout(tmp_out): try: fp.submit(env=env, jobs=[job], names=bundle, pretend=True, force=True, bundle_size=len(bundle), **parameters) except jinja2.TemplateError as e: print('ERROR:', e) # Shows template error in output script # Filter out non-header lines tmp_out.seek(0) with open(fn, 'w') as f: with redirect_stdout(f): print(tmp_out.read(), end='') else: for op in fp.operations: if 'partition' in parameters: # Don't try to submit GPU operations to CPU partitions # and vice versa. We should be able to relax this # requirement if we make our error checking more # consistent. if operator.xor( 'gpu' in parameters['partition'].lower(), 'gpu' in op.lower()): continue fn = 'script_{}.sh'.format(op) tmp_out = io.TextIOWrapper(io.BytesIO(), sys.stdout.encoding) with redirect_stdout(tmp_out): try: fp.submit(env=env, jobs=[job], names=[op], pretend=True, force=True, **parameters) except jinja2.TemplateError as e: print( 'ERROR:', e) # Shows template error in output script # Filter out non-header lines and the job-name line tmp_out.seek(0) with open(fn, 'w') as f: with redirect_stdout(f): print(tmp_out.read(), end='') # For compactness, we move the output into an ARCHIVE_DIR then delete the original data. fp.export_to(target=ARCHIVE_DIR)
def test_env(env, monkeypatch): monkeypatch.setattr(flow.FlowProject, "_store_bundled", gen._store_bundled) # Force asserts to show the full file when failures occur. # Useful to debug errors that arise. # Must import the data into the project. with signac.TemporaryProject(name=gen.PROJECT_NAME) as p: fp = gen.get_masked_flowproject(p) # Here we set the appropriate executable for all the operations. This # is necessary as otherwise the default executable between submitting # and running could look different depending on the environment. executable = "/usr/local/bin/python" for group in fp.groups.values(): for op_key in group.operations: if op_key in group.operation_directives: group.operation_directives[op_key][ "executable"] = executable fp.import_from(origin=gen.ARCHIVE_DIR) jobs = fp.find_jobs(dict(environment=_env_name(env))) if not len(jobs): raise RuntimeError("No reference data for environment {}!".format( _env_name(env))) reference = [] generated = [] for job in jobs: parameters = job.sp.parameters() if "bundle" in parameters: bundle = parameters.pop("bundle") tmp_out = io.TextIOWrapper(io.BytesIO(), sys.stdout.encoding) with open(os.devnull, "w") as devnull: with redirect_stderr(devnull): with redirect_stdout(tmp_out): fp.submit( env=env, jobs=[job], names=bundle, pretend=True, force=True, bundle_size=len(bundle), **parameters, ) tmp_out.seek(0) msg = f"---------- Bundled submission of job {job}" generated.extend([msg] + tmp_out.read().splitlines()) with open(job.fn("script_{}.sh".format( "_".join(bundle)))) as file: reference.extend([msg] + file.read().splitlines()) else: for op in {**fp.operations, **fp.groups}: if "partition" in parameters: # Don't try to submit GPU operations to CPU partitions # and vice versa. We should be able to relax this # requirement if we make our error checking more # consistent. if operator.xor( "gpu" in parameters["partition"].lower(), "gpu" in op.lower(), ): continue tmp_out = io.TextIOWrapper(io.BytesIO(), sys.stdout.encoding) with open(os.devnull, "w") as devnull: with redirect_stderr(devnull): with redirect_stdout(tmp_out): fp.submit( env=env, jobs=[job], names=[op], pretend=True, force=True, **parameters, ) tmp_out.seek(0) msg = f"---------- Submission of operation {op} for job {job}." generated.extend([msg] + tmp_out.read().splitlines()) with open(job.fn(f"script_{op}.sh")) as file: reference.extend([msg] + file.read().splitlines()) assert "\n".join(reference) == "\n".join(generated)
def test_env(env, monkeypatch): monkeypatch.setattr(flow.FlowProject, "_store_bundled", gen._store_bundled) # We need to set the scheduler manually. The FakeScheduler is used for two # reasons. First, the FakeScheduler prints scripts to screen on submission # and we can capture that output. Second, the FakeScheduler won't try to # call any cluster executable (e.g. squeue) associated with the real # schedulers used on supported clusters. Otherwise submission would fail # when attempting to determine what jobs already exist on the scheduler. monkeypatch.setattr(env, "scheduler_type", FakeScheduler) # Force asserts to show the full file when failures occur. # Useful to debug errors that arise. # Must import the data into the project. with signac.TemporaryProject(name=gen.PROJECT_NAME) as p: with gen.get_masked_flowproject(p, environment=env) as fp: # Here we set the appropriate executable for all the operations. This # is necessary as otherwise the default executable between submitting # and running could look different depending on the environment. for group in fp.groups.values(): for op_key in group.operations: if op_key in group.operation_directives: monkeypatch.setitem( group.operation_directives[op_key], "executable", gen.MOCK_EXECUTABLE, ) fp.import_from(origin=gen.ARCHIVE_DIR) jobs = fp.find_jobs(dict(environment=_env_name(env))) if not len(jobs): raise RuntimeError( "No reference data for environment {}!".format( _env_name(env))) reference = [] generated = [] for job in jobs: parameters = job.sp.parameters() if "bundle" in parameters: bundle = parameters.pop("bundle") tmp_out = io.TextIOWrapper(io.BytesIO(), sys.stdout.encoding) with open(os.devnull, "w") as devnull: with redirect_stderr(devnull): with redirect_stdout(tmp_out): fp.submit( jobs=[job], names=bundle, pretend=True, force=True, bundle_size=len(bundle), **parameters, ) tmp_out.seek(0) msg = f"---------- Bundled submission of job {job}" generated.extend([msg] + tmp_out.read().splitlines()) with open(job.fn("script_{}.sh".format( "_".join(bundle)))) as file: reference.extend([msg] + file.read().splitlines()) else: for op in {**fp.operations, **fp.groups}: if "partition" in parameters: # Don't try to submit GPU operations to CPU partitions # and vice versa. We should be able to relax this # requirement if we make our error checking more # consistent. if operator.xor( "gpu" in parameters["partition"].lower(), "gpu" in op.lower(), ): continue tmp_out = io.TextIOWrapper(io.BytesIO(), sys.stdout.encoding) with open(os.devnull, "w") as devnull: with redirect_stderr(devnull): with redirect_stdout(tmp_out): fp.submit( jobs=[job], names=[op], pretend=True, force=True, **parameters, ) tmp_out.seek(0) msg = f"---------- Submission of operation {op} for job {job}." generated.extend([msg] + tmp_out.read().splitlines()) with open(job.fn(f"script_{op}.sh")) as file: reference.extend([msg] + file.read().splitlines()) assert "\n".join(generated) == "\n".join(reference)
def main(args): # If the ARCHIVE_DIR already exists, only recreate if forced. if os.path.exists(ARCHIVE_DIR): if args.force: print(f"Removing existing archive '{ARCHIVE_DIR}'.") os.unlink(ARCHIVE_DIR) else: print( "Archive '{}' already exists, exiting. " "Use `-f/--force` to overwrite.".format(ARCHIVE_DIR) ) return with signac.TemporaryProject(name=PROJECT_NAME) as p: init(p) fp = get_masked_flowproject(p) for job in fp: with job: kwargs = job.statepoint() env = get_nested_attr(flow, kwargs["environment"]) parameters = kwargs["parameters"] if "bundle" in parameters: bundle = parameters.pop("bundle") fn = "script_{}.sh".format("_".join(bundle)) tmp_out = io.TextIOWrapper(io.BytesIO(), sys.stdout.encoding) with redirect_stdout(tmp_out): try: fp.submit( env=env, jobs=[job], names=bundle, pretend=True, force=True, bundle_size=len(bundle), **parameters, ) except jinja2.TemplateError as e: print("ERROR:", e) # Shows template error in output script # Filter out non-header lines tmp_out.seek(0) with open(fn, "w") as f: with redirect_stdout(f): print(tmp_out.read(), end="") else: for op in {**fp.operations, **fp.groups}: if "partition" in parameters: # Don't try to submit GPU operations to CPU partitions # and vice versa. We should be able to relax this # requirement if we make our error checking more # consistent. if operator.xor( "gpu" in parameters["partition"].lower(), "gpu" in op.lower(), ): continue fn = f"script_{op}.sh" tmp_out = io.TextIOWrapper(io.BytesIO(), sys.stdout.encoding) with redirect_stdout(tmp_out): try: fp.submit( env=env, jobs=[job], names=[op], pretend=True, force=True, **parameters, ) except jinja2.TemplateError as e: print( "ERROR:", e ) # Shows template error in output script # Filter out non-header lines and the job-name line tmp_out.seek(0) with open(fn, "w") as f: with redirect_stdout(f): print(tmp_out.read(), end="") # For compactness, we move the output into an ARCHIVE_DIR then delete the original data. fp.export_to(target=ARCHIVE_DIR)
def main(args): # If the ARCHIVE_DIR already exists, only recreate if forced. if os.path.exists(ARCHIVE_DIR): if args.force: print(f"Removing existing archive '{ARCHIVE_DIR}'.") os.unlink(ARCHIVE_DIR) else: print("Archive '{}' already exists, exiting. " "Use `-f/--force` to overwrite.".format(ARCHIVE_DIR)) return with signac.TemporaryProject(name=PROJECT_NAME) as p: init(p) with get_masked_flowproject(p) as fp: # Here we set the appropriate executable for all the operations. This # is necessary as otherwise the default executable between submitting # and running could look different depending on the environment. for group in fp.groups.values(): for op_key in group.operations: if op_key in group.operation_directives: group.operation_directives[op_key][ "executable"] = MOCK_EXECUTABLE for job in fp: with job: kwargs = job.statepoint() env = get_nested_attr(flow, kwargs["environment"]) # We need to set the scheduler manually. The FakeScheduler # is used for two reasons. First, the FakeScheduler prints # scripts to screen on submission and we can capture that # output. Second, the FakeScheduler won't try to call any # cluster executable (e.g. squeue) associated with the real # schedulers used on supported clusters. Otherwise # submission would fail when attempting to determine what # jobs already exist on the scheduler. env.scheduler_type = FakeScheduler fp._environment = env parameters = kwargs["parameters"] if "bundle" in parameters: bundle = parameters.pop("bundle") fn = "script_{}.sh".format("_".join(bundle)) tmp_out = io.TextIOWrapper(io.BytesIO(), sys.stdout.encoding) with redirect_stdout(tmp_out): try: fp.submit( jobs=[job], names=bundle, pretend=True, force=True, bundle_size=len(bundle), **parameters, ) except jinja2.TemplateError as e: print( "ERROR:", e) # Shows template error in output script # Filter out non-header lines tmp_out.seek(0) with open(fn, "w") as f: with redirect_stdout(f): print(tmp_out.read(), end="") else: for op in {**fp.operations, **fp.groups}: if "partition" in parameters: # Don't try to submit GPU operations to CPU partitions # and vice versa. We should be able to relax this # requirement if we make our error checking more # consistent. if operator.xor( "gpu" in parameters["partition"].lower(), "gpu" in op.lower(), ): continue fn = f"script_{op}.sh" tmp_out = io.TextIOWrapper(io.BytesIO(), sys.stdout.encoding) with redirect_stdout(tmp_out): try: fp.submit( jobs=[job], names=[op], pretend=True, force=True, **parameters, ) except jinja2.TemplateError as e: print( "ERROR:", e ) # Shows template error in output script # Filter out non-header lines and the job-name line tmp_out.seek(0) with open(fn, "w") as f: with redirect_stdout(f): print(tmp_out.read(), end="") # For compactness, we move the output into an ARCHIVE_DIR then delete the original data. fp.export_to(target=ARCHIVE_DIR)