def test_safe_load(self):
"""Safe loader is used by default."""
# Attempt to load an object.
# This should give a ConstructorError because we used the safe loader.
yaml_doc = "!!python/object:int {}"
buf = io.StringIO(yaml_doc)
with self.assertRaises(yaml.constructor.ConstructorError):
custom_yaml.load(buf)
def pipeline_description(pipeline_file):
"""
Load pipeline description file and pre-process the config.
"""
pipe_config = phyre_engine.pipeline.PipelineConfig(
resolve_yml(
load_default_config(),
lambda doc: doc,
{"ENV": os.environ}))
# Parse pipeline descriptor from YAML file
with open(pipeline_file, "r") as yml_in:
pipeline_desc = yaml.load(yml_in, yaml.ImplicitLoader)
# Merge default configuration with the current unresolved config
pipe_config = pipe_config.merge_params(
phyre_engine.pipeline.PipelineConfig(
pipeline_desc.unresolved["pipeline"].get("config", {})))
pipeline_desc.unresolved["pipeline"]["config"] = pipe_config
pipeline_desc = resolve_yml(
pipeline_desc,
lambda doc: doc["pipeline"]["config"],
{"ENV": os.environ})
def _main():
# Capture warnings with the logger
logging.captureWarnings(True)
try:
parser = arg_parser()
args = parser.parse_args()
if args.config is not None:
with open(args.config, "r") as yml_in:
phyre_engine.test.config = yaml.load(yml_in)
loader = unittest.TestLoader()
tests = loader.discover(args.test_dir, args.pattern)
if args.xml:
runner = xmlrunner.XMLTestRunner(output=args.xml)
else:
runner = unittest.TextTestRunner(verbosity=args.verbosity)
results = runner.run(tests)
if results.wasSuccessful():
return 0
return 1
except KeyboardInterrupt:
### handle keyboard interrupt ###
return 0
def test_represent_pipeline_config(self):
"""PipelineConfig objects can be represented as dicts."""
source = {"a": "b"}
conf = phyre_engine.pipeline.PipelineConfig(source)
buf = io.StringIO()
custom_yaml.dump(conf, buf)
buf.seek(0)
result = custom_yaml.load(buf)
self.assertEqual(result, source)
def load_default_config():
"""
Load default configuration file, if any. This is pre-processed in the
same way as the main pipeline configuration.
"""
config_file = default_config_file()
if not config_file.exists():
return {}
else:
with config_file.open("r") as conf_in:
return yaml.load(conf_in, yaml.ImplicitLoader)
def main(): # IGNORE:C0111
'''Command line options.'''
try:
logging.setLoggerClass(phyre_engine.logutils.PhyreEngineLogger)
parser = arg_parser()
args = parser.parse_args()
with open(args.pipeline, "r") as pipeline_fh:
pipeline_dict = yaml.load(pipeline_fh)
pipeline = phyre_engine.pipeline.Pipeline.load(pipeline_dict)
with open(args.state, "r+b") as state_fh:
state = pickle.load(state_fh)
# Initialise loggers
if "logging" in pipeline.config:
log_config = pipeline.config["logging"]
else:
log_config = phyre_engine.run.default_log_config()
if "disable_existing_loggers" not in log_config:
log_config["disable_existing_loggers"] = False
logging.config.dictConfig(log_config)
pipeline.start = state
data = pipeline.run()
data["qsub_complete"] = True
state_fh.seek(0)
state_fh.truncate()
pickle.dump(data, state_fh)
return 0
except KeyboardInterrupt:
### handle keyboard interrupt ###
return 0
except Exception as error:
logging.getLogger(ROOT_LOGGER).error(
"Uncaught exception encountered: exiting.",
exc_info=error)
raise error
def test_load_tuple(self):
"""Tuples supported as dictionary keys."""
yaml_doc = "[1, 2]: foo"
buf = io.StringIO(yaml_doc)
self.assertEqual(custom_yaml.load(buf), {(1, 2): "foo"})
def main(): # IGNORE:C0111
'''Command line options.'''
# Before doing anything, tell the loggers to capture warnings.
logging.captureWarnings(True)
try:
parser = arg_parser()
args = parser.parse_args()
if args.dump:
return 0
pipe_config = phyre_engine.pipeline.PipelineConfig(
resolve_yml(
load_default_config(),
lambda doc: doc,
{"ENV": os.environ}))
# Parse pipeline descriptor from YAML file
with open(args.pipeline, "r") as yml_in:
pipeline_desc = yaml.load(yml_in, yaml.ImplicitLoader)
# Merge default configuration with the current unresolved config
pipe_config = pipe_config.merge_params(
phyre_engine.pipeline.PipelineConfig(
pipeline_desc.unresolved["pipeline"].get("config", {})))
pipeline_desc.unresolved["pipeline"]["config"] = pipe_config
pipeline_desc = resolve_yml(
pipeline_desc,
lambda doc: doc["pipeline"]["config"],
{"ENV": os.environ})
# Set default start values"
pipeline_desc["pipeline"].setdefault("start", {})
if args.start is not None:
pipeline_desc["pipeline"]["start"].update(args.start)
# Set up logging if a logging section was given in the pipeline
init_logging(pipeline_desc["pipeline"]["config"])
# Add extra configuration keys to the pipeline config
for dotted_key, value in args.config.items():
apply_dotted_key(pipeline_desc["pipeline"]["config"],
dotted_key, value)
# Load a pipeline
pipeline = phyre_engine.pipeline.Pipeline.load(
pipeline_desc["pipeline"])
# Check that the parameters each component requires are present. This
# is non-deterministic, so this can be silenced with the
# --no-static-check command-line option.
if not (args.no_static_check
or pipeline.config.get("no_static_check", False)):
static_validate(pipeline)
pipeline.run()
return 0
except KeyboardInterrupt:
### handle keyboard interrupt ###
return 0
except phyre_engine.pipeline.Pipeline.ValidationError as error:
logging.getLogger(ROOT_LOGGER).error(
"Component %s expected the missing keys %s in the pipeline state",
error.component.qualname, error.missing)
except Exception as error:
logging.getLogger(ROOT_LOGGER).error(
"Uncaught exception encountered: exiting.",
exc_info=error)
raise error