def test_unknown_parse():
cfg = parse('[{ city_name: Constantinople }]')
lint_result = cfg.lint()
assert lint_result.warning_count == 1
assert list(
lint_result.warnings
)[0]['message'] == "No parser for {'city_name': 'Constantinople'}"
def test_yaml_update_from_source(tmpdir, original_yaml, source_python, expected_yaml):
yaml_path = os.path.join(tmpdir, "valohai.yaml")
source_path = os.path.join(tmpdir, "test.py")
# Build repository with test.py and valohai.yaml
if os.path.isfile(original_yaml):
shutil.copy(original_yaml, yaml_path)
shutil.copy(source_python, source_path)
# Load original valohai.yaml
old_config = None
if os.path.isfile(yaml_path):
with open(yaml_path) as yaml_file:
old_config = parse(yaml_file)
# Parse new config from .py
new_config = parse_config_from_source(source_path, yaml_path)
# Merge original and new
if old_config:
new_config = old_config.merge_with(new_config, python_to_yaml_merge_strategy)
# Check against expected result
with open(expected_yaml) as expected_yaml:
new_yaml = config_to_yaml(new_config)
assert new_yaml == expected_yaml.read()
def _parse_config(self, config_fp: TextIO, filename: str = '<config file>') -> Config:
try:
return valohai_yaml.parse(config_fp)
except OSError as err:
raise InvalidConfig(f'Could not read {filename}') from err
except valohai_yaml.ValidationErrors as ves:
raise InvalidConfig(f'{filename} is invalid ({len(ves.errors)} errors); see `vh lint`')
def test_yaml_update_from_source(tmpdir, original_yaml, source_python,
expected_yaml_filename):
yaml_path = os.path.join(tmpdir, "valohai.yaml")
filename, file_extension = os.path.splitext(source_python)
source_path = os.path.join(tmpdir, f"test{file_extension}")
# Build repository with test.py and valohai.yaml
if os.path.isfile(original_yaml):
shutil.copy(original_yaml, yaml_path)
shutil.copy(source_python, source_path)
# Load original valohai.yaml
old_config = None
if os.path.isfile(yaml_path):
with open(yaml_path) as yaml_file:
old_config = parse(yaml_file)
# Parse new config from .py
new_config = parse_config_from_source(source_path, yaml_path)
# Merge original and new
if old_config:
new_config = old_config.merge_with(new_config,
python_to_yaml_merge_strategy)
# Check against expected result
with open(expected_yaml_filename) as fp:
compare_yaml(new_config, fp.read())
def _parse_config(self, config_fp, filename='<config file>'):
try:
config = valohai_yaml.parse(config_fp)
config.project = self
return config
except IOError as ioe:
six.raise_from(InvalidConfig('Could not read %s' % filename), ioe)
except valohai_yaml.ValidationErrors as ves:
raise InvalidConfig(
'{filename} is invalid ({n} errors); see `vh lint`'.format(
filename=filename,
n=len(ves.errors),
))
def _parse_config(self,
config_fp: TextIO,
filename: str = '<config file>') -> Config:
try:
config = valohai_yaml.parse(config_fp)
config.project = self
return config
except OSError as err:
raise InvalidConfig(f'Could not read {filename}') from err
except valohai_yaml.ValidationErrors as ves:
raise InvalidConfig(
'{filename} is invalid ({n} errors); see `vh lint`'.format(
filename=filename,
n=len(ves.errors),
))
def get_config(self):
filename = self.get_config_filename()
try:
with open(filename) as infp:
config = valohai_yaml.parse(infp)
config.project = self
return config
except IOError as ioe:
six.raise_from(InvalidConfig('Could not read %s' % filename), ioe)
except ValidationErrors as ves:
raise InvalidConfig(
'{filename} is invalid ({n} errors); see `vh lint`'.format(
filename=filename,
n=len(ves.errors),
))
def cli(argv=None):
ap = get_argument_parser()
args, rest_argv = ap.parse_known_args(argv)
directory = (args.directory or os.getcwd())
if not os.path.isdir(directory):
ap.error('Invalid --directory')
has_git = (args.use_git and not args.adhoc and os.path.isdir(os.path.join(directory, '.git')))
try:
if args.adhoc and args.commit:
raise BadUsage('--adhoc and --commit are mutually exclusive')
args.commit, config_data = resolve_commit_and_config(directory, has_git, args.commit)
config = valohai_yaml.parse(StringIO(config_data))
step = config.steps[match_step(config, args.step)]
except BadUsage as be:
ap.error(be)
return
add_step_arguments(ap, step)
# We add the help argument only here so the step's arguments are also listed
ap.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS, help='show this help message and exit')
opt_args = ap.parse_args(argv)
dicts = defaultdict(dict)
for name, value in vars(opt_args).items():
if name.startswith(':'):
_, dict_name, name = name.split(':', 2)
dicts[dict_name][name] = value
executor = LocalExecutor(
command=args.command,
commit=args.commit,
directory=directory,
image=args.image,
inputs=dicts['inputs'],
output_root=os.path.realpath(args.output_root),
parameters=dicts['parameters'],
project_id=args.project_id,
step=step,
docker_command=args.docker_command,
docker_add_args=args.docker_add_args,
gitless=(not has_git),
)
ret = executor.execute(verbose=True, save_logs=args.save_logs)
sys.exit(ret) # Exit with the container's exit code
def test_pipeline_yaml_update_from_source(tmpdir, original_yaml, source_python,
expected_yaml_filename):
yaml_path = os.path.join(tmpdir, "valohai.yaml")
source_path = os.path.join(tmpdir, "test.py")
# Build repository with test.py and valohai.yaml
if os.path.isfile(original_yaml):
shutil.copy(original_yaml, yaml_path)
shutil.copy(source_python, source_path)
# Load original valohai.yaml
with open(yaml_path) as yaml_file:
old_config = parse(yaml_file)
# Parse new config from .py
new_config = get_pipeline_from_source(source_path, old_config)
# Merge original and new
new_config = old_config.merge_with(new_config)
# Check against expected result
with open(expected_yaml_filename) as fp:
compare_yaml(new_config, fp.read())
def _load_config(filename, roundtrip):
with open(os.path.join(examples_path, filename), 'r') as infp:
config = parse(infp)
if roundtrip:
config = parse(config.serialize())
return config
def load_config(self) -> Config:
with open(self.valohai_yaml_path, "r") as fp:
return valohai_yaml.parse(fp)
def test_unknown_parse():
with pytest.raises(ValueError) as e:
fail_config = '[{ city_name: Constantinople }]'
parse(fail_config)
assert e.value.args[0] == "No parser for {'city_name': 'Constantinople'}"