def _validate_yaml_project(project, yaml_file):
"""Validate project in clowder loaded from yaml file"""
_validate_type_dict(project, 'project', yaml_file)
if not project:
error = fmt.invalid_entries_error('project', project, yaml_file)
raise ClowderError(error)
if 'name' not in project:
error = fmt.missing_entry_error('name', 'project', yaml_file)
raise ClowderError(error)
_validate_type_str(project['name'], 'name', yaml_file)
del project['name']
if 'path' not in project:
error = fmt.missing_entry_error('path', 'project', yaml_file)
raise ClowderError(error)
_validate_type_str(project['path'], 'path', yaml_file)
del project['path']
_validate_yaml_project_optional(project, yaml_file)
if project:
error = fmt.invalid_entries_error('project', project, yaml_file)
raise ClowderError(error)
def _validate_yaml_sources(sources, yaml_file):
"""Validate sources in clowder loaded from yaml file"""
_validate_type_list(sources, 'sources', yaml_file)
if not sources:
error = fmt.invalid_entries_error('sources', sources, yaml_file)
raise ClowderError(error)
for source in sources:
_validate_type_dict(source, 'source', yaml_file)
if not source:
error = fmt.invalid_entries_error('source', source, yaml_file)
raise ClowderError(error)
if 'name' not in source:
error = fmt.missing_entry_error('name', 'source', yaml_file)
raise ClowderError(error)
_validate_type_str(source['name'], 'name', yaml_file)
del source['name']
if 'url' not in source:
error = fmt.missing_entry_error('url', 'source', yaml_file)
raise ClowderError(error)
_validate_type_str(source['url'], 'url', yaml_file)
del source['url']
if source:
error = fmt.invalid_entries_error('source', source, yaml_file)
raise ClowderError(error)
def _validate_yaml_import_defaults(defaults, yaml_file):
"""Validate clowder.yaml defaults with an import"""
_validate_type_dict(defaults, 'defaults', yaml_file)
if 'recursive' in defaults:
_validate_type_bool(defaults['recursive'], 'recursive', yaml_file)
del defaults['recursive']
if 'ref' in defaults:
_validate_type_str(defaults['ref'], 'ref', yaml_file)
if not _valid_ref_type(defaults['ref']):
error = fmt.invalid_ref_error(defaults['ref'], yaml_file)
raise ClowderError(error)
del defaults['ref']
if 'remote' in defaults:
_validate_type_str(defaults['remote'], 'remote', yaml_file)
del defaults['remote']
if 'source' in defaults:
_validate_type_str(defaults['source'], 'source', yaml_file)
del defaults['source']
if 'depth' in defaults:
_validate_type_depth(defaults['depth'], yaml_file)
del defaults['depth']
if 'timestamp_author' in defaults:
_validate_type_str(defaults['timestamp_author'], 'timestamp_author', yaml_file)
del defaults['timestamp_author']
if defaults:
error = fmt.invalid_entries_error('defaults', defaults, yaml_file)
raise ClowderError(error)
def validate_yaml_import(yaml_file):
"""Validate clowder.yaml with an import"""
parsed_yaml = parse_yaml(yaml_file)
_validate_type_dict(parsed_yaml, fmt.yaml_file('clowder.yaml'), yaml_file)
if 'import' not in parsed_yaml:
error = fmt.missing_entry_error('import', fmt.yaml_file('clowder.yaml'), yaml_file)
raise ClowderError(error)
_validate_type_str(parsed_yaml['import'], 'import', yaml_file)
del parsed_yaml['import']
if not parsed_yaml:
error = fmt.empty_yaml_error(yaml_file)
raise ClowderError(error)
if 'defaults' in parsed_yaml:
_validate_yaml_import_defaults(parsed_yaml['defaults'], yaml_file)
del parsed_yaml['defaults']
if 'sources' in parsed_yaml:
_validate_yaml_sources(parsed_yaml['sources'], yaml_file)
del parsed_yaml['sources']
if 'groups' in parsed_yaml:
_validate_yaml_import_groups(parsed_yaml['groups'], yaml_file)
del parsed_yaml['groups']
if parsed_yaml:
error = fmt.invalid_entries_error(fmt.yaml_file('clowder.yaml'), parsed_yaml, yaml_file)
raise ClowderError(error)
def _validate_type_depth(value, yaml_file):
"""Validate depth value"""
error = fmt.depth_error(value, yaml_file)
if not isinstance(value, int):
raise ClowderError(error)
if int(value) < 0:
raise ClowderError(error)
def execute_command(command, path, **kwargs):
"""Execute command via thread
.. py:function:: execute_command(command, path, shell=True, env=None, print_output=True)
:param command: Command to run
:type command: str or list[str]
:param str path: Path to set as ``cwd``
Keyword Args:
shell (bool): Whether to execute subprocess as ``shell``
env (dict): Enviroment to set as ``env``
print_output (bool): Whether to print output
:return: Command return code
:rtype: int
:raise ClowderError:
"""
shell = kwargs.get('shell', True)
env = kwargs.get('env', None)
print_output = kwargs.get('print_output', True)
cmd_env = os.environ.copy()
if env:
cmd_env.update(env)
if print_output:
pipe = None
else:
pipe = subprocess.PIPE
pool = ThreadPool()
try:
result = pool.apply(execute_subprocess_command,
args=(command, path),
kwds={
'shell': shell,
'env': cmd_env,
'stdout': pipe,
'stderr': pipe
})
pool.close()
pool.join()
return result
except (KeyboardInterrupt, SystemExit):
if pool:
pool.close()
pool.terminate()
raise ClowderError(colored('- Command interrupted', 'red'))
except Exception as err:
if pool:
pool.close()
pool.terminate()
raise ClowderError(
colored('\n - Command failed', 'red') + str(err) + '\n')
def _validate_yaml_import_group(group, yaml_file):
"""Validate group in clowder loaded from yaml file with import"""
_validate_type_dict(group, 'group', yaml_file)
if not group:
error = fmt.invalid_entries_error('group', group, yaml_file)
raise ClowderError(error)
if 'name' not in group:
error = fmt.missing_entry_error('name', 'group', yaml_file)
raise ClowderError(error)
_validate_type_str(group['name'], 'name', yaml_file)
del group['name']
if not group:
error = fmt.invalid_entries_error('group', group, yaml_file)
raise ClowderError(error)
if 'projects' in group:
_validate_yaml_projects(group['projects'], yaml_file, is_import=True)
del group['projects']
if 'recursive' in group:
_validate_type_bool(group['recursive'], 'recursive', yaml_file)
del group['recursive']
if 'ref' in group:
_validate_type_str(group['ref'], 'ref', yaml_file)
if not _valid_ref_type(group['ref']):
error = fmt.invalid_ref_error(group['ref'], yaml_file)
raise ClowderError(error)
del group['ref']
if 'remote' in group:
_validate_type_str(group['remote'], 'remote', yaml_file)
del group['remote']
if 'source' in group:
_validate_type_str(group['source'], 'source', yaml_file)
del group['source']
if 'depth' in group:
_validate_type_depth(group['depth'], yaml_file)
del group['depth']
if 'timestamp_author' in group:
_validate_type_str(group['timestamp_author'], 'timestamp_author', yaml_file)
del group['timestamp_author']
if group:
error = fmt.invalid_entries_error('group', group, yaml_file)
raise ClowderError(error)
def _validate_yaml_project_optional(project, yaml_file):
"""Validate optional args in project in clowder loaded from yaml file"""
if 'remote' in project:
_validate_type_str(project['remote'], 'remote', yaml_file)
del project['remote']
if 'recursive' in project:
_validate_type_bool(project['recursive'], 'recursive', yaml_file)
del project['recursive']
if 'timestamp_author' in project:
_validate_type_str(project['timestamp_author'], 'timestamp_author', yaml_file)
del project['timestamp_author']
if 'ref' in project:
_validate_type_str(project['ref'], 'ref', yaml_file)
if not _valid_ref_type(project['ref']):
error = fmt.invalid_ref_error(project['ref'], yaml_file)
raise ClowderError(error)
del project['ref']
if 'source' in project:
_validate_type_str(project['source'], 'source', yaml_file)
del project['source']
if 'depth' in project:
_validate_type_depth(project['depth'], yaml_file)
del project['depth']
if 'fork' in project:
fork = project['fork']
_validate_yaml_fork(fork, yaml_file)
del project['fork']
def _validate_yaml_groups(groups, yaml_file):
"""Validate groups in clowder loaded from yaml file"""
_validate_type_list(groups, 'groups', yaml_file)
if not groups:
error = fmt.invalid_entries_error('groups', groups, yaml_file)
raise ClowderError(error)
for group in groups:
_validate_yaml_group(group, yaml_file)
def _validate_yaml_defaults(defaults, yaml_file):
"""Validate defaults in clowder loaded from yaml file"""
_validate_type_dict(defaults, 'defaults', yaml_file)
if not defaults:
error = fmt.invalid_entries_error('defaults', defaults, yaml_file)
raise ClowderError(error)
if 'ref' not in defaults:
error = fmt.missing_entry_error('ref', 'defaults', yaml_file)
raise ClowderError(error)
_validate_type_str(defaults['ref'], 'ref', yaml_file)
if not _valid_ref_type(defaults['ref']):
error = fmt.invalid_ref_error(defaults['ref'], yaml_file)
raise ClowderError(error)
del defaults['ref']
if 'remote' not in defaults:
error = fmt.missing_entry_error('remote', 'defaults', yaml_file)
raise ClowderError(error)
_validate_type_str(defaults['remote'], 'remote', yaml_file)
del defaults['remote']
if 'source' not in defaults:
error = fmt.missing_entry_error('source', 'defaults', yaml_file)
raise ClowderError(error)
_validate_type_str(defaults['source'], 'source', yaml_file)
del defaults['source']
_validate_yaml_defaults_optional(defaults, yaml_file)
if defaults:
error = fmt.invalid_entries_error('defaults', defaults, yaml_file)
raise ClowderError(error)
def _validate_yaml_projects(projects, yaml_file, is_import):
"""Validate projects in clowder loaded from yaml file"""
_validate_type_list(projects, 'projects', yaml_file)
if not projects:
error = fmt.invalid_entries_error('projects', projects, yaml_file)
raise ClowderError(error)
for project in projects:
if is_import:
_validate_yaml_import_project(project, yaml_file)
else:
_validate_yaml_project(project, yaml_file)
def _validate_yaml_fork(fork, yaml_file):
"""Validate fork in clowder loaded from yaml file"""
_validate_type_dict(fork, 'fork', yaml_file)
if not fork:
error = fmt.invalid_entries_error('fork', fork, yaml_file)
raise ClowderError(error)
if 'name' not in fork:
error = fmt.missing_entry_error('name', 'fork', yaml_file)
raise ClowderError(error)
_validate_type_str(fork['name'], 'name', yaml_file)
del fork['name']
if 'remote' not in fork:
error = fmt.missing_entry_error('remote', 'fork', yaml_file)
raise ClowderError(error)
_validate_type_str(fork['remote'], 'remote', yaml_file)
del fork['remote']
if fork:
error = fmt.invalid_entries_error('fork', fork, yaml_file)
raise ClowderError(error)
def execute_subprocess_command(command, path, **kwargs):
"""Execute subprocess command
.. py:function:: execute_subprocess_command(command, path, shell=True, env=None, stdout=None, stderr=None)
:param command: Command to run
:type command: str or list[str]
:param str path: Path to set as ``cwd``
Keyword Args:
shell (bool): Whether to execute subprocess as ``shell``
env (dict): Enviroment to set as ``env``
stdout (int): Value to set as ``stdout``
stderr (int): Value to set as ``stderr``
:return: Subprocess return code
:rtype: int
:raise ClowderError:
"""
shell = kwargs.get('shell', True)
env = kwargs.get('env', None)
stdout = kwargs.get('stdout', None)
stderr = kwargs.get('stderr', None)
if isinstance(command, list):
cmd = ' '.join(command)
else:
cmd = command
try:
process = subprocess.Popen(cmd,
shell=shell,
env=env,
cwd=path,
stdout=stdout,
stderr=stderr)
atexit.register(subprocess_exit_handler, process)
process.communicate()
if process.returncode != 0:
raise ClowderError
except (KeyboardInterrupt, SystemExit):
raise
except Exception as err:
raise ClowderError(err)
def _validate_yaml(self, yaml_file, max_import_depth):
"""Validate clowder.yaml"""
parsed_yaml = clowder_yaml.parse_yaml(yaml_file)
if max_import_depth < 0:
print(fmt.invalid_yaml_error())
print(fmt.recursive_import_error(self._max_import_depth))
print()
sys.exit(1)
if 'import' not in parsed_yaml:
clowder_yaml.validate_yaml(yaml_file)
return
clowder_yaml.validate_yaml_import(yaml_file)
imported_clowder = parsed_yaml['import']
try:
if imported_clowder == 'default':
imported_yaml_file = os.path.join(self.root_directory,
'.clowder', 'clowder.yaml')
else:
imported_yaml_file = os.path.join(self.root_directory,
'.clowder', 'versions',
imported_clowder,
'clowder.yaml')
if not os.path.isfile(imported_yaml_file):
error = fmt.missing_imported_yaml_error(
imported_yaml_file, yaml_file)
raise ClowderError(error)
yaml_file = imported_yaml_file
self._validate_yaml(yaml_file, max_import_depth - 1)
except ClowderError as err:
print(fmt.invalid_yaml_error())
print(fmt.error(err))
sys.exit(1)
except (KeyboardInterrupt, SystemExit):
sys.exit(1)
def _run_forall_command(self, command, env, ignore_errors, parallel):
"""Run command or script in project directory
:param str command: Command to run
:param dict env: Environment variables
:param bool ignore_errors: Whether to exit if command returns a non-zero exit code
:param bool parallel: Whether command is being run in parallel, affects output
Raises:
ClowderError
ClowderExit
"""
self._print(fmt.command(command))
try:
execute_forall_command(command, self.full_path(), env,
self._print_output)
except ClowderError:
if not ignore_errors:
err = fmt.command_failed_error(command)
self._print(err)
if parallel:
raise ClowderError(err)
raise ClowderExit(1)
def _validate_type_dict(value, name, yaml_file):
"""Validate value is a dict"""
if not isinstance(value, dict):
error = fmt.not_dictionary_error(name, yaml_file)
raise ClowderError(error)
def _exit(message, parallel=False, return_code=1):
"""Exit based on serial or parallel job"""
if parallel:
raise ClowderError(message)
sys.exit(return_code)
def _validate_type_list(value, name, yaml_file):
"""Validate value is a list"""
if not isinstance(value, list):
error = fmt.not_list_error(name, yaml_file)
raise ClowderError(error)
def _validate_type_str(value, name, yaml_file):
"""Validate value is a str"""
if not isinstance(value, str):
error = fmt.not_string_error(name, yaml_file)
raise ClowderError(error)
def _validate_type_bool(value, name, yaml_file):
"""Validate value is a bool"""
if not isinstance(value, bool):
error = fmt.not_bool_error(name, yaml_file)
raise ClowderError(error)