def make_symlink(src, dst):
"""Makes symlinks for directories.
Args:
src (str): target path, where the files are to be stored.
dst (str): full path of link that will point to src.
"""
if os.path.exists(dst):
if os.path.islink(dst) and os.path.samefile(dst, src):
# correct symlink already exists
return False
# symlink name is in use by a physical file or directory
# log and return
LOG.debug(
f"Unable to create {src} symlink. The path {dst} already exists.")
return False
elif os.path.islink(dst):
# remove a bad symlink.
try:
os.unlink(dst)
except Exception:
raise WorkflowFilesError(
f"Error when symlinking. Failed to unlink bad symlink {dst}.")
os.makedirs(src, exist_ok=True)
os.makedirs(os.path.dirname(dst), exist_ok=True)
try:
os.symlink(src, dst, target_is_directory=True)
return True
except Exception as exc:
raise WorkflowFilesError(f"Error when symlinking\n{exc}")
def main(parser: COP, opts: 'Values', args: Optional[str] = None) -> None:
run_dir: Optional[Path]
workflow_id: str
if args is None:
try:
workflow_id = str(Path.cwd().relative_to(
Path(get_cylc_run_dir()).resolve()))
except ValueError:
raise WorkflowFilesError(
"The current working directory is not a workflow run directory"
)
else:
workflow_id, *_ = parse_id(
args,
constraint='workflows',
)
run_dir = Path(get_workflow_run_dir(workflow_id))
if not run_dir.is_dir():
raise WorkflowFilesError(
f'"{workflow_id}" is not an installed workflow.')
source, source_symlink = get_workflow_source_dir(run_dir)
if not source:
raise WorkflowFilesError(
f'"{workflow_id}" was not installed with cylc install.')
if not Path(source).is_dir():
raise WorkflowFilesError(
f'Workflow source dir is not accessible: "{source}".\n'
f'Restore the source or modify the "{source_symlink}"'
' symlink to continue.')
for entry_point in iter_entry_points('cylc.pre_configure'):
try:
entry_point.resolve()(srcdir=source, opts=opts)
except Exception as exc:
# NOTE: except Exception (purposefully vague)
# this is to separate plugin from core Cylc errors
raise PluginError('cylc.pre_configure', entry_point.name,
exc) from None
reinstall_workflow(
named_run=workflow_id,
rundir=run_dir,
source=source,
dry_run=False # TODO: ready for dry run implementation
)
for entry_point in iter_entry_points('cylc.post_install'):
try:
entry_point.resolve()(srcdir=source,
opts=opts,
rundir=str(run_dir))
except Exception as exc:
# NOTE: except Exception (purposefully vague)
# this is to separate plugin from core Cylc errors
raise PluginError('cylc.post_install', entry_point.name,
exc) from None
def validate_flow_name(flow_name: str) -> None:
"""Check workflow name is valid and not an absolute path.
Raise WorkflowFilesError if not valid.
"""
is_valid, message = SuiteNameValidator.validate(flow_name)
if not is_valid:
raise WorkflowFilesError(
f"invalid workflow name '{flow_name}' - {message}")
if os.path.isabs(flow_name):
raise WorkflowFilesError(
f"workflow name cannot be an absolute path: {flow_name}")
def main(parser: COP,
opts: 'Options',
named_run: Optional[str] = None) -> None:
if not named_run:
source, _ = get_workflow_source_dir(Path.cwd())
if source is None:
raise WorkflowFilesError(
f'"{Path.cwd()}" is not a workflow run directory.')
base_run_dir = Path(get_workflow_run_dir(''))
named_run = str(Path.cwd().relative_to(base_run_dir.resolve()))
run_dir = Path(get_workflow_run_dir(named_run))
if not run_dir.exists():
raise WorkflowFilesError(
f'"{named_run}" is not an installed workflow.')
if run_dir.name in [SuiteFiles.FLOW_FILE, SuiteFiles.SUITE_RC]:
run_dir = run_dir.parent
named_run = named_run.rsplit('/', 1)[0]
source, source_path = get_workflow_source_dir(run_dir)
if not source:
raise WorkflowFilesError(
f'"{named_run}" was not installed with cylc install.')
if not Path(source).exists():
raise WorkflowFilesError(
f'Workflow source dir is not accessible: "{source}".\n'
f'Restore the source or modify the "{source_path}"'
' symlink to continue.')
for entry_point in pkg_resources.iter_entry_points('cylc.pre_configure'):
try:
entry_point.resolve()(srcdir=source, opts=opts)
except Exception as exc:
# NOTE: except Exception (purposefully vague)
# this is to separate plugin from core Cylc errors
raise PluginError('cylc.pre_configure', entry_point.name,
exc) from None
reinstall_workflow(
named_run=named_run,
rundir=run_dir,
source=source,
dry_run=False # TODO: ready for dry run implementation
)
for entry_point in pkg_resources.iter_entry_points('cylc.post_install'):
try:
entry_point.resolve()(srcdir=source,
opts=opts,
rundir=str(run_dir))
except Exception as exc:
# NOTE: except Exception (purposefully vague)
# this is to separate plugin from core Cylc errors
raise PluginError('cylc.post_install', entry_point.name,
exc) from None
def search_install_source_dirs(flow_name: str) -> Path:
"""Return the path of a workflow source dir if it is present in the
'global.cylc[install]source dirs' search path."""
search_path: List[str] = glbl_cfg().get(['install', 'source dirs'])
if not search_path:
raise WorkflowFilesError(
"Cannot find workflow as 'global.cylc[install]source dirs' "
"does not contain any paths")
for path in search_path:
try:
flow_file = check_flow_file(Path(path, flow_name))
return flow_file.parent
except WorkflowFilesError:
continue
raise WorkflowFilesError(
f"Could not find workflow '{flow_name}' in: {', '.join(search_path)}")
def check_flow_file(
path: Union[Path, str], symlink_suiterc: bool = False,
logger: 'Logger' = LOG
) -> Path:
"""Raises WorkflowFilesError if no flow file in path sent.
Args:
path: Path to check for a flow.cylc and/or suite.rc file.
symlink_suiterc: If True and suite.rc exists but not flow.cylc, create
flow.cylc as a symlink to suite.rc.
logger: A custom logger to use to log warnings.
Returns the path of the flow file if present.
"""
flow_file_path = Path(expand_path(path), SuiteFiles.FLOW_FILE)
if flow_file_path.is_file():
# Note: this includes if flow.cylc is a symlink
return flow_file_path
suite_rc_path = Path(path, SuiteFiles.SUITE_RC)
if suite_rc_path.is_file():
if symlink_suiterc:
flow_file_path.symlink_to(suite_rc_path)
logger.warning(
f'The filename "{SuiteFiles.SUITE_RC}" is deprecated in '
f'favour of "{SuiteFiles.FLOW_FILE}". Symlink created.')
return suite_rc_path
raise WorkflowFilesError(
f"no {SuiteFiles.FLOW_FILE} or {SuiteFiles.SUITE_RC} in {path}")
def make_localhost_symlinks(rund, named_sub_dir):
"""Creates symlinks for any configured symlink dirs from glbl_cfg.
Args:
rund: the entire run directory path
named_sub_dir: e.g flow_name/run1
Returns:
dict - A dictionary of Symlinks with sources as keys and
destinations as values: ``{source: destination}``
"""
dirs_to_symlink = get_dirs_to_symlink(get_localhost_install_target(),
named_sub_dir)
symlinks_created = {}
for key, value in dirs_to_symlink.items():
if key == 'run':
dst = rund
else:
dst = os.path.join(rund, key)
src = expand_path(value)
if '$' in src:
raise WorkflowFilesError(
f'Unable to create symlink to {src}.'
f' \'{value}\' contains an invalid environment variable.'
' Please check configuration.')
symlink_success = make_symlink(src, dst)
# symlink info returned for logging purposes, symlinks created
# before logs as this dir may be a symlink.
if symlink_success:
symlinks_created[src] = dst
return symlinks_created
def clean(reg):
"""Remove a stopped workflow from the local filesystem only.
Deletes the workflow run directory and any symlink dirs. Note: if the
run dir has already been manually deleted, it will not be possible to
clean the symlink dirs.
Args:
reg (str): Workflow name.
"""
run_dir = Path(get_workflow_run_dir(reg))
try:
_clean_check(reg, run_dir)
except FileNotFoundError as exc:
LOG.info(str(exc))
return
# Note: 'share/cycle' must come first, and '' must come last
for possible_symlink in (
SuiteFiles.SHARE_CYCLE_DIR, SuiteFiles.SHARE_DIR,
SuiteFiles.LOG_DIR, SuiteFiles.WORK_DIR, ''):
name = Path(possible_symlink)
path = Path(run_dir, possible_symlink)
if path.is_symlink():
# Ensure symlink is pointing to expected directory. If not,
# something is wrong and we should abort
target = path.resolve()
if target.exists() and not target.is_dir():
raise WorkflowFilesError(
f'Invalid Cylc symlink directory {path} -> {target}\n'
f'Target is not a directory')
expected_end = str(Path('cylc-run', reg, name))
if not str(target).endswith(expected_end):
raise WorkflowFilesError(
f'Invalid Cylc symlink directory {path} -> {target}\n'
f'Expected target to end with "{expected_end}"')
# Remove <symlink_dir>/cylc-run/<reg>
target_cylc_run_dir = str(target).rsplit(str(reg), 1)[0]
target_reg_dir = Path(target_cylc_run_dir, reg)
if target_reg_dir.is_dir():
remove_dir(target_reg_dir)
# Remove empty parents
_remove_empty_reg_parents(reg, target_reg_dir)
remove_dir(run_dir)
_remove_empty_reg_parents(reg, run_dir)
def _check_child_dirs(path, depth_count=1):
for result in os.scandir(path):
if result.is_dir() and not result.is_symlink():
if is_valid_run_dir(result.path):
raise WorkflowFilesError(
exc_msg %
(flow_name, result.path))
if depth_count < MAX_SCAN_DEPTH:
_check_child_dirs(result.path, depth_count + 1)
def get_run_dir(run_path_base, run_name, no_run_name):
""" Build run directory for current install.
Args:
run_path_base (Path):
The workflow directory.
run_name (str):
Name of the run.
no_run_name (bool):
Flag as True to indicate no run name - workflow installed into
~/cylc-run/<run_path_base>.
Returns:
relink (bool):
True if runN symlink needs updating.
run_num (int):
Run number of the current install.
rundir (Path):
Run directory.
"""
relink = False
run_num = 0
if no_run_name:
rundir = run_path_base
elif run_name:
rundir = run_path_base.joinpath(run_name)
if (run_path_base.exists() and
detect_flow_exists(run_path_base, True)):
raise WorkflowFilesError(
f"This path: \"{run_path_base}\" contains installed numbered"
" runs. Try again, using cylc install without --run-name.")
else:
run_n = Path(run_path_base, SuiteFiles.RUN_N).expanduser()
run_num = get_next_rundir_number(run_path_base)
rundir = Path(run_path_base, f'run{run_num}')
if run_path_base.exists() and detect_flow_exists(run_path_base, False):
raise WorkflowFilesError(
f"This path: \"{run_path_base}\" contains an installed"
" workflow. Try again, using --run-name.")
unlink_runN(run_n)
relink = True
return relink, run_num, rundir
def make_symlink(path: Union[Path, str], target: Union[Path, str]) -> bool:
"""Makes symlinks for directories.
Args:
path: Absolute path of the desired symlink.
target: Absolute path of the symlink's target directory.
"""
path = Path(path)
target = Path(target)
if path.exists():
# note all three checks are needed here due to case where user has set
# their own symlink which does not match the global config set one.
if path.is_symlink() and target.exists() and path.samefile(target):
# correct symlink already exists
return False
# symlink name is in use by a physical file or directory
# log and return
LOG.debug(
f"Unable to create symlink to {target}. "
f"The path {path} already exists.")
return False
elif path.is_symlink():
# remove a bad symlink.
try:
path.unlink()
except OSError:
raise WorkflowFilesError(
f"Error when symlinking. Failed to unlink bad symlink {path}.")
target.mkdir(parents=True, exist_ok=True)
# This is needed in case share and share/cycle have the same symlink dir:
if path.exists():
return False
path.parent.mkdir(parents=True, exist_ok=True)
try:
path.symlink_to(target)
return True
except OSError as exc:
raise WorkflowFilesError(f"Error when symlinking\n{exc}")
def get_run_dir_info(
run_path_base: Path, run_name: Optional[str], no_run_name: bool
) -> Tuple[bool, Optional[int], Path]:
"""Get (numbered, named or unnamed) run directory info for current install.
Args:
run_path_base: The workflow directory absolute path.
run_name: Name of the run.
no_run_name: Flag as True to indicate no run name - workflow installed
into ~/cylc-run/<run_path_base>.
Returns:
relink: True if runN symlink needs updating.
run_num: Run number of the current install, if using numbered runs.
rundir: Run directory absolute path.
"""
relink = False
run_num = None
if no_run_name:
rundir = run_path_base
elif run_name:
rundir = run_path_base.joinpath(run_name)
if (run_path_base.exists() and
detect_flow_exists(run_path_base, True)):
raise WorkflowFilesError(
f"This path: \"{run_path_base}\" contains installed numbered"
" runs. Try again, using cylc install without --run-name.")
else:
run_num = get_next_rundir_number(run_path_base)
rundir = Path(run_path_base, f'run{run_num}')
if run_path_base.exists() and detect_flow_exists(run_path_base, False):
raise WorkflowFilesError(
f"This path: \"{run_path_base}\" contains an installed"
" workflow. Try again, using --run-name.")
unlink_runN(run_path_base)
relink = True
return relink, run_num, rundir
def validate_source_dir(source, flow_name):
"""Ensure the source directory is valid.
Args:
source (path): Path to source directory
Raises:
WorkflowFilesError:
If log, share, work or _cylc-install directories exist in the
source directory.
Cylc installing from within the cylc-run dir
"""
# Ensure source dir does not contain log, share, work, _cylc-install
for dir_ in SuiteFiles.RESERVED_DIRNAMES:
if Path(source, dir_).exists():
raise WorkflowFilesError(
f'{flow_name} installation failed. - {dir_} exists in source '
'directory.')
cylc_run_dir = Path(get_workflow_run_dir(''))
if (os.path.abspath(os.path.realpath(cylc_run_dir))
in os.path.abspath(os.path.realpath(source))):
raise WorkflowFilesError(
f'{flow_name} installation failed. Source directory should not be '
f'in {cylc_run_dir}')
check_flow_file(source)
def _parse_src_path(id_):
src_path = Path(id_)
if (id_ == os.curdir or id_.startswith(f'{os.curdir}{os.sep}')
or Path(id_).is_absolute()):
src_path = src_path.resolve()
if not src_path.exists():
raise UserInputError(f'Path does not exist: {src_path}')
if src_path.name in {WorkflowFiles.FLOW_FILE, WorkflowFiles.SUITE_RC}:
src_path = src_path.parent
try:
src_file_path = check_flow_file(src_path)
except WorkflowFilesError:
raise WorkflowFilesError(NO_FLOW_FILE_MSG.format(id_))
workflow_id = src_path.name
return workflow_id, src_path, src_file_path
return None
def check_flow_file(
path: Union[Path, str],
symlink_suiterc: bool = False,
logger: Optional['Logger'] = LOG
) -> Path:
"""Raises WorkflowFilesError if no flow file in path sent.
Args:
path: Path to check for a flow.cylc and/or suite.rc file.
symlink_suiterc: If True and suite.rc exists, create flow.cylc as a
symlink to suite.rc. If a flow.cylc symlink already exists but
points elsewhere, it will be replaced.
logger: A custom logger to use to log warnings.
Returns the path of the flow file if present.
"""
flow_file_path = Path(expand_path(path), WorkflowFiles.FLOW_FILE)
suite_rc_path = Path(expand_path(path), WorkflowFiles.SUITE_RC)
depr_msg = (
f'The filename "{WorkflowFiles.SUITE_RC}" is deprecated '
f'in favour of "{WorkflowFiles.FLOW_FILE}"')
if flow_file_path.is_file():
if not flow_file_path.is_symlink():
return flow_file_path
if flow_file_path.resolve() == suite_rc_path.resolve():
# A symlink that points to *existing* suite.rc
if logger:
logger.warning(depr_msg)
return flow_file_path
if suite_rc_path.is_file():
if not symlink_suiterc:
if logger:
logger.warning(depr_msg)
return suite_rc_path
if flow_file_path.is_symlink():
# Symlink broken or points elsewhere - replace
flow_file_path.unlink()
flow_file_path.symlink_to(WorkflowFiles.SUITE_RC)
if logger:
logger.warning(f'{depr_msg}. Symlink created.')
return flow_file_path
raise WorkflowFilesError(
f"no {WorkflowFiles.FLOW_FILE} or {WorkflowFiles.SUITE_RC} in {path}")
def _clean_check(reg, run_dir):
"""Check whether a workflow can be cleaned.
Args:
reg (str): Workflow name.
run_dir (str): Path to the workflow run dir on the filesystem.
"""
validate_flow_name(reg)
reg = os.path.normpath(reg)
if reg.startswith('.'):
raise WorkflowFilesError(
"Workflow name cannot be a path that points to the cylc-run "
"directory or above")
if not run_dir.is_dir() and not run_dir.is_symlink():
msg = f"No directory to clean at {run_dir}"
raise FileNotFoundError(msg)
try:
detect_old_contact_file(reg)
except SuiteServiceFileError as exc:
raise SuiteServiceFileError(
f"Cannot remove running workflow.\n\n{exc}")
def make_localhost_symlinks(
rund: Union[Path, str],
named_sub_dir: str,
symlink_conf: Optional[Dict[str, Dict[str, str]]] = None
) -> Dict[str, Union[Path, str]]:
"""Creates symlinks for any configured symlink dirs from glbl_cfg.
Args:
rund: the entire run directory path
named_sub_dir: e.g workflow_name/run1
symlink_conf: Symlinks dirs configuration passed from cli
Returns:
Dictionary of symlinks with sources as keys and
destinations as values: ``{source: destination}``
"""
symlinks_created = {}
dirs_to_symlink = get_dirs_to_symlink(
get_localhost_install_target(),
named_sub_dir, symlink_conf=symlink_conf
)
for key, value in dirs_to_symlink.items():
if value is None:
continue
if key == 'run':
symlink_path = rund
else:
symlink_path = os.path.join(rund, key)
target = expand_path(value)
if '$' in target:
raise WorkflowFilesError(
f'Unable to create symlink to {target}.'
f' \'{value}\' contains an invalid environment variable.'
' Please check configuration.')
symlink_success = make_symlink(symlink_path, target)
# Symlink info returned for logging purposes. Symlinks should be
# created before logs as the log dir may be a symlink.
if symlink_success:
symlinks_created[target] = symlink_path
return symlinks_created
def check_nested_run_dirs(run_dir: Union[Path, str], flow_name: str) -> None:
"""Disallow nested run dirs e.g. trying to install foo/bar where foo is
already a valid workflow directory.
Args:
run_dir: Absolute workflow run directory path.
flow_name: Workflow name.
Raise:
WorkflowFilesError:
- reg dir is nested inside a run dir
- reg dir contains a nested run dir (if not deeper than max scan
depth)
"""
exc_msg = (
'Nested run directories not allowed - cannot install workflow name '
'"{0}" as "{1}" is already a valid run directory.')
def _check_child_dirs(path: Union[Path, str], depth_count: int = 1):
for result in os.scandir(path):
if result.is_dir() and not result.is_symlink():
if is_valid_run_dir(result.path):
raise WorkflowFilesError(
exc_msg.format(flow_name, result.path)
)
if depth_count < MAX_SCAN_DEPTH:
_check_child_dirs(result.path, depth_count + 1)
reg_path: Union[Path, str] = os.path.normpath(run_dir)
parent_dir = os.path.dirname(reg_path)
while parent_dir not in ['', '/']:
if is_valid_run_dir(parent_dir):
raise WorkflowFilesError(
exc_msg.format(parent_dir, get_cylc_run_abs_path(parent_dir))
)
parent_dir = os.path.dirname(parent_dir)
reg_path = get_cylc_run_abs_path(reg_path)
if os.path.isdir(reg_path):
_check_child_dirs(reg_path)
def install_workflow(
flow_name: Optional[str] = None,
source: Optional[Union[Path, str]] = None,
run_name: Optional[str] = None,
no_run_name: bool = False,
no_symlinks: bool = False
) -> Tuple[Path, Path, str]:
"""Install a workflow, or renew its installation.
Install workflow into new run directory.
Create symlink to suite source location, creating any symlinks for run,
work, log, share, share/cycle directories.
Args:
flow_name: workflow name, default basename($PWD).
source: directory location of flow.cylc file, default $PWD.
run_name: name of the run, overrides run1, run2, run 3 etc...
If specified, cylc install will not create runN symlink.
rundir: for overriding the default cylc-run directory.
no_run_name: Flag as True to install workflow into
~/cylc-run/<flow_name>
no_symlinks: Flag as True to skip making localhost symlink dirs
Return:
source: The source directory.
rundir: The directory the workflow has been installed into.
flow_name: The installed suite name (which may be computed here).
Raise:
WorkflowFilesError:
No flow.cylc file found in source location.
Illegal name (can look like a relative path, but not absolute).
Another suite already has this name (unless --redirect).
Trying to install a workflow that is nested inside of another.
"""
if not source:
source = Path.cwd()
elif Path(source).name == SuiteFiles.FLOW_FILE:
source = Path(source).parent
source = Path(expand_path(source))
if not flow_name:
flow_name = Path.cwd().stem
validate_flow_name(flow_name)
if run_name in SuiteFiles.RESERVED_NAMES:
raise WorkflowFilesError(
f'Run name cannot be "{run_name}".')
validate_source_dir(source, flow_name)
run_path_base = Path(get_workflow_run_dir(flow_name))
relink, run_num, rundir = get_run_dir(run_path_base, run_name, no_run_name)
if Path(rundir).exists():
raise WorkflowFilesError(
f"\"{rundir}\" exists."
" Try using cylc reinstall. Alternatively, install with another"
" name, using the --run-name option.")
check_nested_run_dirs(rundir, flow_name)
symlinks_created = {}
if not no_symlinks:
sub_dir = flow_name
if run_num:
sub_dir += '/' + f'run{run_num}'
symlinks_created = make_localhost_symlinks(rundir, sub_dir)
INSTALL_LOG = _get_logger(rundir, 'cylc-install')
if not no_symlinks and bool(symlinks_created) is True:
for src, dst in symlinks_created.items():
INSTALL_LOG.info(f"Symlink created from {src} to {dst}")
try:
rundir.mkdir(exist_ok=True)
except OSError as e:
if e.strerror == "File exists":
raise WorkflowFilesError(f"Run directory already exists : {e}")
if relink:
link_runN(rundir)
create_workflow_srv_dir(rundir)
rsync_cmd = get_rsync_rund_cmd(source, rundir)
proc = Popen(rsync_cmd, stdout=PIPE, stderr=PIPE, text=True)
stdout, stderr = proc.communicate()
INSTALL_LOG.info(f"Copying files from {source} to {rundir}")
INSTALL_LOG.info(f"{stdout}")
if not proc.returncode == 0:
INSTALL_LOG.warning(
f"An error occurred when copying files from {source} to {rundir}")
INSTALL_LOG.warning(f" Error: {stderr}")
cylc_install = Path(rundir.parent, SuiteFiles.Install.DIRNAME)
check_flow_file(rundir, symlink_suiterc=True, logger=INSTALL_LOG)
if no_run_name:
cylc_install = Path(rundir, SuiteFiles.Install.DIRNAME)
source_link = cylc_install.joinpath(SuiteFiles.Install.SOURCE)
cylc_install.mkdir(parents=True, exist_ok=True)
if not source_link.exists():
INSTALL_LOG.info(f"Creating symlink from {source_link}")
source_link.symlink_to(source)
elif source_link.exists() and (os.readlink(source_link) == str(source)):
INSTALL_LOG.info(
f"Symlink from \"{source_link}\" to \"{source}\" in place.")
else:
raise WorkflowFilesError(
"Source directory between runs are not consistent.")
# check source link matches the source symlink from workflow dir.
INSTALL_LOG.info(f'INSTALLED {flow_name} from {source} -> {rundir}')
print(f'INSTALLED {flow_name} from {source} -> {rundir}')
_close_install_log(INSTALL_LOG)
return source, rundir, flow_name