def fixture_provide_flow(tmp_path_factory):
"""Provide a cylc workflow based on the contents of a folder which can
be either validated or installed.
"""
src_flow_name = '11_reinstall_clean'
workflow_src = Path(__file__).parent / src_flow_name
test_flow_name = f'cylc-rose-test-{str(uuid4())[:8]}'
srcpath = (tmp_path_factory.getbasetemp() / test_flow_name)
flowpath = Path(get_workflow_run_dir(test_flow_name))
shutil.copytree(workflow_src, srcpath)
(srcpath / 'opt').mkdir(exist_ok=True)
for opt in ['foo', 'bar', 'baz']:
(srcpath / f'opt/rose-suite-{opt}.conf').touch()
yield {
'test_flow_name': test_flow_name,
'flowpath': flowpath,
'srcpath': srcpath
}
shutil.rmtree(srcpath)
shutil.rmtree(flowpath)
def file_install(self, platform):
"""Install required files on the remote install target.
Included by default in the file installation:
Files:
.service/server.key (required for ZMQ authentication)
Directories:
app/
bin/
etc/
lib/
"""
install_target = platform['install target']
self.remote_init_map[install_target] = REMOTE_FILE_INSTALL_IN_PROGRESS
src_path = get_workflow_run_dir(self.workflow)
dst_path = get_remote_workflow_run_dir(self.workflow)
install_target = platform['install target']
try:
cmd, host = construct_rsync_over_ssh_cmd(src_path,
dst_path,
platform,
self.rsync_includes,
bad_hosts=self.bad_hosts)
ctx = SubProcContext('file-install', cmd, host)
except NoHostsError:
LOG.error(
TaskRemoteMgmtError(TaskRemoteMgmtError.MSG_INIT,
install_target, '', '', '', ''))
self.remote_init_map[
platform['install target']] = REMOTE_FILE_INSTALL_FAILED
self.bad_hosts -= set(platform['hosts'])
self.ready = True
else:
LOG.debug(f"Begin file installation on {install_target}")
self.proc_pool.put_command(
ctx,
bad_hosts=self.bad_hosts,
callback=self._file_install_callback,
callback_args=[install_target, platform],
callback_255=self._file_install_callback_255,
)
def test_cylc_reinstall_fail_on_clashing_template_vars(tmp_path):
"""If you re-install with a different templating engine in suite.rc
reinstall should fail.
"""
(tmp_path / 'rose-suite.conf').write_text(
'[jinja2:suite.rc]\n'
'Primrose=\'Primula Vulgaris\'\n')
(tmp_path / 'flow.cylc').touch()
test_flow_name = f'cylc-rose-test-{str(uuid4())[:8]}'
install = subprocess.run([
'cylc', 'install', '-C',
str(tmp_path), '--flow-name', test_flow_name, '--no-run-name'
])
assert install.returncode == 0
(tmp_path / 'rose-suite.conf').write_text(
'[empy:suite.rc]\n'
'Primrose=\'Primula Vulgaris\'\n')
reinstall = subprocess.run(['cylc', 'reinstall', test_flow_name],
capture_output=True)
assert reinstall.returncode != 0
assert ('You should not define more than one templating section'
in reinstall.stderr.decode())
# Clean up run dir:
shutil.rmtree(get_workflow_run_dir(test_flow_name))
def file_install(self, platform):
"""Install required files on the remote install target.
Included by default in the file installation:
Files:
.service/server.key (required for ZMQ authentication)
Directories:
app/
bin/
etc/
lib/
"""
install_target = platform['install target']
self.remote_init_map[install_target] = REMOTE_FILE_INSTALL_IN_PROGRESS
src_path = get_workflow_run_dir(self.workflow)
dst_path = get_remote_workflow_run_dir(self.workflow)
install_target = platform['install target']
ctx = SubProcContext(
'file-install',
construct_rsync_over_ssh_cmd(src_path, dst_path, platform,
self.rsync_includes))
LOG.debug(f"Begin file installation on {install_target}")
self.proc_pool.put_command(ctx, self._file_install_callback,
[install_target])
def validate_source_dir(source, flow_name):
"""Ensure the source directory is valid.
Args:
source (path): Path to source directory
Raises:
WorkflowFilesError:
If log, share, work or _cylc-install directories exist in the
source directory.
Cylc installing from within the cylc-run dir
"""
# Ensure source dir does not contain log, share, work, _cylc-install
for dir_ in SuiteFiles.RESERVED_DIRNAMES:
if Path(source, dir_).exists():
raise WorkflowFilesError(
f'{flow_name} installation failed. - {dir_} exists in source '
'directory.')
cylc_run_dir = Path(get_workflow_run_dir(''))
if (os.path.abspath(os.path.realpath(cylc_run_dir))
in os.path.abspath(os.path.realpath(source))):
raise WorkflowFilesError(
f'{flow_name} installation failed. Source directory should not be '
f'in {cylc_run_dir}')
check_flow_file(source)
def workflow_state(workflow,
task,
point,
offset=None,
status='succeeded',
message=None,
cylc_run_dir=None):
"""Connect to a workflow DB and query the requested task state.
* Reports satisfied only if the remote workflow state has been achieved.
* Returns all workflow state args to pass on to triggering tasks.
Arguments:
workflow (str):
The workflow to interrogate.
task (str):
The name of the task to query.
point (str):
The cycle point.
offset (str):
The offset between the cycle this xtrigger is used in and the one
it is querying for as an ISO8601 time duration.
e.g. PT1H (one hour).
status (str):
The task status required for this xtrigger to be satisfied.
message (str):
The custom task output required for this xtrigger to be satisfied.
.. note::
This cannot be specified in conjunction with ``status``.
cylc_run_dir (str):
The directory in which the workflow to interrogate.
.. note::
This only needs to be supplied if the workflow is running in a
different location to what is specified in the global
configuration (usually ``~/cylc-run``).
Returns:
tuple: (satisfied, results)
satisfied (bool):
True if ``satisfied`` else ``False``.
results (dict):
Dictionary containing the args / kwargs which were provided
to this xtrigger.
"""
if cylc_run_dir:
cylc_run_dir = expand_path(cylc_run_dir)
else:
cylc_run_dir = get_workflow_run_dir('')
if offset is not None:
point = str(add_offset(point, offset))
try:
checker = CylcWorkflowDBChecker(cylc_run_dir, workflow)
except (OSError, sqlite3.Error):
# Failed to connect to DB; target workflow may not be started.
return (False, None)
fmt = checker.get_remote_point_format()
if fmt:
my_parser = TimePointParser()
point = str(my_parser.parse(point, dump_format=fmt))
if message is not None:
satisfied = checker.task_state_met(task, point, message=message)
else:
satisfied = checker.task_state_met(task, point, status=status)
results = {
'workflow': workflow,
'task': task,
'point': point,
'offset': offset,
'status': status,
'message': message,
'cylc_run_dir': cylc_run_dir
}
return satisfied, results
def setup_stem_repo(tmp_path_factory, monkeymodule):
"""Setup a Rose Stem Repository for the tests.
creates the following repo structure:
.. code::
|-- baseinstall
| `-- trunk
| `-- rose-stem
|-- conf
| `-- keyword.cfg
|-- cylc-rose-stem-test-1df3e028
| `-- rose-stem
| |-- flow.cylc
| `-- rose-suite.conf
`-- rose-test-battery-stemtest-repo
`-- foo
<truncated>
Yields:
dictionary:
basetemp:
The location of the base temporary file, which allows tests
to modify any part of the rose-stem suite.
workingcopy:
Path to the location of the working copy.
suitename:
The name of the suite, which will be name of the suite/workflow
installed in ``~/cylc-run``.
suite_install_directory:
The path to the installed suite/workflow. Handy for cleaning
up after tests.
"""
# Set up required folders:
basetemp = tmp_path_factory.getbasetemp()
baseinstall = basetemp / 'baseinstall'
rose_stem_dir = baseinstall / 'trunk/rose-stem'
repo = basetemp / 'rose-test-battery-stemtest-repo'
confdir = basetemp / 'conf'
workingcopy = basetemp / f'cylc-rose-stem-test-{str(uuid4())[:8]}'
for dir_ in [baseinstall, repo, rose_stem_dir, confdir, workingcopy]:
dir_.mkdir(parents=True)
# Turn repo into an svn repo:
subprocess.run(['svnadmin', 'create', f'{repo}/foo'])
url = f'file://{repo}/foo'
old = Path().cwd()
os.chdir(baseinstall)
subprocess.run(['svn', 'import', '-q', '-m', '""', f'{url}'])
os.chdir(old)
# Set Keywords for repository.
(basetemp /
'conf/keyword.cfg').write_text(f"location{{primary}}[foo.x]={url}")
monkeymodule.setenv('FCM_CONF_PATH', str(confdir))
# Check out a working copy of the repo:
suitename = workingcopy.parts[-1]
subprocess.run(split(f'fcm checkout -q fcm:foo.x_tr {workingcopy}'))
# Copy suite into working copy.
test_src_dir = Path(__file__).parent / '12_rose_stem'
for file in ['rose-suite.conf', 'flow.cylc']:
src = str(test_src_dir / file)
dest = str(workingcopy / 'rose-stem')
shutil.copy2(src, dest)
suite_install_dir = get_workflow_run_dir(suitename)
yield {
'basetemp': basetemp,
'workingcopy': workingcopy,
'suitename': suitename,
'suite_install_dir': suite_install_dir
}
async def scan(run_dir=None, scan_dir=None, max_depth=MAX_SCAN_DEPTH):
"""List flows installed on the filesystem.
Args:
run_dir (pathlib.Path):
The run dir to look for workflows in, defaults to ~/cylc-run.
All workflow registrations will be given relative to this path.
scan_dir(pathlib.Path):
The directory to scan for workflows in.
Use in combination with run_dir if you want to scan a subdir
within the run_dir.
max_depth (int):
The maximum number of levels to descend before bailing.
* ``max_depth=1`` will pick up top-level suites (e.g. ``foo``).
* ``max_depth=2`` will pick up nested suites (e.g. ``foo/bar``).
Yields:
dict - Dictionary containing information about the flow.
"""
if not run_dir:
run_dir = Path(get_workflow_run_dir(''))
if not scan_dir:
scan_dir = run_dir
running = []
# wrapper for scandir to preserve context
async def _scandir(path, depth):
contents = await scandir(path)
return path, depth, contents
# perform the first directory listing
for subdir in await scandir(scan_dir):
if subdir.is_dir():
running.append(
asyncio.create_task(
_scandir(subdir, 1)
)
)
# perform all further directory listings
while running:
# wait here until there's something to do
done, _ = await asyncio.wait(
running,
return_when=asyncio.FIRST_COMPLETED
)
for task in done:
path, depth, contents = task.result()
running.remove(task)
if dir_is_flow(contents):
# this is a flow directory
yield {
'name': str(path.relative_to(run_dir)),
'path': path,
}
elif depth < max_depth:
# we may have a nested flow, lets see...
for subdir in contents:
if (subdir.is_dir()
and subdir.stem not in EXCLUDE_FILES):
running.append(
asyncio.create_task(
_scandir(subdir, depth + 1)
)
)
# don't allow this to become blocking
await asyncio.sleep(0)
def install_workflow(
flow_name: Optional[str] = None,
source: Optional[Union[Path, str]] = None,
run_name: Optional[str] = None,
no_run_name: bool = False,
no_symlinks: bool = False
) -> Tuple[Path, Path, str]:
"""Install a workflow, or renew its installation.
Install workflow into new run directory.
Create symlink to suite source location, creating any symlinks for run,
work, log, share, share/cycle directories.
Args:
flow_name: workflow name, default basename($PWD).
source: directory location of flow.cylc file, default $PWD.
run_name: name of the run, overrides run1, run2, run 3 etc...
If specified, cylc install will not create runN symlink.
rundir: for overriding the default cylc-run directory.
no_run_name: Flag as True to install workflow into
~/cylc-run/<flow_name>
no_symlinks: Flag as True to skip making localhost symlink dirs
Return:
source: The source directory.
rundir: The directory the workflow has been installed into.
flow_name: The installed suite name (which may be computed here).
Raise:
WorkflowFilesError:
No flow.cylc file found in source location.
Illegal name (can look like a relative path, but not absolute).
Another suite already has this name (unless --redirect).
Trying to install a workflow that is nested inside of another.
"""
if not source:
source = Path.cwd()
elif Path(source).name == SuiteFiles.FLOW_FILE:
source = Path(source).parent
source = Path(expand_path(source))
if not flow_name:
flow_name = Path.cwd().stem
validate_flow_name(flow_name)
if run_name in SuiteFiles.RESERVED_NAMES:
raise WorkflowFilesError(
f'Run name cannot be "{run_name}".')
validate_source_dir(source, flow_name)
run_path_base = Path(get_workflow_run_dir(flow_name))
relink, run_num, rundir = get_run_dir(run_path_base, run_name, no_run_name)
if Path(rundir).exists():
raise WorkflowFilesError(
f"\"{rundir}\" exists."
" Try using cylc reinstall. Alternatively, install with another"
" name, using the --run-name option.")
check_nested_run_dirs(rundir, flow_name)
symlinks_created = {}
if not no_symlinks:
sub_dir = flow_name
if run_num:
sub_dir += '/' + f'run{run_num}'
symlinks_created = make_localhost_symlinks(rundir, sub_dir)
INSTALL_LOG = _get_logger(rundir, 'cylc-install')
if not no_symlinks and bool(symlinks_created) is True:
for src, dst in symlinks_created.items():
INSTALL_LOG.info(f"Symlink created from {src} to {dst}")
try:
rundir.mkdir(exist_ok=True)
except OSError as e:
if e.strerror == "File exists":
raise WorkflowFilesError(f"Run directory already exists : {e}")
if relink:
link_runN(rundir)
create_workflow_srv_dir(rundir)
rsync_cmd = get_rsync_rund_cmd(source, rundir)
proc = Popen(rsync_cmd, stdout=PIPE, stderr=PIPE, text=True)
stdout, stderr = proc.communicate()
INSTALL_LOG.info(f"Copying files from {source} to {rundir}")
INSTALL_LOG.info(f"{stdout}")
if not proc.returncode == 0:
INSTALL_LOG.warning(
f"An error occurred when copying files from {source} to {rundir}")
INSTALL_LOG.warning(f" Error: {stderr}")
cylc_install = Path(rundir.parent, SuiteFiles.Install.DIRNAME)
check_flow_file(rundir, symlink_suiterc=True, logger=INSTALL_LOG)
if no_run_name:
cylc_install = Path(rundir, SuiteFiles.Install.DIRNAME)
source_link = cylc_install.joinpath(SuiteFiles.Install.SOURCE)
cylc_install.mkdir(parents=True, exist_ok=True)
if not source_link.exists():
INSTALL_LOG.info(f"Creating symlink from {source_link}")
source_link.symlink_to(source)
elif source_link.exists() and (os.readlink(source_link) == str(source)):
INSTALL_LOG.info(
f"Symlink from \"{source_link}\" to \"{source}\" in place.")
else:
raise WorkflowFilesError(
"Source directory between runs are not consistent.")
# check source link matches the source symlink from workflow dir.
INSTALL_LOG.info(f'INSTALLED {flow_name} from {source} -> {rundir}')
print(f'INSTALLED {flow_name} from {source} -> {rundir}')
_close_install_log(INSTALL_LOG)
return source, rundir, flow_name
def main(parser: COP, options: 'Values', workflow: str) -> None:
if options.use_task_point and options.cycle:
raise UserInputError(
"cannot specify a cycle point and use environment variable")
if options.use_task_point:
if "CYLC_TASK_CYCLE_POINT" in os.environ:
options.cycle = os.environ["CYLC_TASK_CYCLE_POINT"]
else:
raise UserInputError("CYLC_TASK_CYCLE_POINT is not defined")
if options.offset and not options.cycle:
raise UserInputError("You must target a cycle point to use an offset")
# Attempt to apply specified offset to the targeted cycle
if options.offset:
options.cycle = str(add_offset(options.cycle, options.offset))
# Exit if both task state and message are to being polled
if options.status and options.msg:
raise UserInputError("cannot poll both status and custom output")
if options.msg and not options.task and not options.cycle:
raise UserInputError("need a taskname and cyclepoint")
# Exit if an invalid status is requested
if (options.status and options.status not in TASK_STATUSES_ORDERED
and options.status not in CylcWorkflowDBChecker.STATE_ALIASES):
raise UserInputError(f"invalid status '{options.status}'")
# this only runs locally
if options.run_dir:
run_dir = expand_path(options.run_dir)
else:
run_dir = get_workflow_run_dir('')
pollargs = {
'workflow': workflow,
'run_dir': run_dir,
'task': options.task,
'cycle': options.cycle,
'status': options.status,
'message': options.msg,
}
spoller = WorkflowPoller("requested state",
options.interval,
options.max_polls,
args=pollargs)
connected, formatted_pt = spoller.connect()
if not connected:
raise CylcError("cannot connect to the workflow DB")
if options.status and options.task and options.cycle:
# check a task status
spoller.condition = options.status
if not spoller.poll():
sys.exit(1)
elif options.msg:
# Check for a custom task output
spoller.condition = "output: %s" % options.msg
if not spoller.poll():
sys.exit(1)
else:
# just display query results
spoller.checker.display_maps(
spoller.checker.workflow_state_query(task=options.task,
cycle=formatted_pt,
status=options.status))
def run_dir():
"""The cylc run directory for this host."""
path = Path(get_workflow_run_dir(''))
path.mkdir(exist_ok=True)
yield path