async def test_max_depth(nested_run_dir):
"""It should descend only as far as permitted."""
assert await listify(
scan(nested_run_dir, max_depth=1)
) == [
'a'
]
assert await listify(
scan(nested_run_dir, max_depth=3)
) == [
'a',
'b/c',
'd/e/f'
]
async def test_workflow_params(flow, scheduler, start, one_conf, run_dir,
mod_test_dir):
"""It should extract workflow params from the workflow database.
Note:
For this test we ensure that the workflow UUID is present in the params
table.
"""
reg = flow(one_conf)
schd = scheduler(reg)
async with start(schd):
pipe = (
# scan just this workflow
scan(scan_dir=mod_test_dir)
| filter_name(rf'^{reg}$')
| is_active(True)
| workflow_params)
async for flow in pipe:
# check the workflow_params field has been provided
assert 'workflow_params' in flow
# check the workflow uuid key has been read from the DB
uuid_key = WorkflowDatabaseManager.KEY_UUID_STR
assert uuid_key in flow['workflow_params']
# check the workflow uuid key matches the scheduler value
assert flow['workflow_params'][uuid_key] == schd.uuid_str
def __init__(self, uiserver, log, context=None, run_dir=None) -> None:
self.uiserver = uiserver
self.log = log
if context is None:
self.context = zmq.asyncio.Context()
else:
self.context = context
self.owner = getuser()
# all workflows currently tracked
self.workflows: 'Dict[str, Dict]' = {}
# the "workflow pipe" used to detect workflows on the filesystem
self._scan_pipe = (
# all flows on the filesystem
scan(run_dir)
# stop here is the flow is stopped, else...
| is_active(is_active=True, filter_stop=False)
# extract info from the contact file
| contact_info
# only flows which are using the same api version
| api_version(f'=={API}'))
# queue for requesting new scans, valid queued values are:
# * True - (stop=True) The stop signal (stops the scanner)
# * False - (stop=False) Request a new scan
# * None - The "stopped" signal, sent after the scan task has stooped
self._queue: 'asyncio.Queue[Union[bool, None]]' = asyncio.Queue()
# signal that the scanner is stopping, subsequent scan requests
# will be ignored
self._stopping = False
async def test_scan_sigstop(flow, scheduler, run, one_conf, test_dir, caplog):
"""It should log warnings if workflows are un-contactable.
Note:
This replaces tests/functional/cylc-scan/02-sigstop.t
last found in Cylc Flow 8.0a2 which used sigstop to make the flow
unresponsive.
"""
# run a workflow
reg = flow(one_conf)
schd = scheduler(reg)
async with run(schd):
# stop the server to make the flow un-responsive
schd.server.stop()
# try scanning the workflow
pipe = scan(test_dir) | graphql_query(['status'])
caplog.clear()
async for flow in pipe:
raise Exception("There shouldn't be any scan results")
# there should, however, be a warning
name = Path(reg).name
assert [(level, msg) for _, level, msg in caplog.record_tuples] == [
(30, f'Workflow not running: {name}')
]
async def test_workflow_params(
one,
start,
one_conf,
run_dir,
mod_test_dir
):
"""It should extract workflow params from the workflow database.
Note:
For this test we ensure that the workflow UUID is present in the params
table.
"""
async with start(one):
pipe = (
# scan just this workflow
scan(scan_dir=mod_test_dir)
| filter_name(rf'^{re.escape(one.workflow)}$')
| is_active(True)
| workflow_params
)
async for flow in pipe:
# check the workflow_params field has been provided
assert 'workflow_params' in flow
# check the workflow uuid key has been read from the DB
uuid_key = WorkflowDatabaseManager.KEY_UUID_STR
assert uuid_key in flow['workflow_params']
# check the workflow uuid key matches the scheduler value
assert flow['workflow_params'][uuid_key] == one.uuid_str
break
else:
raise Exception('Expected one scan result')
async def test_scan_symlinks(run_dir_with_symlinks):
"""It should follow symlinks to flows in other dirs."""
assert await listify(
scan(run_dir_with_symlinks)
) == [
'bar/baz',
'foo'
]
def get_pipe(opts, formatter, scan_dir=None):
"""Construct a pipe for listing flows."""
if scan_dir:
pipe = scan(scan_dir=scan_dir)
elif opts.source:
pipe = scan_multi(
Path(path).expanduser()
for path in glbl_cfg().get(['install', 'source dirs'])
)
opts.states = {'stopped'}
else:
pipe = scan
show_running = 'running' in opts.states
show_paused = 'paused' in opts.states
show_active = show_running or show_paused or 'stopping' in opts.states
# show_active = bool({'running', 'paused'} & opts.states)
show_inactive = bool({'stopped'} & opts.states)
# filter by flow name
if opts.name:
pipe |= filter_name(*opts.name)
# filter by flow state
if show_active:
pipe |= is_active(True, filter_stop=(not show_inactive))
elif show_inactive:
pipe |= is_active(False)
# get contact file information
if show_active:
pipe |= contact_info
graphql_fields = {}
graphql_filters = set()
# filter paused/running flows
if show_active and not (show_running and show_paused):
graphql_fields['status'] = None
graphql_filters.add((('status',), tuple(opts.states)))
# get fancy data if requested
if formatter == _format_rich:
# graphql_fields['status'] = None
graphql_fields.update(RICH_FIELDS)
# add graphql queries / filters to the pipe
if show_active and graphql_fields:
pipe |= graphql_query(graphql_fields, filters=graphql_filters)
elif opts.ping:
# check the flow is running even if not required
# by display format or filters
pipe |= graphql_query({'status': None})
# yield results as they are processed
pipe.preserve_order = False
return pipe
async def test_scan_horrible_mess(badly_messed_up_run_dir):
"""It shouldn't be affected by erroneous cylc files/dirs.
How could you end up with a .service dir in cylc-run, well misuse of
Cylc7 can result in this situation so this test ensures Cylc7 suites
can't mess up a Cylc8 scan.
"""
assert await listify(scan(badly_messed_up_run_dir)) == ['foo']
async def test_scan_nasty_symlinks(run_dir_with_nasty_symlinks):
"""It should handle strange symlinks because users can be nasty."""
assert await listify(
scan(run_dir_with_nasty_symlinks)
) == [
'bar', # well you got what you asked for
'foo'
]
async def test_max_depth_configurable(nested_dir, mock_glbl_cfg):
"""Default scan depth should be configurable in global.cylc."""
mock_glbl_cfg('cylc.flow.network.scan.glbl_cfg', '''
[install]
max depth = 2
''')
assert await listify(scan(nested_dir)) == [
'a',
'b/c',
]
async def test_scan_with_files(sample_run_dir):
"""It shouldn't be perturbed by arbitrary files."""
Path(sample_run_dir, 'abc').touch()
Path(sample_run_dir, 'def').touch()
assert await listify(scan(sample_run_dir)) == [
'bar/pub',
'baz',
'cheese/run1',
'cheese/run2',
'foo',
]
async def test_scan(sample_run_dir):
"""It should list all flows."""
assert await listify(
scan(sample_run_dir)
) == [
'bar/pub',
'baz',
'cheese/run1',
'cheese/run2',
'foo'
]
async def test_scan_one(one, start, test_dir):
"""Ensure that a running workflow appears in the scan results."""
async with start(one):
pipe = (
# scan just this workflow
scan(scan_dir=test_dir)
| filter_name(rf'^{re.escape(one.workflow)}$')
| is_active(True)
| workflow_params
)
async for flow in pipe:
assert flow['name'] == one.workflow
break
else:
raise Exception('Expected one scan result')
def get_pipe(opts, formatter, scan_dir=None):
"""Construct a pipe for listing flows."""
if scan_dir:
pipe = scan(scan_dir=scan_dir)
else:
pipe = scan
show_running = 'running' in opts.states
show_held = 'held' in opts.states
show_active = show_running or show_held or 'stopping' in opts.states
# show_active = bool({'running', 'held'} & opts.states)
show_inactive = bool({'stopped'} & opts.states)
# filter by flow name
if opts.name:
pipe |= filter_name(*opts.name)
# filter by flow state
if show_active:
pipe |= is_active(True, filter_stop=(not show_inactive))
elif show_inactive:
pipe |= is_active(False)
# get contact file information
if show_active:
pipe |= contact_info
graphql_fields = {}
graphql_filters = set()
# filter held/running flows
if show_active and not (show_running and show_held):
graphql_fields['status'] = None
graphql_filters.add((('status', ), tuple(opts.states)))
# get fancy data if requested
if formatter == _format_rich:
# graphql_fields['status'] = None
graphql_fields.update(RICH_FIELDS)
# add graphql queries / filters to the pipe
if show_active and graphql_fields:
pipe |= graphql_query(graphql_fields, filters=graphql_filters)
# yield results as they are processed
pipe.preserve_order = False
return pipe
def __init__(self, uiserver, context=None, run_dir=None):
self.uiserver = uiserver
if context is None:
self.context = zmq.asyncio.Context()
else:
self.context = context
self.owner = getuser()
self.active = {}
self.stopping = set()
self.inactive = set()
self._scan_pipe = (
# all flows on the filesystem
scan(run_dir)
# only flows which have a contact file
# | is_active(True)
# stop here is the flow is stopped, else...
| is_active(True, filter_stop=False)
# extract info from the contact file
| contact_info
# only flows which are using the same api version
| api_version(f'=={API}'))
async def test_max_depth(nested_dir, depth: int, expected: List[str]):
"""It should descend only as far as permitted."""
assert await listify(scan(nested_dir, max_depth=depth)) == expected
