async def test_max_depth(nested_run_dir): """It should descend only as far as permitted.""" assert await listify( scan(nested_run_dir, max_depth=1) ) == [ 'a' ] assert await listify( scan(nested_run_dir, max_depth=3) ) == [ 'a', 'b/c', 'd/e/f' ]
async def test_workflow_params(flow, scheduler, start, one_conf, run_dir, mod_test_dir): """It should extract workflow params from the workflow database. Note: For this test we ensure that the workflow UUID is present in the params table. """ reg = flow(one_conf) schd = scheduler(reg) async with start(schd): pipe = ( # scan just this workflow scan(scan_dir=mod_test_dir) | filter_name(rf'^{reg}$') | is_active(True) | workflow_params) async for flow in pipe: # check the workflow_params field has been provided assert 'workflow_params' in flow # check the workflow uuid key has been read from the DB uuid_key = WorkflowDatabaseManager.KEY_UUID_STR assert uuid_key in flow['workflow_params'] # check the workflow uuid key matches the scheduler value assert flow['workflow_params'][uuid_key] == schd.uuid_str
def __init__(self, uiserver, log, context=None, run_dir=None) -> None: self.uiserver = uiserver self.log = log if context is None: self.context = zmq.asyncio.Context() else: self.context = context self.owner = getuser() # all workflows currently tracked self.workflows: 'Dict[str, Dict]' = {} # the "workflow pipe" used to detect workflows on the filesystem self._scan_pipe = ( # all flows on the filesystem scan(run_dir) # stop here is the flow is stopped, else... | is_active(is_active=True, filter_stop=False) # extract info from the contact file | contact_info # only flows which are using the same api version | api_version(f'=={API}')) # queue for requesting new scans, valid queued values are: # * True - (stop=True) The stop signal (stops the scanner) # * False - (stop=False) Request a new scan # * None - The "stopped" signal, sent after the scan task has stooped self._queue: 'asyncio.Queue[Union[bool, None]]' = asyncio.Queue() # signal that the scanner is stopping, subsequent scan requests # will be ignored self._stopping = False
async def test_scan_sigstop(flow, scheduler, run, one_conf, test_dir, caplog): """It should log warnings if workflows are un-contactable. Note: This replaces tests/functional/cylc-scan/02-sigstop.t last found in Cylc Flow 8.0a2 which used sigstop to make the flow unresponsive. """ # run a workflow reg = flow(one_conf) schd = scheduler(reg) async with run(schd): # stop the server to make the flow un-responsive schd.server.stop() # try scanning the workflow pipe = scan(test_dir) | graphql_query(['status']) caplog.clear() async for flow in pipe: raise Exception("There shouldn't be any scan results") # there should, however, be a warning name = Path(reg).name assert [(level, msg) for _, level, msg in caplog.record_tuples] == [ (30, f'Workflow not running: {name}') ]
async def test_workflow_params( one, start, one_conf, run_dir, mod_test_dir ): """It should extract workflow params from the workflow database. Note: For this test we ensure that the workflow UUID is present in the params table. """ async with start(one): pipe = ( # scan just this workflow scan(scan_dir=mod_test_dir) | filter_name(rf'^{re.escape(one.workflow)}$') | is_active(True) | workflow_params ) async for flow in pipe: # check the workflow_params field has been provided assert 'workflow_params' in flow # check the workflow uuid key has been read from the DB uuid_key = WorkflowDatabaseManager.KEY_UUID_STR assert uuid_key in flow['workflow_params'] # check the workflow uuid key matches the scheduler value assert flow['workflow_params'][uuid_key] == one.uuid_str break else: raise Exception('Expected one scan result')
async def test_scan_symlinks(run_dir_with_symlinks): """It should follow symlinks to flows in other dirs.""" assert await listify( scan(run_dir_with_symlinks) ) == [ 'bar/baz', 'foo' ]
def get_pipe(opts, formatter, scan_dir=None): """Construct a pipe for listing flows.""" if scan_dir: pipe = scan(scan_dir=scan_dir) elif opts.source: pipe = scan_multi( Path(path).expanduser() for path in glbl_cfg().get(['install', 'source dirs']) ) opts.states = {'stopped'} else: pipe = scan show_running = 'running' in opts.states show_paused = 'paused' in opts.states show_active = show_running or show_paused or 'stopping' in opts.states # show_active = bool({'running', 'paused'} & opts.states) show_inactive = bool({'stopped'} & opts.states) # filter by flow name if opts.name: pipe |= filter_name(*opts.name) # filter by flow state if show_active: pipe |= is_active(True, filter_stop=(not show_inactive)) elif show_inactive: pipe |= is_active(False) # get contact file information if show_active: pipe |= contact_info graphql_fields = {} graphql_filters = set() # filter paused/running flows if show_active and not (show_running and show_paused): graphql_fields['status'] = None graphql_filters.add((('status',), tuple(opts.states))) # get fancy data if requested if formatter == _format_rich: # graphql_fields['status'] = None graphql_fields.update(RICH_FIELDS) # add graphql queries / filters to the pipe if show_active and graphql_fields: pipe |= graphql_query(graphql_fields, filters=graphql_filters) elif opts.ping: # check the flow is running even if not required # by display format or filters pipe |= graphql_query({'status': None}) # yield results as they are processed pipe.preserve_order = False return pipe
async def test_scan_horrible_mess(badly_messed_up_run_dir): """It shouldn't be affected by erroneous cylc files/dirs. How could you end up with a .service dir in cylc-run, well misuse of Cylc7 can result in this situation so this test ensures Cylc7 suites can't mess up a Cylc8 scan. """ assert await listify(scan(badly_messed_up_run_dir)) == ['foo']
async def test_scan_nasty_symlinks(run_dir_with_nasty_symlinks): """It should handle strange symlinks because users can be nasty.""" assert await listify( scan(run_dir_with_nasty_symlinks) ) == [ 'bar', # well you got what you asked for 'foo' ]
async def test_max_depth_configurable(nested_dir, mock_glbl_cfg): """Default scan depth should be configurable in global.cylc.""" mock_glbl_cfg('cylc.flow.network.scan.glbl_cfg', ''' [install] max depth = 2 ''') assert await listify(scan(nested_dir)) == [ 'a', 'b/c', ]
async def test_scan_with_files(sample_run_dir): """It shouldn't be perturbed by arbitrary files.""" Path(sample_run_dir, 'abc').touch() Path(sample_run_dir, 'def').touch() assert await listify(scan(sample_run_dir)) == [ 'bar/pub', 'baz', 'cheese/run1', 'cheese/run2', 'foo', ]
async def test_scan(sample_run_dir): """It should list all flows.""" assert await listify( scan(sample_run_dir) ) == [ 'bar/pub', 'baz', 'cheese/run1', 'cheese/run2', 'foo' ]
async def test_scan_one(one, start, test_dir): """Ensure that a running workflow appears in the scan results.""" async with start(one): pipe = ( # scan just this workflow scan(scan_dir=test_dir) | filter_name(rf'^{re.escape(one.workflow)}$') | is_active(True) | workflow_params ) async for flow in pipe: assert flow['name'] == one.workflow break else: raise Exception('Expected one scan result')
def get_pipe(opts, formatter, scan_dir=None): """Construct a pipe for listing flows.""" if scan_dir: pipe = scan(scan_dir=scan_dir) else: pipe = scan show_running = 'running' in opts.states show_held = 'held' in opts.states show_active = show_running or show_held or 'stopping' in opts.states # show_active = bool({'running', 'held'} & opts.states) show_inactive = bool({'stopped'} & opts.states) # filter by flow name if opts.name: pipe |= filter_name(*opts.name) # filter by flow state if show_active: pipe |= is_active(True, filter_stop=(not show_inactive)) elif show_inactive: pipe |= is_active(False) # get contact file information if show_active: pipe |= contact_info graphql_fields = {} graphql_filters = set() # filter held/running flows if show_active and not (show_running and show_held): graphql_fields['status'] = None graphql_filters.add((('status', ), tuple(opts.states))) # get fancy data if requested if formatter == _format_rich: # graphql_fields['status'] = None graphql_fields.update(RICH_FIELDS) # add graphql queries / filters to the pipe if show_active and graphql_fields: pipe |= graphql_query(graphql_fields, filters=graphql_filters) # yield results as they are processed pipe.preserve_order = False return pipe
def __init__(self, uiserver, context=None, run_dir=None): self.uiserver = uiserver if context is None: self.context = zmq.asyncio.Context() else: self.context = context self.owner = getuser() self.active = {} self.stopping = set() self.inactive = set() self._scan_pipe = ( # all flows on the filesystem scan(run_dir) # only flows which have a contact file # | is_active(True) # stop here is the flow is stopped, else... | is_active(True, filter_stop=False) # extract info from the contact file | contact_info # only flows which are using the same api version | api_version(f'=={API}'))
async def test_max_depth(nested_dir, depth: int, expected: List[str]): """It should descend only as far as permitted.""" assert await listify(scan(nested_dir, max_depth=depth)) == expected