async def test_query(gql_query, dummy_workflow):
"""Test sending the most basic GraphQL query."""
# configure two dummy workflows so we have something to look at
await dummy_workflow('foo')
await dummy_workflow('bar')
# perform the request
response = await gql_query(
*('cylc', 'graphql'),
query='''
query {
workflows {
id
status
}
}
''',
)
assert response.code == 200
# we should find the two dummy workflows in the response
body = json.loads(response.body)
assert body['data'] == {
'workflows': [
{
'id': Tokens(user='******', workflow='foo').id,
'status': 'stopped',
},
{
'id': Tokens(user='******', workflow='bar').id,
'status': 'stopped',
},
]
}
def node_filter(node, node_type, args):
"""Filter nodes based on attribute arguments"""
tokens: Tokens
if node_type in DEF_TYPES:
# namespace nodes don't fit into the universal ID scheme so must
# be tokenised manually
tokens = Tokens(
cycle=None,
task=node.name,
job=None,
)
else:
# live objects can be represented by a universal ID
tokens = Tokens(node.id)
state = getattr(node, 'state', None)
return (
(args.get('ghosts') or state or node_type in DEF_TYPES)
and (not args.get('states') or state in args['states'])
and not (args.get('exstates') and state in args['exstates'])
and (args.get('is_held') is None or (node.is_held == args['is_held']))
and (args.get('is_queued') is None or
(node.is_queued == args['is_queued']))
and (args.get('mindepth', -1) < 0 or node.depth >= args['mindepth'])
and (args.get('maxdepth', -1) < 0 or node.depth <= args['maxdepth'])
# Now filter node against id arg lists
and
(not args.get('ids') or node_ids_filter(tokens, state, args['ids']))
and not (args.get('exids')
and node_ids_filter(tokens, state, args['exids'])))
def extract_context(selection):
"""Return a dictionary of all component types in the selection.
Args:
selection (list):
List of element id's as extracted from the data store / graphql.
Examples:
>>> extract_context(['~a/b', '~a/c'])
{'user': ['a'], 'workflow': ['b', 'c']}
>>> extract_context(['~a/b//c/d/e']
... ) # doctest: +NORMALIZE_WHITESPACE
{'user': ['a'], 'workflow': ['b'], 'cycle': ['c'],
'task': ['d'], 'job': ['e']}
"""
ret = {}
for item in selection:
tokens = Tokens(item)
for key, value in tokens.items():
if (value and not key.endswith('_sel') # ignore selectors
):
lst = ret.setdefault(key, [])
if value not in lst:
lst.append(value)
return ret
def test_validate_number():
_validate_number(Tokens('a'), max_workflows=1)
with pytest.raises(UserInputError):
_validate_number(Tokens('a'), Tokens('b'), max_workflows=1)
t1 = Tokens(cycle='1')
t2 = Tokens(cycle='2')
_validate_number(t1, max_tasks=1)
with pytest.raises(UserInputError):
_validate_number(t1, t2, max_tasks=1)
def test_validate_workflow_ids(tmp_run_dir):
_validate_workflow_ids(Tokens('workflow'), src_path='')
with pytest.raises(UserInputError):
_validate_workflow_ids(Tokens('~alice/workflow'), src_path='')
run_dir = tmp_run_dir('b')
with pytest.raises(UserInputError):
_validate_workflow_ids(
Tokens('workflow'),
src_path=run_dir / 'flow.cylc',
)
def format_directives(self, job_conf):
"""Format the job directives for a job file."""
job_file_path = job_conf['job_file_path']
directives = job_conf["directives"].__class__() # an ordereddict
tokens = Tokens(job_conf['task_id'], relative=True)
directives["-N"] = (
f'{tokens["task"]}.{tokens["cycle"]}.{job_conf["workflow_name"]}'
)
directives["-o"] = job_file_path + ".out"
directives["-e"] = job_file_path + ".err"
if (job_conf["execution_time_limit"] and
directives.get("-l walltime") is None):
directives["-l walltime"] = "%d" % job_conf["execution_time_limit"]
# restartable?
directives.update(job_conf["directives"])
lines = []
for key, value in directives.items():
if value and " " in key:
# E.g. -l walltime=3:00:00
lines.append("%s%s=%s" % (self.DIRECTIVE_PREFIX, key, value))
elif value:
# E.g. -q queue_name
lines.append("%s%s %s" % (self.DIRECTIVE_PREFIX, key, value))
else:
# E.g. -V
lines.append(self.DIRECTIVE_PREFIX + key)
return lines
def format_directives(self, job_conf):
"""Format the job directives for a job file."""
job_file_path = job_conf['job_file_path']
directives = job_conf["directives"].__class__() # an ordereddict
# Change task/runM to task-runM in the job name
# (PBS 19.2.1+ does not allow '/' in job names)
tokens = Tokens(job_conf['task_id'], relative=True)
directives["-N"] = (f'{tokens["task"]}.{tokens["cycle"]}'
f".{job_conf['workflow_name'].replace('/', '-')}")
job_name_len_max = job_conf['platform']["job name length maximum"]
if job_name_len_max:
directives["-N"] = directives["-N"][0:job_name_len_max]
directives["-o"] = job_file_path + ".out"
directives["-e"] = job_file_path + ".err"
if (job_conf["execution_time_limit"]
and directives.get("-l walltime") is None):
directives["-l walltime"] = "%d" % job_conf["execution_time_limit"]
for key, value in list(job_conf["directives"].items()):
directives[key] = value
lines = []
for key, value in directives.items():
if value and " " in key:
# E.g. -l walltime=3:00:00
lines.append("%s%s=%s" % (self.DIRECTIVE_PREFIX, key, value))
elif value:
# E.g. -q queue_name
lines.append("%s%s %s" % (self.DIRECTIVE_PREFIX, key, value))
else:
# E.g. -V
lines.append(self.DIRECTIVE_PREFIX + key)
return lines
def get_broadcast(self, task_id=None):
"""Retrieve all broadcast variables that target a given task ID."""
if task_id == "None":
task_id = None
if not task_id:
# all broadcasts requested
return self.broadcasts
try:
tokens = Tokens(task_id, relative=True)
name = tokens['task']
point_string = tokens['cycle']
except ValueError:
raise Exception("Can't split task_id %s" % task_id)
ret = {}
# The order is:
# all:root -> all:FAM -> ... -> all:task
# -> tag:root -> tag:FAM -> ... -> tag:task
for cycle in ALL_CYCLE_POINTS_STRS + [point_string]:
if cycle not in self.broadcasts:
continue
for namespace in reversed(self.linearized_ancestors[name]):
if namespace in self.broadcasts[cycle]:
addict(ret, self.broadcasts[cycle][namespace])
return ret
def format_directives(self, job_conf):
"""Format the job directives for a job file."""
job_file_path = job_conf['job_file_path']
directives = job_conf['directives'].__class__()
tokens = Tokens(job_conf['task_id'], relative=True)
directives['-N'] = (
f'{job_conf["workflow_name"]}.{tokens["task"]}.{tokens["cycle"]}'
)
directives['-o'] = job_file_path + ".out"
directives['-e'] = job_file_path + ".err"
if (job_conf["execution_time_limit"] and
directives.get("-l h_rt") is None):
directives["-l h_rt"] = "%d:%02d:%02d" % (
job_conf["execution_time_limit"] / 3600,
(job_conf["execution_time_limit"] / 60) % 60,
job_conf["execution_time_limit"] % 60)
for key, value in list(job_conf['directives'].items()):
directives[key] = value
lines = []
for key, value in directives.items():
if value and " " in key:
# E.g. -l h_rt=3:00:00
lines.append("%s%s=%s" % (self.DIRECTIVE_PREFIX, key, value))
elif value:
# E.g. -q queue_name
lines.append("%s%s %s" % (self.DIRECTIVE_PREFIX, key, value))
else:
# E.g. -V
lines.append("%s%s" % (self.DIRECTIVE_PREFIX, key))
return lines
async def test_workflow_connect_fail(
data_store_mgr: DataStoreMgr,
port_range,
monkeypatch,
caplog,
):
"""Simulate a failure during workflow connection.
The data store should "rollback" any incidental changes made during the
failed connection attempt by disconnecting from the workflow afterwards.
Not an ideal test as we don't actually get a communication failure,
the data store manager just skips contacting the workflow because
we aren't providing a client for it to connect to, however, probably the
best we can achieve without actually running a workflow.
"""
# patch the zmq logic so that the connection doesn't fail at the first
# hurdle
monkeypatch.setattr(
'cylc.flow.network.ZMQSocketBase._socket_bind', lambda *a, **k: None,
)
# start a ZMQ REPLY socket in order to claim an unused port
w_id = Tokens(user='******', workflow='workflow_id').id
try:
context = zmq.Context()
server = ZMQSocketBase(
zmq.REP,
context=context,
workflow=w_id,
bind=True,
)
server._socket_bind(*port_range)
# register the workflow with the data store
await data_store_mgr.register_workflow(w_id=w_id, is_active=False)
contact_data = {
'name': 'workflow_id',
'owner': 'cylc',
CFF.HOST: 'localhost',
CFF.PORT: server.port,
CFF.PUBLISH_PORT: server.port,
CFF.API: 1
}
# try to connect to the workflow
caplog.set_level(logging.DEBUG, data_store_mgr.log.name)
await data_store_mgr.connect_workflow(w_id, contact_data)
# the connection should fail because our ZMQ socket is not a
# WorkflowRuntimeServer with the correct endpoints and auth
assert [record.message for record in caplog.records] == [
"[data-store] connect_workflow('~user/workflow_id', <dict>)",
'failed to connect to ~user/workflow_id',
"[data-store] disconnect_workflow('~user/workflow_id')",
]
finally:
# tidy up
server.stop()
context.destroy()
def format_directives(cls, job_conf):
"""Format the job directives for a job file."""
job_file_path = job_conf['job_file_path']
directives = job_conf['directives'].__class__()
tokens = Tokens(job_conf['task_id'], relative=True)
directives['--job-name'] = (
f'{tokens["task"]}.{tokens["cycle"]}.{job_conf["workflow_name"]}')
directives['--output'] = job_file_path.replace('%', '%%') + ".out"
directives['--error'] = job_file_path.replace('%', '%%') + ".err"
if (job_conf["execution_time_limit"]
and directives.get("--time") is None):
directives["--time"] = "%d:%02d" % (
job_conf["execution_time_limit"] / 60,
job_conf["execution_time_limit"] % 60)
for key, value in list(job_conf['directives'].items()):
directives[key] = value
lines = []
seen = set()
for key, value in directives.items():
m = cls.REC_HETJOB.match(key)
if m:
n = m.groups()[0]
if n != "0" and n not in seen:
lines.append(cls.SEP_HETJOB)
seen.add(n)
newkey = cls.REC_HETJOB.sub('', key)
else:
newkey = key
if value:
lines.append("%s%s=%s" % (cls.DIRECTIVE_PREFIX, newkey, value))
else:
lines.append("%s%s" % (cls.DIRECTIVE_PREFIX, newkey))
return lines
async def test_workflow_state_changes(tmp_path, active_before, active_after):
"""It correctly identifies workflow state changes from the filesystem."""
tmp_path /= str(random())
tmp_path.mkdir()
# mock the results of the previous scan
wfm = WorkflowsManager(None, LOG, context=None, run_dir=tmp_path)
wid = Tokens(user=wfm.owner, workflow='a').id
ret = (
{wid} if active_before == 'active' else set(),
{wid} if active_before == 'inactive' else set(),
)
wfm.get_workflows = lambda: ret
# mock the filesystem in the new state
if active_after == 'active':
mk_flow(tmp_path, 'a', active=True)
if active_after == 'inactive':
mk_flow(tmp_path, 'a', active=False)
# see what state changes the workflow manager detects
changes = []
async for change in wfm._workflow_state_changes():
changes.append(change)
# compare those changes to expectations
if active_before == active_after:
assert changes == []
else:
assert len(changes) == 1
assert (wid, active_before, active_after) == changes[0][:3]
async def test_register_workflow(data_store_mgr: DataStoreMgr):
"""Passing a workflow ID through register_workflow creates
an entry for the workflow in the data store .data map, and another
entry in the data store .delta_queues map."""
w_id = Tokens(user='******', workflow='workflow_id').id
await data_store_mgr.register_workflow(w_id=w_id, is_active=False)
assert w_id in data_store_mgr.data
assert w_id in data_store_mgr.delta_queues
async def test_update_contact_no_contact_data(data_store_mgr: DataStoreMgr):
"""Updating contact with no contact data results in default values
for the workflow in the data store, like the API version set to zero."""
w_id = Tokens(user='******', workflow='workflow_id').id
api_version = 0
await data_store_mgr.register_workflow(w_id=w_id, is_active=False)
data_store_mgr._update_contact(w_id=w_id, contact_data=None)
assert api_version == data_store_mgr.data[w_id]['workflow'].api_version
def sort_integer_node(id_):
"""Return sort tokens for nodes with cyclepoints in integer format.
Example:
>>> sort_integer_node('11/foo')
('foo', 11)
"""
tokens = Tokens(id_, relative=True)
return (tokens['task'], int(tokens['cycle']))
def idpop(id_):
"""Remove the last element of a node id.
Examples:
>>> idpop('c/t/j')
'c/t'
>>> idpop('c/t')
'c'
>>> idpop('c')
Traceback (most recent call last):
ValueError: No tokens provided
>>> idpop('')
Traceback (most recent call last):
ValueError: Invalid Cylc identifier: //
"""
relative = '//' not in id_
tokens = Tokens(id_, relative=relative)
tokens.pop_token()
return tokens.relative_id
def get_task_job_log(workflow, point, name, submit_num=None, suffix=None):
"""Return the full job log path."""
args = [
get_workflow_run_job_dir(workflow),
Tokens(
cycle=str(point),
task=name,
job=str(submit_num or NN),
).relative_id
]
if suffix is not None:
args.append(suffix)
return os.path.join(*args)
def test_validate_constraint():
"""It should validate tokens against the constraint."""
# constraint=workflows
_validate_constraint(Tokens(workflow='a'), constraint='workflows')
with pytest.raises(UserInputError):
_validate_constraint(Tokens(cycle='a'), constraint='workflows')
with pytest.raises(UserInputError):
_validate_constraint(Tokens(), constraint='workflows')
# constraint=tasks
_validate_constraint(Tokens(cycle='a'), constraint='tasks')
with pytest.raises(UserInputError):
_validate_constraint(Tokens(workflow='a'), constraint='tasks')
with pytest.raises(UserInputError):
_validate_constraint(Tokens(), constraint='tasks')
# constraint=mixed
_validate_constraint(Tokens(workflow='a'), constraint='mixed')
_validate_constraint(Tokens(cycle='a'), constraint='mixed')
with pytest.raises(UserInputError):
_validate_constraint(Tokens(), constraint='mixed')
async def mutator(root,
info,
command=None,
workflows=None,
exworkflows=None,
**args):
"""Call the resolver method that act on the workflow service
via the internal command queue."""
if workflows is None:
workflows = []
if exworkflows is None:
exworkflows = []
w_args = {
'workflows': [Tokens(w_id) for w_id in workflows],
'exworkflows': [Tokens(w_id) for w_id in exworkflows],
}
if args.get('args', False):
args.update(args.get('args', {}))
args.pop('args')
resolvers = info.context.get('resolvers')
res = await resolvers.service(info, command, w_args, args)
return GenericResponse(result=res)
async def test_workflow_state_change_uuid(tmp_path, active_before,
active_after):
"""It identifies discontinuities in workflow runs.
A workflow run is defined by its UUID. If this changes the workflow manager
must detect the change and re-register the workflow because any data known
about the workflow is now out of date.
This can happen because:
* A workflow was deleted and installed between scans.
* The user deleted the workflow database.
"""
# mock the result of the previous scan
wfm = WorkflowsManager(None, LOG, context=None, run_dir=tmp_path)
wid = Tokens(user=wfm.owner, workflow='a').id
wfm.workflows[wid] = {CFF.API: API, CFF.UUID: '41'}
if active_before:
wfm.get_workflows = lambda: ({wid}, set())
else:
wfm.get_workflows = lambda: (set(), {wid})
if active_after:
# create a workflow with the same name but a different UUID
# (simulates a new workflow run being started)
mk_flow(tmp_path, 'a', active=True)
else:
# create a workflow without a database
# (simulates the database being removed or workflow re-created)
mk_flow(tmp_path, 'a', active=False, database=False)
# see what state changes the workflow manager detects
changes = []
async for change in wfm._workflow_state_changes():
changes.append(change)
# the flow should be marked as becoming inactive then active again
assert [change[:3] for change in changes] == [(
wid,
('/active' if active_before else '/inactive'),
('active' if active_after else 'inactive'),
)]
# it should have picked up the new uuid too
if active_after:
assert changes[0][3][CFF.UUID] == '42'
async def test_update_contact_with_contact_data(data_store_mgr: DataStoreMgr):
"""Updating contact with contact data sets the values int he data store
for the workflow."""
w_id = Tokens(user='******', workflow='workflow_id').id
api_version = 1
await data_store_mgr.register_workflow(w_id=w_id, is_active=False)
contact_data = {
'name': 'workflow_id',
'owner': 'cylc',
CFF.HOST: 'localhost',
CFF.PORT: 40000,
CFF.API: api_version
}
data_store_mgr._update_contact(w_id=w_id, contact_data=contact_data)
assert api_version == data_store_mgr.data[w_id]['workflow'].api_version
def _task_proxy(id_, hier):
tokens = Tokens(id_, relative=True)
itask = SimpleNamespace()
itask.id_ = id_
itask.point = int(tokens['cycle'])
itask.state = SimpleNamespace()
itask.state.status = tokens['task_sel']
itask.tdef = SimpleNamespace()
itask.tdef.name = tokens['task']
if tokens['task'] in hier:
hier = hier[tokens['task']]
else:
hier = []
hier.append('root')
itask.tdef.namespace_hierarchy = hier
return itask
async def get_node_by_id(self, node_type, args):
"""Return protobuf node object for given id."""
n_id = args.get('id')
w_id = Tokens(n_id).workflow_id
# Both cases just as common so 'if' not 'try'
try:
if 'sub_id' in args and args.get('delta_store'):
flow = self.delta_store[args['sub_id']][w_id][
args['delta_type']]
else:
flow = self.data_store_mgr.data[w_id]
except KeyError:
return None
if node_type == PROXY_NODES:
return (flow[TASK_PROXIES].get(n_id)
or flow[FAMILY_PROXIES].get(n_id))
return flow[node_type].get(n_id)
async def test_data_store_changes(
one_workflow_aiter,
monkeypatch,
before,
after,
actions,
):
monkeypatch.setattr('cylc.uiserver.workflows_mgr.WorkflowRuntimeClient',
lambda *a, **k: True)
workflow_name = 'register-me'
wid = Tokens(user=getuser(), workflow=workflow_name).id
uiserver = dummy_uis()
# mock the state of the data store
ret = [
{wid} if before == 'active' else set(),
{wid} if before == 'inactive' else set(),
]
uiserver.data_store_mgr.get_workflows = lambda: ret
# mock the scan result
if after == 'active':
uiserver.workflows_mgr._scan_pipe = one_workflow_aiter(
**{
'name': workflow_name,
'contact': True,
CFF.HOST: 'localhost',
CFF.PORT: 0,
CFF.PUBLISH_PORT: 0,
CFF.API: 1
})
elif after == 'inactive':
uiserver.workflows_mgr._scan_pipe = one_workflow_aiter(
**{
'name': workflow_name,
'contact': False,
})
else:
uiserver.workflows_mgr._scan_pipe = one_workflow_aiter(none=True)
# run the update
await uiserver.workflows_mgr.update()
# see which data store methods got hit
assert uiserver.data_store_mgr.calls == actions
async def test_disconnect_workflow(data_store_mgr: DataStoreMgr):
"""Telling a data store to stop a workflow, is the same as updating
contact with no contact data."""
w_id = Tokens(user='******', workflow='workflow_id').id
api_version = 1
await data_store_mgr.register_workflow(w_id=w_id, is_active=False)
contact_data = {
'name': 'workflow_id',
'owner': 'cylc',
CFF.HOST: 'localhost',
CFF.PORT: 40000,
CFF.API: api_version
}
data_store_mgr._update_contact(w_id=w_id, contact_data=contact_data)
assert api_version == data_store_mgr.data[w_id]['workflow'].api_version
data_store_mgr.disconnect_workflow(w_id=w_id)
assert data_store_mgr.data[w_id]['workflow'].api_version == 0
def _update_contact(
self,
w_id,
contact_data=None,
status=None,
status_msg=None,
pruned=False,
):
delta = DELTAS_MAP[ALL_DELTAS]()
delta.workflow.time = time.time()
flow = delta.workflow.updated
flow.id = w_id
flow.stamp = f'{w_id}@{delta.workflow.time}'
if contact_data:
# update with contact file data
flow.name = contact_data['name']
flow.owner = contact_data['owner']
flow.host = contact_data[CFF.HOST]
flow.port = int(contact_data[CFF.PORT])
flow.api_version = int(contact_data[CFF.API])
else:
# wipe pre-existing contact-file data
w_tokens = Tokens(w_id)
flow.owner = w_tokens['user']
flow.name = w_tokens['workflow']
flow.host = ''
flow.port = 0
flow.api_version = 0
if status is not None:
flow.status = status
if status_msg is not None:
flow.status_msg = status_msg
if pruned:
flow.pruned = True
delta.workflow.pruned = w_id
# Apply to existing workflow data
if 'delta_times' not in self.data[w_id]:
self.data[w_id]['delta_times'] = {WORKFLOW: 0.0}
self._apply_all_delta(w_id, delta)
# Queue delta for subscription push
self._delta_store_to_queues(w_id, ALL_DELTAS, delta)
async def test_get_node_by_id(mock_flow, node_args):
"""Test method returning a workflow node message
who's ID is a match to that given."""
node_args['id'] = Tokens(
user='******',
workflow='mine',
cycle='20500808T00',
task='jin',
).id
node_args['workflows'].append({
'user': mock_flow.owner,
'workflow': mock_flow.name,
'workflow_sel': None
})
node = await mock_flow.resolvers.get_node_by_id(TASK_PROXIES, node_args)
assert node is None
node_args['id'] = mock_flow.node_ids[0]
node = await mock_flow.resolvers.get_node_by_id(TASK_PROXIES, node_args)
assert node in mock_flow.data[TASK_PROXIES].values()
def _get_status_msg(self, w_id: str, is_active: bool) -> str:
"""Derive a status message for the workflow.
Running schedulers provide their own status messages.
We must derive a status message for stopped workflows.
"""
if is_active:
# this will get overridden when we sync with the workflow
# set a sensible default here incase the sync takes a while
return 'Running'
w_id = Tokens(w_id)['workflow']
db_file = Path(get_workflow_srv_dir(w_id), WorkflowFiles.Service.DB)
if db_file.exists():
# the workflow has previously run
return 'Stopped'
else:
# the workflow has not yet run
return 'Not yet run'
async def mutator(root: Optional[Any],
info: 'ResolveInfo',
*,
command: str,
workflows: Optional[List[str]] = None,
**kwargs: Any):
"""Call the resolver method that act on the workflow service
via the internal command queue."""
if workflows is None:
workflows = []
parsed_workflows = [Tokens(w_id) for w_id in workflows]
if kwargs.get('args', False):
kwargs.update(kwargs.get('args', {}))
kwargs.pop('args')
resolvers: 'Resolvers' = (
info.context.get('resolvers') # type: ignore[union-attr]
)
res = await resolvers.service(info, command, parsed_workflows, kwargs)
return GenericResponse(result=res)
async def test_get_nodes_all(mock_flow, node_args):
"""Test method returning workflow(s) node message satisfying filter args.
"""
node_args['workflows'].append({
'user': mock_flow.owner,
'workflow': mock_flow.name,
'workflow_sel': None,
})
node_args['states'].append('failed')
nodes = await mock_flow.resolvers.get_nodes_all(TASK_PROXIES, node_args)
assert len(nodes) == 0
node_args['ghosts'] = True
node_args['states'] = []
node_args['ids'].append(Tokens(mock_flow.node_ids[0]))
nodes = [
n for n in await mock_flow.resolvers.get_nodes_all(
TASK_PROXIES, node_args)
if n in mock_flow.data[TASK_PROXIES].values()
]
assert len(nodes) == 1
