def test_reverse_lookup_two_spices(job, remote, returns):
platforms = {
'sugar': {
'hosts': ['sugar', 'localhost'],
'batch system': 'slurm',
},
'pepper': {
'batch system': 'slurm',
'hosts': 'pepper'
},
}
assert reverse_lookup(platforms, job, remote) == returns
def test_reverse_lookup_similar_platforms(job, remote, returns):
platforms = {
'my-platform-with-bash': {
'hosts': 'desktop01',
'shell': '/bin/bash',
'batch system': 'background'
},
# An extra platform to check that we only pick up the first match
'my-platform-with-fish-not-this-one': {
'hosts': 'desktop01',
'shell': '/bin/fish',
'batch system': 'background'
},
'my-platform-with-fish': {
'hosts': 'desktop01',
'shell': '/bin/fish',
'batch system': 'background'
},
}
assert reverse_lookup(platforms, job, remote) == returns
def test_reverse_PlatformLookupError(job, remote):
with pytest.raises(PlatformLookupError):
reverse_lookup(PLATFORMS, job, remote)
def test_reverse_lookup_basic(job, remote, returns):
assert reverse_lookup(PLATFORMS, job, remote) == returns
def host_to_platform_upgrader(cfg):
"""Upgrade a config with host settings to a config with platform settings
if it is appropriate to do so.
+-------------------------------+
| Is platform set in this |
| [runtime][TASK]? |
+-------------------------------+
|YES |NO
| |
+---------------------v---------+ +--------+--------------+
| Are any forbidden items set | | host == $(function)? |
| in any [runtime][TASK] | +-+---------------------+
| [job] or [remote] section | NO | |YES
| | | +-------v------------------+
+-------------------------------+ | | Log - evaluate at task |
|YES |NO | | submit |
| +-------+ | | |
| | | +--------------------------+
+---------v---------------------+ | |
| Fail Loudly | | +-v-----------------------------+
+-------------------------------+ | | * Run reverse_lookup() |
| | * handle reverse lookup fail |
| | * add platform |
| | * delete forbidden settings |
| +-------------------------------+
|
| +-------------------------------+
+----> Return without changes |
+-------------------------------+
Args (cfg):
config object to be upgraded
Returns (cfg):
upgraded config object
"""
# If platform and old settings are set fail
# and remote should be added to this forbidden list
forbidden_with_platform = {
'host', 'batch system', 'batch submit command template'
}
for task_name, task_spec in cfg['runtime'].items():
# if task_name == 'delta':
# breakpoint(header=f"task_name = {task_name}")
if ('platform' in task_spec and 'job' in task_spec
or 'platform' in task_spec and 'remote' in task_spec):
if ('platform' in task_spec and forbidden_with_platform
& {*task_spec['job'], *task_spec['remote']}):
# Fail Loudly and Horribly
raise PlatformLookupError(
f"A mixture of Cylc 7 (host) and Cylc 8 (platform logic)"
f" should not be used. Task {task_name} set platform "
f"and item in {forbidden_with_platform}")
elif 'platform' in task_spec:
# Return config unchanged
continue
else:
# Add empty dicts if appropriate sections not present.
if 'job' in task_spec:
task_spec_job = task_spec['job']
else:
task_spec_job = {}
if 'remote' in task_spec:
task_spec_remote = task_spec['remote']
else:
task_spec_remote = {}
# Deal with case where host is a function and we cannot auto
# upgrade at the time of loading the config.
if ('host' in task_spec_remote
and REC_COMMAND.match(task_spec['remote']['host'])):
LOG.debug(
f"The host setting of '{task_name}' is a function: "
f"Cylc will try to upgrade this task on job submission.")
continue
# Attempt to use the reverse lookup
try:
platform = reverse_lookup(
glbl_cfg(cached=False).get(['job platforms']),
task_spec_job, task_spec_remote)
except PlatformLookupError as exc:
raise PlatformLookupError(f"for task {task_name}: {exc}")
else:
# Set platform in config
cfg['runtime'][task_name].update({'platform': platform})
LOG.warning(f"Platform {platform} auto selected from ")
# Remove deprecated items from config
for old_spec_item in forbidden_with_platform:
for task_section in ['job', 'remote']:
if (task_section in cfg['runtime'][task_name]
and old_spec_item in cfg['runtime'][task_name]
[task_section].keys()):
poppable = cfg['runtime'][task_name][task_section]
poppable.pop(old_spec_item)
LOG.warning(f"Cylc 7 {old_spec_item} removed.")
return cfg
def upgrade_to_platforms(self):
"""upgrade [job]batch system and [remote]host to platform
* Add 'platform' and 'user' columns to table task_jobs.
* Remove 'user_at_host' and 'batch_sys_name' columns
Returns:
bool - True if upgrade performed, False if upgrade skipped.
"""
conn = self.connect()
# check if upgrade required
schema = conn.execute(rf'PRAGMA table_info({self.TABLE_TASK_JOBS})')
for _, name, *_ in schema:
if name == 'platform':
LOG.debug('platform column present - skipping db upgrade')
return False
# Perform upgrade:
table = self.TABLE_TASK_JOBS
LOG.info('Upgrade to Cylc 8 platforms syntax')
conn.execute(
rf'''
ALTER TABLE
{table}
ADD COLUMN
user TEXT
'''
)
conn.execute(
rf'''
ALTER TABLE
{table}
ADD COLUMN
platform TEXT
'''
)
job_platforms = glbl_cfg(cached=False).get(['job platforms'])
for cycle, name, user_at_host, batch_system in conn.execute(rf'''
SELECT
cycle, name, user_at_host, batch_system
FROM
{table}
'''):
match = re.match(r"(?P<user>\S+)@(?P<host>\S+)", user_at_host)
if match:
user = match.group('user')
host = match.group('host')
else:
user = ''
host = user_at_host
platform = reverse_lookup(
job_platforms,
{'batch system': batch_system},
{'host': host}
)
conn.execute(
rf'''
UPDATE
{table}
SET
user=?,
platform=?
WHERE
cycle==?
AND name==?
''',
(user, platform, cycle, name)
)
conn.commit()
return True
