def test_reverse_lookup_two_spices(job, remote, returns): platforms = { 'sugar': { 'hosts': ['sugar', 'localhost'], 'batch system': 'slurm', }, 'pepper': { 'batch system': 'slurm', 'hosts': 'pepper' }, } assert reverse_lookup(platforms, job, remote) == returns
def test_reverse_lookup_similar_platforms(job, remote, returns): platforms = { 'my-platform-with-bash': { 'hosts': 'desktop01', 'shell': '/bin/bash', 'batch system': 'background' }, # An extra platform to check that we only pick up the first match 'my-platform-with-fish-not-this-one': { 'hosts': 'desktop01', 'shell': '/bin/fish', 'batch system': 'background' }, 'my-platform-with-fish': { 'hosts': 'desktop01', 'shell': '/bin/fish', 'batch system': 'background' }, } assert reverse_lookup(platforms, job, remote) == returns
def test_reverse_PlatformLookupError(job, remote): with pytest.raises(PlatformLookupError): reverse_lookup(PLATFORMS, job, remote)
def test_reverse_lookup_basic(job, remote, returns): assert reverse_lookup(PLATFORMS, job, remote) == returns
def host_to_platform_upgrader(cfg): """Upgrade a config with host settings to a config with platform settings if it is appropriate to do so. +-------------------------------+ | Is platform set in this | | [runtime][TASK]? | +-------------------------------+ |YES |NO | | +---------------------v---------+ +--------+--------------+ | Are any forbidden items set | | host == $(function)? | | in any [runtime][TASK] | +-+---------------------+ | [job] or [remote] section | NO | |YES | | | +-------v------------------+ +-------------------------------+ | | Log - evaluate at task | |YES |NO | | submit | | +-------+ | | | | | | +--------------------------+ +---------v---------------------+ | | | Fail Loudly | | +-v-----------------------------+ +-------------------------------+ | | * Run reverse_lookup() | | | * handle reverse lookup fail | | | * add platform | | | * delete forbidden settings | | +-------------------------------+ | | +-------------------------------+ +----> Return without changes | +-------------------------------+ Args (cfg): config object to be upgraded Returns (cfg): upgraded config object """ # If platform and old settings are set fail # and remote should be added to this forbidden list forbidden_with_platform = { 'host', 'batch system', 'batch submit command template' } for task_name, task_spec in cfg['runtime'].items(): # if task_name == 'delta': # breakpoint(header=f"task_name = {task_name}") if ('platform' in task_spec and 'job' in task_spec or 'platform' in task_spec and 'remote' in task_spec): if ('platform' in task_spec and forbidden_with_platform & {*task_spec['job'], *task_spec['remote']}): # Fail Loudly and Horribly raise PlatformLookupError( f"A mixture of Cylc 7 (host) and Cylc 8 (platform logic)" f" should not be used. Task {task_name} set platform " f"and item in {forbidden_with_platform}") elif 'platform' in task_spec: # Return config unchanged continue else: # Add empty dicts if appropriate sections not present. if 'job' in task_spec: task_spec_job = task_spec['job'] else: task_spec_job = {} if 'remote' in task_spec: task_spec_remote = task_spec['remote'] else: task_spec_remote = {} # Deal with case where host is a function and we cannot auto # upgrade at the time of loading the config. if ('host' in task_spec_remote and REC_COMMAND.match(task_spec['remote']['host'])): LOG.debug( f"The host setting of '{task_name}' is a function: " f"Cylc will try to upgrade this task on job submission.") continue # Attempt to use the reverse lookup try: platform = reverse_lookup( glbl_cfg(cached=False).get(['job platforms']), task_spec_job, task_spec_remote) except PlatformLookupError as exc: raise PlatformLookupError(f"for task {task_name}: {exc}") else: # Set platform in config cfg['runtime'][task_name].update({'platform': platform}) LOG.warning(f"Platform {platform} auto selected from ") # Remove deprecated items from config for old_spec_item in forbidden_with_platform: for task_section in ['job', 'remote']: if (task_section in cfg['runtime'][task_name] and old_spec_item in cfg['runtime'][task_name] [task_section].keys()): poppable = cfg['runtime'][task_name][task_section] poppable.pop(old_spec_item) LOG.warning(f"Cylc 7 {old_spec_item} removed.") return cfg
def upgrade_to_platforms(self): """upgrade [job]batch system and [remote]host to platform * Add 'platform' and 'user' columns to table task_jobs. * Remove 'user_at_host' and 'batch_sys_name' columns Returns: bool - True if upgrade performed, False if upgrade skipped. """ conn = self.connect() # check if upgrade required schema = conn.execute(rf'PRAGMA table_info({self.TABLE_TASK_JOBS})') for _, name, *_ in schema: if name == 'platform': LOG.debug('platform column present - skipping db upgrade') return False # Perform upgrade: table = self.TABLE_TASK_JOBS LOG.info('Upgrade to Cylc 8 platforms syntax') conn.execute( rf''' ALTER TABLE {table} ADD COLUMN user TEXT ''' ) conn.execute( rf''' ALTER TABLE {table} ADD COLUMN platform TEXT ''' ) job_platforms = glbl_cfg(cached=False).get(['job platforms']) for cycle, name, user_at_host, batch_system in conn.execute(rf''' SELECT cycle, name, user_at_host, batch_system FROM {table} '''): match = re.match(r"(?P<user>\S+)@(?P<host>\S+)", user_at_host) if match: user = match.group('user') host = match.group('host') else: user = '' host = user_at_host platform = reverse_lookup( job_platforms, {'batch system': batch_system}, {'host': host} ) conn.execute( rf''' UPDATE {table} SET user=?, platform=? WHERE cycle==? AND name==? ''', (user, platform, cycle, name) ) conn.commit() return True