def main(parser, opts, *args): if opts.list: print('\n'.join(list_resources())) sys.exit(0) elif not args: print(parser.usage) sys.exit(0) target_dir, *resources = args extract_resources(target_dir, resources or None)
def test_extract_resources_one(self): """Test extraction of a specific resource. Just check that a file of the right name gets extracted, but not its content - which may change in the future. """ tmpdir = tempfile.gettempdir() extract_resources(tmpdir, resources=['etc/job.sh']) extracted = os.path.join(tmpdir, 'etc', 'job.sh') self.assertTrue(os.path.isfile(extracted)) shutil.rmtree(os.path.dirname(extracted), ignore_errors=True)
def scheduler_cli(parser, options, args, is_restart=False): """CLI main.""" reg = args[0] # Check suite is not already running before start of host selection. try: suite_files.detect_old_contact_file(reg) except SuiteServiceFileError as exc: sys.exit(exc) suite_run_dir = get_suite_run_dir(reg) if not os.path.exists(suite_run_dir): sys.stderr.write(f'suite service directory not found ' f'at: {suite_run_dir}\n') sys.exit(1) # Create auth files if needed. suite_files.create_auth_files(reg) # Extract job.sh from library, for use in job scripts. extract_resources(suite_files.get_suite_srv_dir(reg), ['etc/job.sh']) # Check whether a run host is explicitly specified, else select one. if not options.host: try: host = HostAppointer().appoint_host() except EmptyHostList as exc: if cylc.flow.flags.debug: raise else: sys.exit(str(exc)) if is_remote_host(host): if is_restart: base_cmd = ["restart"] + sys.argv[1:] else: base_cmd = ["run"] + sys.argv[1:] # Prevent recursive host selection base_cmd.append("--host=localhost") return remote_cylc_cmd(base_cmd, host=host) if remrun(set_rel_local=True): # State localhost as above. sys.exit() try: suite_files.get_suite_source_dir(args[0], options.owner) except SuiteServiceFileError: # Source path is assumed to be the run directory suite_files.register(args[0], get_suite_run_dir(args[0])) try: scheduler = Scheduler(is_restart, options, args) except SuiteServiceFileError as exc: sys.exit(exc) scheduler.start()
def test_extract_resources_all(self): """Test extraction of all resources under 'etc'. Just check that file of the right names gets extracted, but not their content - which may change in the future. """ tmpdir = tempfile.gettempdir() extract_resources(tmpdir, None) for resource in resource_names: extracted = os.path.join(tmpdir, resource) self.assertTrue(os.path.isfile(extracted)) shutil.rmtree(os.path.join(tmpdir, 'etc'), ignore_errors=True)
def remote_init(install_target, rund, indirect_comm=None): """cylc remote-init Arguments: install_target (str): target to be initialised rund (str): suite run directory *indirect_comm (str): use indirect communication via e.g. 'ssh' """ rund = os.path.expandvars(rund) srvd = os.path.join(rund, SuiteFiles.Service.DIRNAME) os.makedirs(srvd, exist_ok=True) client_pub_keyinfo = KeyInfo( KeyType.PUBLIC, KeyOwner.CLIENT, suite_srv_dir=srvd, install_target=install_target, server_held=False) pattern = re.compile(r"^client_\S*key$") for filepath in os.listdir(srvd): if pattern.match(filepath) and f"{install_target}" not in filepath: # client key for a different install target exists print(REMOTE_INIT_FAILED) try: remove_keys_on_client(srvd, install_target) create_client_keys(srvd, install_target) except Exception: # Catching all exceptions as need to fail remote init if any problems # with key generation. print(REMOTE_INIT_FAILED) return oldcwd = os.getcwd() os.chdir(rund) # Extract job.sh from library, for use in job scripts. extract_resources(SuiteFiles.Service.DIRNAME, ['etc/job.sh']) try: tarhandle = tarfile.open(fileobj=sys.stdin.buffer, mode='r|') tarhandle.extractall() tarhandle.close() finally: os.chdir(oldcwd) if indirect_comm: fname = os.path.join(srvd, SuiteFiles.Service.CONTACT) with open(fname, 'a') as handle: handle.write('%s=%s\n' % ( ContactFileFields.COMMS_PROTOCOL_2, indirect_comm)) print("KEYSTART", end='') with open(client_pub_keyinfo.full_key_path) as keyfile: print(keyfile.read(), end='KEYEND') print(REMOTE_INIT_DONE) return
def remote_init(uuid_str, rund, indirect_comm=None): """cylc remote-init Arguments: uuid_str (str): suite host UUID rund (str): suite run directory *indirect_comm (str): use indirect communication via e.g. 'ssh' """ rund = os.path.expandvars(rund) srvd = os.path.join(rund, SuiteSrvFilesManager.DIR_BASE_SRV) try: orig_uuid_str = open(os.path.join(srvd, FILE_BASE_UUID)).read() except IOError: pass else: if orig_uuid_str == uuid_str: print(REMOTE_INIT_NOT_REQUIRED) return os.makedirs(rund, exist_ok=True) oldcwd = os.getcwd() os.chdir(rund) # Extract job.sh from library, for use in job scripts. extract_resources(SuiteSrvFilesManager.DIR_BASE_SRV, ['etc/job.sh']) try: tarhandle = tarfile.open(fileobj=sys.stdin.buffer, mode='r|') tarhandle.extractall() tarhandle.close() finally: os.chdir(oldcwd) if indirect_comm: fname = os.path.join(srvd, SuiteSrvFilesManager.FILE_BASE_CONTACT2) with open(fname, 'w') as handle: handle.write( '%s=%s\n' % (SuiteSrvFilesManager.KEY_COMMS_PROTOCOL_2, indirect_comm)) print(REMOTE_INIT_DONE) return
def main(parser, _, target_dir, *resources): extract_resources(target_dir, resources or None)
def remote_init(install_target, rund, *dirs_to_symlink): """cylc remote-init Arguments: install_target (str): target to be initialised rund (str): workflow run directory dirs_to_symlink (list): directories to be symlinked in form [directory=symlink_location, ...] """ rund = os.path.expandvars(rund) for item in dirs_to_symlink: key, val = item.split("=", 1) if key == 'run': dst = rund else: dst = os.path.join(rund, key) src = os.path.expandvars(val) if '$' in src: print(REMOTE_INIT_FAILED) print(f'Error occurred when symlinking.' f' {src} contains an invalid environment variable.') return make_symlink(src, dst) srvd = os.path.join(rund, WorkflowFiles.Service.DIRNAME) os.makedirs(srvd, exist_ok=True) client_pub_keyinfo = KeyInfo( KeyType.PUBLIC, KeyOwner.CLIENT, workflow_srv_dir=srvd, install_target=install_target, server_held=False ) # Check for existence of client key dir (should only exist on server) # Fail if one exists - this may occur on mis-configuration of install # target in global.cylc client_key_dir = os.path.join( srvd, f"{KeyOwner.CLIENT.value}_{KeyType.PUBLIC.value}_keys") if os.path.exists(client_key_dir): print(REMOTE_INIT_FAILED) print(f"Unexpected key directory exists: {client_key_dir}" " Check global.cylc install target is configured correctly " "for this platform.") return pattern = re.compile(r"^client_\S*key$") for filepath in os.listdir(srvd): if pattern.match(filepath) and f"{install_target}" not in filepath: # client key for a different install target exists print(REMOTE_INIT_FAILED) print(f"Unexpected authentication key \"{filepath}\" exists. " "Check global.cylc install target is configured correctly " "for this platform.") return try: remove_keys_on_client(srvd, install_target) create_client_keys(srvd, install_target) except Exception: # Catching all exceptions as need to fail remote init if any problems # with key generation. print(REMOTE_INIT_FAILED) return oldcwd = os.getcwd() os.chdir(rund) # Extract job.sh from library, for use in job scripts. extract_resources(WorkflowFiles.Service.DIRNAME, ['etc/job.sh']) try: tarhandle = tarfile.open(fileobj=sys.stdin.buffer, mode='r|') tarhandle.extractall() tarhandle.close() finally: os.chdir(oldcwd) print("KEYSTART", end='') with open(client_pub_keyinfo.full_key_path) as keyfile: print(keyfile.read(), end='KEYEND') print(REMOTE_INIT_DONE) return
def main(parser, options, suite, *task_ids): """cylc submit CLI. No TASK EVENT HOOKS are set for the submit command because there is no scheduler instance watching for task failure etc. Note: a suite contact env file is not written by this command (it would overwrite the real one if the suite is running). """ if not options.verbose and not options.debug: LOG.setLevel(WARNING) for task_id in task_ids: if not TaskID.is_valid_id(task_id): raise UserInputError("Invalid task ID %s" % task_id) suiterc = get_suite_rc(suite) suite_dir = os.path.dirname(suiterc) # For user-defined batch system handlers sys.path.append(os.path.join(suite_dir, 'python')) # Load suite config and tasks config = SuiteConfig( suite, suiterc, options, load_template_vars(options.templatevars, options.templatevars_file)) itasks = [] for task_id in task_ids: name_str, point_str = TaskID.split(task_id) taskdefs = config.find_taskdefs(name_str) if not taskdefs: raise UserInputError("No task found for %s" % task_id) for taskdef in taskdefs: itasks.append( TaskProxy(taskdef, get_point(point_str).standardise(), is_startup=True)) # Initialise job submit environment make_suite_run_tree(suite) # Extract job.sh from library, for use in job scripts. extract_resources(get_suite_srv_dir(suite), ['etc/job.sh']) pool = SubProcPool() owner = get_user() job_pool = JobPool(suite, owner) db_mgr = SuiteDatabaseManager() task_job_mgr = TaskJobManager( suite, pool, db_mgr, TaskEventsManager(suite, pool, db_mgr, BroadcastMgr(db_mgr), job_pool), job_pool) task_job_mgr.task_remote_mgr.single_task_mode = True task_job_mgr.job_file_writer.set_suite_env({ 'CYLC_UTC': str(config.cfg['cylc']['UTC mode']), 'CYLC_DEBUG': str(cylc.flow.flags.debug).lower(), 'CYLC_VERBOSE': str(cylc.flow.flags.verbose).lower(), 'CYLC_SUITE_NAME': suite, 'CYLC_CYCLING_MODE': str(config.cfg['scheduling']['cycling mode']), 'CYLC_SUITE_INITIAL_CYCLE_POINT': str(config.cfg['scheduling']['initial cycle point']), 'CYLC_SUITE_FINAL_CYCLE_POINT': str(config.cfg['scheduling']['final cycle point']), }) ret_code = 0 waiting_tasks = list(itasks) if options.dry_run: while waiting_tasks: prep_tasks, bad_tasks = task_job_mgr.prep_submit_task_jobs( suite, waiting_tasks, dry_run=True) for itask in prep_tasks + bad_tasks: waiting_tasks.remove(itask) if waiting_tasks: task_job_mgr.proc_pool.process() sleep(1.0) for itask in itasks: if itask.local_job_file_path: print(('JOB SCRIPT=%s' % itask.local_job_file_path)) else: print(('Unable to prepare job file for %s' % itask.identity), file=sys.stderr) ret_code = 1 else: while waiting_tasks: for itask in task_job_mgr.submit_task_jobs(suite, waiting_tasks): waiting_tasks.remove(itask) if waiting_tasks: task_job_mgr.proc_pool.process() sleep(1.0) while task_job_mgr.proc_pool.is_not_done(): task_job_mgr.proc_pool.process() for itask in itasks: if itask.summary.get('submit_method_id') is not None: print(('[%s] Job ID: %s' % (itask.identity, itask.summary['submit_method_id']))) if itask.state(TASK_STATUS_SUBMIT_FAILED): ret_code = 1 sys.exit(ret_code)