def main(parser, opts, *args):
if opts.list:
print('\n'.join(list_resources()))
sys.exit(0)
elif not args:
print(parser.usage)
sys.exit(0)
target_dir, *resources = args
extract_resources(target_dir, resources or None)
def test_extract_resources_one(self):
"""Test extraction of a specific resource.
Just check that a file of the right name gets extracted, but not its
content - which may change in the future.
"""
tmpdir = tempfile.gettempdir()
extract_resources(tmpdir, resources=['etc/job.sh'])
extracted = os.path.join(tmpdir, 'etc', 'job.sh')
self.assertTrue(os.path.isfile(extracted))
shutil.rmtree(os.path.dirname(extracted), ignore_errors=True)
def scheduler_cli(parser, options, args, is_restart=False):
"""CLI main."""
reg = args[0]
# Check suite is not already running before start of host selection.
try:
suite_files.detect_old_contact_file(reg)
except SuiteServiceFileError as exc:
sys.exit(exc)
suite_run_dir = get_suite_run_dir(reg)
if not os.path.exists(suite_run_dir):
sys.stderr.write(f'suite service directory not found '
f'at: {suite_run_dir}\n')
sys.exit(1)
# Create auth files if needed.
suite_files.create_auth_files(reg)
# Extract job.sh from library, for use in job scripts.
extract_resources(suite_files.get_suite_srv_dir(reg), ['etc/job.sh'])
# Check whether a run host is explicitly specified, else select one.
if not options.host:
try:
host = HostAppointer().appoint_host()
except EmptyHostList as exc:
if cylc.flow.flags.debug:
raise
else:
sys.exit(str(exc))
if is_remote_host(host):
if is_restart:
base_cmd = ["restart"] + sys.argv[1:]
else:
base_cmd = ["run"] + sys.argv[1:]
# Prevent recursive host selection
base_cmd.append("--host=localhost")
return remote_cylc_cmd(base_cmd, host=host)
if remrun(set_rel_local=True): # State localhost as above.
sys.exit()
try:
suite_files.get_suite_source_dir(args[0], options.owner)
except SuiteServiceFileError:
# Source path is assumed to be the run directory
suite_files.register(args[0], get_suite_run_dir(args[0]))
try:
scheduler = Scheduler(is_restart, options, args)
except SuiteServiceFileError as exc:
sys.exit(exc)
scheduler.start()
def test_extract_resources_all(self):
"""Test extraction of all resources under 'etc'.
Just check that file of the right names gets extracted, but not their
content - which may change in the future.
"""
tmpdir = tempfile.gettempdir()
extract_resources(tmpdir, None)
for resource in resource_names:
extracted = os.path.join(tmpdir, resource)
self.assertTrue(os.path.isfile(extracted))
shutil.rmtree(os.path.join(tmpdir, 'etc'), ignore_errors=True)
def remote_init(install_target, rund, indirect_comm=None):
"""cylc remote-init
Arguments:
install_target (str): target to be initialised
rund (str): suite run directory
*indirect_comm (str): use indirect communication via e.g. 'ssh'
"""
rund = os.path.expandvars(rund)
srvd = os.path.join(rund, SuiteFiles.Service.DIRNAME)
os.makedirs(srvd, exist_ok=True)
client_pub_keyinfo = KeyInfo(
KeyType.PUBLIC,
KeyOwner.CLIENT,
suite_srv_dir=srvd, install_target=install_target, server_held=False)
pattern = re.compile(r"^client_\S*key$")
for filepath in os.listdir(srvd):
if pattern.match(filepath) and f"{install_target}" not in filepath:
# client key for a different install target exists
print(REMOTE_INIT_FAILED)
try:
remove_keys_on_client(srvd, install_target)
create_client_keys(srvd, install_target)
except Exception:
# Catching all exceptions as need to fail remote init if any problems
# with key generation.
print(REMOTE_INIT_FAILED)
return
oldcwd = os.getcwd()
os.chdir(rund)
# Extract job.sh from library, for use in job scripts.
extract_resources(SuiteFiles.Service.DIRNAME, ['etc/job.sh'])
try:
tarhandle = tarfile.open(fileobj=sys.stdin.buffer, mode='r|')
tarhandle.extractall()
tarhandle.close()
finally:
os.chdir(oldcwd)
if indirect_comm:
fname = os.path.join(srvd, SuiteFiles.Service.CONTACT)
with open(fname, 'a') as handle:
handle.write('%s=%s\n' % (
ContactFileFields.COMMS_PROTOCOL_2, indirect_comm))
print("KEYSTART", end='')
with open(client_pub_keyinfo.full_key_path) as keyfile:
print(keyfile.read(), end='KEYEND')
print(REMOTE_INIT_DONE)
return
def remote_init(uuid_str, rund, indirect_comm=None):
"""cylc remote-init
Arguments:
uuid_str (str): suite host UUID
rund (str): suite run directory
*indirect_comm (str): use indirect communication via e.g. 'ssh'
"""
rund = os.path.expandvars(rund)
srvd = os.path.join(rund, SuiteSrvFilesManager.DIR_BASE_SRV)
try:
orig_uuid_str = open(os.path.join(srvd, FILE_BASE_UUID)).read()
except IOError:
pass
else:
if orig_uuid_str == uuid_str:
print(REMOTE_INIT_NOT_REQUIRED)
return
os.makedirs(rund, exist_ok=True)
oldcwd = os.getcwd()
os.chdir(rund)
# Extract job.sh from library, for use in job scripts.
extract_resources(SuiteSrvFilesManager.DIR_BASE_SRV, ['etc/job.sh'])
try:
tarhandle = tarfile.open(fileobj=sys.stdin.buffer, mode='r|')
tarhandle.extractall()
tarhandle.close()
finally:
os.chdir(oldcwd)
if indirect_comm:
fname = os.path.join(srvd, SuiteSrvFilesManager.FILE_BASE_CONTACT2)
with open(fname, 'w') as handle:
handle.write(
'%s=%s\n' %
(SuiteSrvFilesManager.KEY_COMMS_PROTOCOL_2, indirect_comm))
print(REMOTE_INIT_DONE)
return
def main(parser, _, target_dir, *resources):
extract_resources(target_dir, resources or None)
def remote_init(install_target, rund, *dirs_to_symlink):
"""cylc remote-init
Arguments:
install_target (str): target to be initialised
rund (str): workflow run directory
dirs_to_symlink (list): directories to be symlinked in form
[directory=symlink_location, ...]
"""
rund = os.path.expandvars(rund)
for item in dirs_to_symlink:
key, val = item.split("=", 1)
if key == 'run':
dst = rund
else:
dst = os.path.join(rund, key)
src = os.path.expandvars(val)
if '$' in src:
print(REMOTE_INIT_FAILED)
print(f'Error occurred when symlinking.'
f' {src} contains an invalid environment variable.')
return
make_symlink(src, dst)
srvd = os.path.join(rund, WorkflowFiles.Service.DIRNAME)
os.makedirs(srvd, exist_ok=True)
client_pub_keyinfo = KeyInfo(
KeyType.PUBLIC,
KeyOwner.CLIENT,
workflow_srv_dir=srvd,
install_target=install_target,
server_held=False
)
# Check for existence of client key dir (should only exist on server)
# Fail if one exists - this may occur on mis-configuration of install
# target in global.cylc
client_key_dir = os.path.join(
srvd, f"{KeyOwner.CLIENT.value}_{KeyType.PUBLIC.value}_keys")
if os.path.exists(client_key_dir):
print(REMOTE_INIT_FAILED)
print(f"Unexpected key directory exists: {client_key_dir}"
" Check global.cylc install target is configured correctly "
"for this platform.")
return
pattern = re.compile(r"^client_\S*key$")
for filepath in os.listdir(srvd):
if pattern.match(filepath) and f"{install_target}" not in filepath:
# client key for a different install target exists
print(REMOTE_INIT_FAILED)
print(f"Unexpected authentication key \"{filepath}\" exists. "
"Check global.cylc install target is configured correctly "
"for this platform.")
return
try:
remove_keys_on_client(srvd, install_target)
create_client_keys(srvd, install_target)
except Exception:
# Catching all exceptions as need to fail remote init if any problems
# with key generation.
print(REMOTE_INIT_FAILED)
return
oldcwd = os.getcwd()
os.chdir(rund)
# Extract job.sh from library, for use in job scripts.
extract_resources(WorkflowFiles.Service.DIRNAME, ['etc/job.sh'])
try:
tarhandle = tarfile.open(fileobj=sys.stdin.buffer, mode='r|')
tarhandle.extractall()
tarhandle.close()
finally:
os.chdir(oldcwd)
print("KEYSTART", end='')
with open(client_pub_keyinfo.full_key_path) as keyfile:
print(keyfile.read(), end='KEYEND')
print(REMOTE_INIT_DONE)
return
def main(parser, options, suite, *task_ids):
"""cylc submit CLI.
No TASK EVENT HOOKS are set for the submit command because there is
no scheduler instance watching for task failure etc.
Note: a suite contact env file is not written by this command (it
would overwrite the real one if the suite is running).
"""
if not options.verbose and not options.debug:
LOG.setLevel(WARNING)
for task_id in task_ids:
if not TaskID.is_valid_id(task_id):
raise UserInputError("Invalid task ID %s" % task_id)
suiterc = get_suite_rc(suite)
suite_dir = os.path.dirname(suiterc)
# For user-defined batch system handlers
sys.path.append(os.path.join(suite_dir, 'python'))
# Load suite config and tasks
config = SuiteConfig(
suite, suiterc, options,
load_template_vars(options.templatevars, options.templatevars_file))
itasks = []
for task_id in task_ids:
name_str, point_str = TaskID.split(task_id)
taskdefs = config.find_taskdefs(name_str)
if not taskdefs:
raise UserInputError("No task found for %s" % task_id)
for taskdef in taskdefs:
itasks.append(
TaskProxy(taskdef,
get_point(point_str).standardise(),
is_startup=True))
# Initialise job submit environment
make_suite_run_tree(suite)
# Extract job.sh from library, for use in job scripts.
extract_resources(get_suite_srv_dir(suite), ['etc/job.sh'])
pool = SubProcPool()
owner = get_user()
job_pool = JobPool(suite, owner)
db_mgr = SuiteDatabaseManager()
task_job_mgr = TaskJobManager(
suite, pool, db_mgr,
TaskEventsManager(suite, pool, db_mgr, BroadcastMgr(db_mgr), job_pool),
job_pool)
task_job_mgr.task_remote_mgr.single_task_mode = True
task_job_mgr.job_file_writer.set_suite_env({
'CYLC_UTC':
str(config.cfg['cylc']['UTC mode']),
'CYLC_DEBUG':
str(cylc.flow.flags.debug).lower(),
'CYLC_VERBOSE':
str(cylc.flow.flags.verbose).lower(),
'CYLC_SUITE_NAME':
suite,
'CYLC_CYCLING_MODE':
str(config.cfg['scheduling']['cycling mode']),
'CYLC_SUITE_INITIAL_CYCLE_POINT':
str(config.cfg['scheduling']['initial cycle point']),
'CYLC_SUITE_FINAL_CYCLE_POINT':
str(config.cfg['scheduling']['final cycle point']),
})
ret_code = 0
waiting_tasks = list(itasks)
if options.dry_run:
while waiting_tasks:
prep_tasks, bad_tasks = task_job_mgr.prep_submit_task_jobs(
suite, waiting_tasks, dry_run=True)
for itask in prep_tasks + bad_tasks:
waiting_tasks.remove(itask)
if waiting_tasks:
task_job_mgr.proc_pool.process()
sleep(1.0)
for itask in itasks:
if itask.local_job_file_path:
print(('JOB SCRIPT=%s' % itask.local_job_file_path))
else:
print(('Unable to prepare job file for %s' % itask.identity),
file=sys.stderr)
ret_code = 1
else:
while waiting_tasks:
for itask in task_job_mgr.submit_task_jobs(suite, waiting_tasks):
waiting_tasks.remove(itask)
if waiting_tasks:
task_job_mgr.proc_pool.process()
sleep(1.0)
while task_job_mgr.proc_pool.is_not_done():
task_job_mgr.proc_pool.process()
for itask in itasks:
if itask.summary.get('submit_method_id') is not None:
print(('[%s] Job ID: %s' %
(itask.identity, itask.summary['submit_method_id'])))
if itask.state(TASK_STATUS_SUBMIT_FAILED):
ret_code = 1
sys.exit(ret_code)
