def get_create_job_log_path(cls, suite, task_name, task_point, submit_num):
"""Return a new job log path on the suite host, in two parts.
/part1/part2
* part1: the top level job log directory on the suite host.
* part2: the rest, which is also used on remote task hosts.
The full local job log directory is created if necessary, and its
parent symlinked to NN (submit number).
"""
suite_job_log_dir = GLOBAL_CFG.get_derived_host_item(
suite, "suite job log directory")
the_rest_dir = os.path.join(
str(task_point), task_name, "%02d" % int(submit_num))
the_rest = os.path.join(the_rest_dir, "job")
local_log_dir = os.path.join(suite_job_log_dir, the_rest_dir)
mkdir_p(local_log_dir)
target = os.path.join(os.path.dirname(local_log_dir), "NN")
try:
os.unlink(target)
except OSError:
pass
try:
os.symlink(os.path.basename(local_log_dir), target)
except OSError as exc:
if not exc.filename:
exc.filename = target
raise exc
return suite_job_log_dir, the_rest
def create_directory(self, d, name):
try:
mkdir_p(d)
except Exception, x:
print >> sys.stderr, str(x)
raise GlobalConfigError(
'Failed to create directory "' + name + '"')
def update(self):
#print "Attempting Update"
if ( self.last_update_time is not None and
self.last_update_time >= self.updater.last_update_time ):
if self.action_required:
return True
return False
if self.updater.status == "stopped":
gobject.idle_add(self.connection_lost)
return False
self.updater.set_update(False)
self.task_list = deepcopy(self.updater.task_list)
self.live_graph_movie = self.updater.live_graph_movie
self.live_graph_dir = self.updater.live_graph_dir
states_full = deepcopy(self.updater.state_summary)
fam_states_full = deepcopy(self.updater.fam_state_summary)
self.ancestors = deepcopy(self.updater.ancestors)
self.descendants = deepcopy(self.updater.descendants)
self.all_families = deepcopy(self.updater.all_families)
self.triggering_families = deepcopy(self.updater.triggering_families)
self.global_summary = deepcopy(self.updater.global_summary)
self.updater.set_update(True)
if self.last_update_time is None:
self.first_update = True
if self.live_graph_movie:
try:
mkdir_p( self.live_graph_dir )
except Exception, x:
print >> sys.stderr, x
print >> sys.stderr, "Disabling live graph movie"
self.live_graph_movie = False
def create_directory(self, d, name):
try:
mkdir_p(d)
except Exception, x:
print >> sys.stderr, str(x)
raise GlobalConfigError(
'Failed to create directory "' + name + '"')
def _dump_passphrase_to_dir(self, path, passphrase=None):
"""Dump passphrase to "passphrase" file in "path".
1. File permission should already be user-read-write-only on
creation by mkstemp.
2. The combination of os.fsync and os.rename should guarentee
that we don't end up with an incomplete passphrase file.
3. Perhaps we should use uuid.uuid4() to generate the passphrase?
"""
mkdir_p(path)
from tempfile import NamedTemporaryFile
handle = NamedTemporaryFile(
prefix=self.PASSPHRASE_FILE_BASE, dir=path, delete=False)
# Note: Perhaps a UUID might be better here?
if passphrase is None:
import random
passphrase = ''.join(
random.sample(self.PASSPHRASE_CHARSET, self.PASSPHRASE_LEN))
handle.write(passphrase)
os.fsync(handle.fileno())
handle.close()
passphrase_file_name = os.path.join(
path, self.PASSPHRASE_FILE_BASE)
os.rename(handle.name, passphrase_file_name)
if cylc.flags.verbose:
print 'Generated suite passphrase: %s' % passphrase_file_name
def _job_submit_prepare_remote(self, job_file_path):
"""Prepare a remote job file.
On remote mode, write job file, content from STDIN Modify job
script's CYLC_DIR for this host. Extract job submission method
and job submission command template.
Return (batch_sys_name, batch_sys_submit)
"""
batch_sys_name = None
batch_submit_cmd_tmpl = None
mkdir_p(os.path.dirname(job_file_path))
job_file = open(job_file_path + ".tmp", "w")
while True: # Note: "for line in sys.stdin:" may hang
line = sys.stdin.readline()
if not line:
sys.stdin.close()
break
if line.startswith(self.LINE_PREFIX_CYLC_DIR):
old_line = line
line = "%s'%s'\n" % (self.LINE_PREFIX_CYLC_DIR, os.environ["CYLC_DIR"])
if old_line != line:
job_file.write(self.LINE_UPDATE_CYLC_DIR)
elif line.startswith(self.LINE_PREFIX_BATCH_SYS_NAME):
batch_sys_name = line.replace(self.LINE_PREFIX_BATCH_SYS_NAME, "").strip()
elif line.startswith(self.LINE_PREFIX_BATCH_SUBMIT_CMD_TMPL):
batch_submit_cmd_tmpl = line.replace(self.LINE_PREFIX_BATCH_SUBMIT_CMD_TMPL, "").strip()
job_file.write(line)
job_file.close()
os.rename(job_file_path + ".tmp", job_file_path)
os.chmod(job_file_path, (os.stat(job_file_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
return batch_sys_name, batch_submit_cmd_tmpl
def remote_init(uuid_str, rund):
"""cylc remote-init
Arguments:
uuid_str (str): suite host UUID
rund (str): suite run directory
"""
rund = os.path.expandvars(rund)
srvd = os.path.join(rund, SuiteSrvFilesManager.DIR_BASE_SRV)
try:
orig_uuid_str = open(os.path.join(srvd, 'uuid')).read()
except IOError:
pass
else:
if orig_uuid_str == uuid_str:
print(REMOTE_INIT_NOT_REQUIRED)
return
mkdir_p(rund)
oldcwd = os.getcwd()
os.chdir(rund)
try:
tarhandle = tarfile.open(fileobj=sys.stdin, mode='r|')
tarhandle.extractall()
tarhandle.close()
finally:
os.chdir(oldcwd)
print(REMOTE_INIT_DONE)
return
def remote_init(uuid_str, rund, indirect_comm=None):
"""cylc remote-init
Arguments:
uuid_str (str): suite host UUID
rund (str): suite run directory
*indirect_comm (str): use indirect communication via e.g. 'ssh'
"""
rund = os.path.expandvars(rund)
srvd = os.path.join(rund, SuiteSrvFilesManager.DIR_BASE_SRV)
try:
orig_uuid_str = open(os.path.join(srvd, FILE_BASE_UUID)).read()
except IOError:
pass
else:
if orig_uuid_str == uuid_str:
print(REMOTE_INIT_NOT_REQUIRED)
return
mkdir_p(rund)
oldcwd = os.getcwd()
os.chdir(rund)
try:
tarhandle = tarfile.open(fileobj=sys.stdin, mode='r|')
tarhandle.extractall()
tarhandle.close()
finally:
os.chdir(oldcwd)
if indirect_comm:
fname = os.path.join(srvd, SuiteSrvFilesManager.FILE_BASE_CONTACT2)
with open(fname, 'w') as handle:
handle.write(
'%s=%s\n' %
(SuiteSrvFilesManager.KEY_COMMS_PROTOCOL_2, indirect_comm))
print(REMOTE_INIT_DONE)
return
def remote_init(uuid_str, rund, indirect_comm=None):
"""cylc remote-init
Arguments:
uuid_str (str): suite host UUID
rund (str): suite run directory
*indirect_comm (str): use indirect communication via e.g. 'ssh'
"""
rund = os.path.expandvars(rund)
srvd = os.path.join(rund, SuiteSrvFilesManager.DIR_BASE_SRV)
try:
orig_uuid_str = open(os.path.join(srvd, FILE_BASE_UUID)).read()
except IOError:
pass
else:
if orig_uuid_str == uuid_str:
print(REMOTE_INIT_NOT_REQUIRED)
return
mkdir_p(rund)
oldcwd = os.getcwd()
os.chdir(rund)
try:
tarhandle = tarfile.open(fileobj=sys.stdin, mode='r|')
tarhandle.extractall()
tarhandle.close()
finally:
os.chdir(oldcwd)
if indirect_comm:
fname = os.path.join(srvd, SuiteSrvFilesManager.FILE_BASE_CONTACT2)
with open(fname, 'w') as handle:
handle.write('%s=%s\n' % (
SuiteSrvFilesManager.KEY_COMMS_PROTOCOL_2, indirect_comm))
print(REMOTE_INIT_DONE)
return
def get_create_job_log_path(cls, suite, task_name, task_point, submit_num):
"""Return a new job log path on the suite host, in two parts.
/part1/part2
* part1: the top level job log directory on the suite host.
* part2: the rest, which is also used on remote task hosts.
The full local job log directory is created if necessary, and its
parent symlinked to NN (submit number).
"""
suite_job_log_dir = GLOBAL_CFG.get_derived_host_item(
suite, "suite job log directory")
the_rest_dir = os.path.join(str(task_point), task_name,
"%02d" % int(submit_num))
the_rest = os.path.join(the_rest_dir, "job")
local_log_dir = os.path.join(suite_job_log_dir, the_rest_dir)
mkdir_p(local_log_dir)
target = os.path.join(os.path.dirname(local_log_dir), "NN")
try:
os.unlink(target)
except OSError:
pass
try:
os.symlink(os.path.basename(local_log_dir), target)
except OSError as exc:
if not exc.filename:
exc.filename = target
raise exc
return suite_job_log_dir, the_rest
def create_directory(dir_, name):
"""Create directory. Raise GlobalConfigError on error."""
try:
mkdir_p(dir_)
except OSError as exc:
LOG.exception(exc)
raise GlobalConfigError('Failed to create directory "' + name +
'"')
def create_directory(dir_, name):
"""Create directory. Raise GlobalConfigError on error."""
try:
mkdir_p(dir_)
except OSError as exc:
print >> sys.stderr, str(exc)
raise GlobalConfigError(
'Failed to create directory "' + name + '"')
def create_directory(self, d, name):
"""Create directory. Raise GlobalConfigError on error."""
try:
mkdir_p(d)
except Exception, exc:
print >> sys.stderr, str(exc)
raise GlobalConfigError(
'Failed to create directory "' + name + '"')
def create_directory(dir_, name):
"""Create directory. Raise GlobalConfigError on error."""
try:
mkdir_p(dir_)
except OSError as exc:
print >> sys.stderr, str(exc)
raise GlobalConfigError('Failed to create directory "' + name +
'"')
def create_directory(self, d, name):
"""Create directory. Raise GlobalConfigError on error."""
try:
mkdir_p(d)
except Exception, exc:
print >> sys.stderr, str(exc)
raise GlobalConfigError('Failed to create directory "' + name +
'"')
def append_to_log(self, submit_num, log_type, out=None, err=None):
"""Write new command output to the appropriate log file."""
sub_num = "%02d" % int(submit_num)
dir_ = os.path.join(self.base_path, sub_num)
mkdir_p(dir_)
job_log_handle = open(os.path.join(dir_, "job-activity.log"), "a")
timestamp = get_current_time_string()
self._write_to_log(job_log_handle, timestamp, log_type + "-OUT", out)
self._write_to_log(job_log_handle, timestamp, log_type + "-ERR", err)
job_log_handle.close()
def append_to_log(self, submit_num, log_type, out=None, err=None):
"""Write new command output to the appropriate log file."""
sub_num = "%02d" % int(submit_num)
dir_ = os.path.join(self.base_path, sub_num)
mkdir_p(dir_)
job_log_handle = open(os.path.join(dir_, "job-activity.log"), "a")
timestamp = get_current_time_string()
self._write_to_log(job_log_handle, timestamp, log_type + "-OUT", out)
self._write_to_log(job_log_handle, timestamp, log_type + "-ERR", err)
job_log_handle.close()
def __init__(self, dbpath=None):
self.dbpath = dbpath or REGDB_PATH
# create initial database directory if necessary
if not os.path.exists(self.dbpath):
try:
mkdir_p(self.dbpath)
except OSError as exc:
sys.exit(str(exc))
self.local_passphrases = set()
self.cached_passphrases = {}
self.can_disk_cache_passphrases = {}
def toggle_write_dot_frames(self):
self.write_dot_frames = not self.write_dot_frames
if self.write_dot_frames:
# Create local share dir if necessary (could be a remote suite).
try:
mkdir_p(self.suite_share_dir)
except Exception as exc:
gobject.idle_add(warning_dialog(
"%s\nCannot create graph frames directory." % (str(exc))
).warn)
self.write_dot_frames = False
def toggle_write_dot_frames(self):
self.write_dot_frames = not self.write_dot_frames
if self.write_dot_frames:
# Create local share dir if necessary (could be a remote suite).
try:
mkdir_p(self.suite_share_dir)
except Exception as exc:
gobject.idle_add(warning_dialog(
"%s\nCannot create graph frames directory." % (str(exc))
).warn)
self.write_dot_frames = False
def register(self, reg, source=None):
"""Generate service files for a suite. Record its source location."""
self.detect_old_contact_file(reg)
srv_d = self.get_suite_srv_dir(reg)
target = os.path.join(srv_d, self.FILE_BASE_SOURCE)
if source is None:
try:
# No change if already registered
source_str = os.readlink(target)
except OSError:
# Source path is assumed to be the run directory
source_str = ".."
else:
# Tidy source path
if os.path.basename(source) == self.FILE_BASE_SUITE_RC:
source = os.path.dirname(source)
if not os.path.isabs(source):
# On AIX on GPFS os.path.abspath(source) returns the source
# with full 'fileset' prefix. Manual use of $PWD to absolutize
# a relative path gives a cleaner result.
source = os.path.join(os.getenv("PWD", os.getcwd()), source)
source = os.path.normpath(source)
if (os.path.abspath(source) == os.path.abspath(
os.path.dirname(srv_d))):
source_str = ".."
else:
source_str = source
# Create target if it does not exist.
# Re-create target if it does not point to specified source.
mkdir_p(srv_d)
try:
orig_source_str = os.readlink(target)
except OSError:
os.symlink(source_str, target)
else:
if orig_source_str != source_str:
os.unlink(target)
os.symlink(source_str, target)
# Create a new passphrase for the suite if necessary.
if not self._locate_item(self.FILE_BASE_PASSPHRASE, srv_d):
import random
self._dump_item(
srv_d, self.FILE_BASE_PASSPHRASE, ''.join(
random.sample(self.PASSPHRASE_CHARSET,
self.PASSPHRASE_LEN)))
# Load or create SSL private key for the suite.
pkey_obj = self._get_ssl_pem(srv_d)
# Load or create SSL certificate for the suite.
self._get_ssl_cert(srv_d, pkey_obj)
def register(self, reg, source=None):
"""Generate service files for a suite. Record its source location."""
self.detect_old_contact_file(reg)
srv_d = self.get_suite_srv_dir(reg)
target = os.path.join(srv_d, self.FILE_BASE_SOURCE)
if source is None:
try:
# No change if already registered
source_str = os.readlink(target)
except OSError:
# Source path is assumed to be the run directory
source_str = ".."
else:
# Tidy source path
if os.path.basename(source) == self.FILE_BASE_SUITE_RC:
source = os.path.dirname(source)
if not os.path.isabs(source):
# On AIX on GPFS os.path.abspath(source) returns the source
# with full 'fileset' prefix. Manual use of $PWD to absolutize
# a relative path gives a cleaner result.
source = os.path.join(os.getenv("PWD", os.getcwd()), source)
source = os.path.normpath(source)
if (os.path.abspath(source) ==
os.path.abspath(os.path.dirname(srv_d))):
source_str = ".."
else:
source_str = source
# Create target if it does not exist.
# Re-create target if it does not point to specified source.
mkdir_p(srv_d)
try:
orig_source_str = os.readlink(target)
except OSError:
os.symlink(source_str, target)
else:
if orig_source_str != source_str:
os.unlink(target)
os.symlink(source_str, target)
# Create a new passphrase for the suite if necessary.
if not self._locate_item(self.FILE_BASE_PASSPHRASE, srv_d):
import random
self._dump_item(srv_d, self.FILE_BASE_PASSPHRASE, ''.join(
random.sample(self.PASSPHRASE_CHARSET, self.PASSPHRASE_LEN)))
# Load or create SSL private key for the suite.
pkey_obj = self._get_ssl_pem(srv_d)
# Load or create SSL certificate for the suite.
self._get_ssl_cert(srv_d, pkey_obj)
def reconnect( self ):
self.prev_graph_id = ()
try:
client = cylc_pyro_client.client(
self.cfg.suite,
self.cfg.pphrase,
self.cfg.owner,
self.cfg.host,
self.cfg.pyro_timeout,
self.cfg.port )
self.god = client.get_proxy( 'state_summary' )
self.sinfo = client.get_proxy( 'suite-info' )
# on reconnection retrieve static info
self.family_nodes = self.sinfo.get( 'family nodes' )
self.graphed_family_nodes = self.sinfo.get( 'graphed family nodes' )
self.descendants = self.sinfo.get( 'first-parent descendants' )
self.ancestors = self.sinfo.get('first-parent ancestors' )
self.live_graph_movie, self.live_graph_dir = self.sinfo.get( 'do live graph movie' )
except:
# connection lost
if self.stop_summary is None:
self.stop_summary = dump.get_stop_state_summary(
self.cfg.suite,
self.cfg.owner,
self.cfg.host)
if self.stop_summary is not None and any(self.stop_summary):
self.info_bar.set_stop_summary(self.stop_summary)
return False
else:
self.stop_summary = None
self.status = "connected"
self.first_update = True
self.info_bar.set_status( self.status )
if self.live_graph_movie:
try:
mkdir_p( self.live_graph_dir )
except Exception, x:
print >> sys.stderr, x
print >> sys.stderr, "Disabling live graph movie"
self.live_graph_movie = False
self.first_update = True
self.status = "connected"
self.poll_schd.stop()
self.info_bar.set_status( self.status )
return True
def _dump_item(path, item, value):
"""Dump "value" to a file called "item" in the directory "path".
1. File permission should already be user-read-write-only on
creation by mkstemp.
2. The combination of os.fsync and os.rename should guarantee
that we don't end up with an incomplete file.
"""
mkdir_p(path)
from tempfile import NamedTemporaryFile
handle = NamedTemporaryFile(prefix=item, dir=path, delete=False)
handle.write(value)
os.fsync(handle.fileno())
handle.close()
fname = os.path.join(path, item)
os.rename(handle.name, fname)
LOG.debug('Generated %s', fname)
def create_auth_files(self, reg):
"""Create or renew passphrase and SSL files for suite 'reg'."""
# Suite service directory.
srv_d = self.get_suite_srv_dir(reg)
mkdir_p(srv_d)
# Create a new passphrase for the suite if necessary.
if not self._locate_item(self.FILE_BASE_PASSPHRASE, srv_d):
import random
self._dump_item(
srv_d, self.FILE_BASE_PASSPHRASE, ''.join(
random.sample(self.PASSPHRASE_CHARSET,
self.PASSPHRASE_LEN)))
# Load or create SSL private key for the suite.
pkey_obj = self._get_ssl_pem(srv_d)
# Load or create SSL certificate for the suite.
self._get_ssl_cert(srv_d, pkey_obj)
def _dump_item(self, path, item, value):
"""Dump "value" to a file called "item" in the directory "path".
1. File permission should already be user-read-write-only on
creation by mkstemp.
2. The combination of os.fsync and os.rename should guarentee
that we don't end up with an incomplete file.
"""
mkdir_p(path)
from tempfile import NamedTemporaryFile
handle = NamedTemporaryFile(prefix=item, dir=path, delete=False)
handle.write(value)
os.fsync(handle.fileno())
handle.close()
fname = os.path.join(path, item)
os.rename(handle.name, fname)
if cylc.flags.verbose:
print 'Generated %s' % fname
def _create_job_log_path(self, suite, itask):
"""Create job log directory for a task job, etc.
Create local job directory, and NN symbolic link.
If NN => 01, remove numbered directories with submit numbers greater
than 01.
Return a string in the form "POINT/NAME/SUBMIT_NUM".
"""
job_file_dir = self.task_events_mgr.get_task_job_log(
suite, itask.point, itask.tdef.name, itask.submit_num)
task_log_dir = os.path.dirname(job_file_dir)
if itask.submit_num == 1:
try:
names = os.listdir(task_log_dir)
except OSError:
pass
else:
for name in names:
if name not in ["01", self.task_events_mgr.NN]:
rmtree(
os.path.join(task_log_dir, name),
ignore_errors=True)
else:
rmtree(job_file_dir, ignore_errors=True)
mkdir_p(job_file_dir)
target = os.path.join(task_log_dir, self.task_events_mgr.NN)
source = os.path.basename(job_file_dir)
try:
prev_source = os.readlink(target)
except OSError:
prev_source = None
if prev_source == source:
return
try:
if prev_source:
os.unlink(target)
os.symlink(source, target)
except OSError as exc:
if not exc.filename:
exc.filename = target
raise exc
def _create_job_log_path(self, suite, itask):
"""Create job log directory for a task job, etc.
Create local job directory, and NN symbolic link.
If NN => 01, remove numbered directories with submit numbers greater
than 01.
Return a string in the form "POINT/NAME/SUBMIT_NUM".
"""
job_file_dir = self.task_events_mgr.get_task_job_log(
suite, itask.point, itask.tdef.name, itask.submit_num)
task_log_dir = os.path.dirname(job_file_dir)
if itask.submit_num == 1:
try:
names = os.listdir(task_log_dir)
except OSError:
pass
else:
for name in names:
if name not in ["01", self.task_events_mgr.NN]:
rmtree(
os.path.join(task_log_dir, name),
ignore_errors=True)
else:
rmtree(job_file_dir, ignore_errors=True)
mkdir_p(job_file_dir)
target = os.path.join(task_log_dir, self.task_events_mgr.NN)
source = os.path.basename(job_file_dir)
try:
prev_source = os.readlink(target)
except OSError:
prev_source = None
if prev_source == source:
return
try:
if prev_source:
os.unlink(target)
os.symlink(source, target)
except OSError as exc:
if not exc.filename:
exc.filename = target
raise exc
def generate(self, dir):
pfile = os.path.join(dir, 'passphrase')
if os.path.isfile(pfile):
try:
self.get(pfile)
return
except PassphraseError:
pass
# Note: Perhaps a UUID might be better here?
char_set = string.ascii_uppercase + string.ascii_lowercase + string.digits
self.passphrase = ''.join(random.sample(char_set, 20))
mkdir_p(dir)
f = open(pfile, 'w')
f.write(self.passphrase)
f.close()
# set passphrase file permissions to owner-only
os.chmod(pfile, 0600)
if cylc.flags.verbose:
print 'Generated suite passphrase: %s@%s:%s' % (
user, get_hostname(), pfile)
def _job_submit_prepare_remote(self, job_file_path):
"""Prepare a remote job file.
On remote mode, write job file, content from STDIN Modify job
script's CYLC_DIR for this host. Extract job submission method
and job submission command template.
Return (batch_sys_name, batch_sys_submit)
"""
batch_sys_name = None
submit_opts = {}
mkdir_p(os.path.dirname(job_file_path))
job_file = open(job_file_path + ".tmp", "w")
while True: # Note: "for line in sys.stdin:" may hang
line = sys.stdin.readline()
if not line:
sys.stdin.close()
break
if line.startswith(self.LINE_PREFIX_CYLC_DIR):
old_line = line
line = "%s'%s'\n" % (self.LINE_PREFIX_CYLC_DIR,
os.environ["CYLC_DIR"])
if old_line != line:
job_file.write(self.LINE_UPDATE_CYLC_DIR)
elif line.startswith(self.LINE_PREFIX_BATCH_SYS_NAME):
batch_sys_name = line.replace(self.LINE_PREFIX_BATCH_SYS_NAME,
"").strip()
elif line.startswith(self.LINE_PREFIX_BATCH_SUBMIT_CMD_TMPL):
submit_opts["batch_submit_cmd_tmpl"] = line.replace(
self.LINE_PREFIX_BATCH_SUBMIT_CMD_TMPL, "").strip()
elif line.startswith(self.LINE_PREFIX_EXECUTION_TIME_LIMIT):
submit_opts["execution_time_limit"] = float(
line.replace(self.LINE_PREFIX_BATCH_SUBMIT_CMD_TMPL,
"").strip())
job_file.write(line)
job_file.close()
os.rename(job_file_path + ".tmp", job_file_path)
os.chmod(job_file_path, (os.stat(job_file_path).st_mode | stat.S_IXUSR
| stat.S_IXGRP | stat.S_IXOTH))
return batch_sys_name, submit_opts
def do_rollover(self):
"""Create and rollover log file if necessary."""
# Generate new file name
self.stamp = get_current_time_string(use_basic_format=True)
filename = self.baseFilename + '.' + self.stamp
mkdir_p(os.path.dirname(filename))
# Touch file
with open(filename, 'w+'):
os.utime(filename, None)
# Update symlink
if (os.path.exists(self.baseFilename)
or os.path.lexists(self.baseFilename)):
os.unlink(self.baseFilename)
os.symlink(os.path.basename(filename), self.baseFilename)
# Housekeep log files
arch_len = glbl_cfg().get([self.GLBL_KEY, 'rolling archive length'])
if arch_len:
log_files = glob(self.baseFilename + '.*')
log_files.sort()
while len(log_files) > arch_len:
os.unlink(log_files.pop(0))
# Reopen stream, redirect STDOUT and STDERR to log
if self.stream:
self.stream.close()
self.stream = None
self.stream = self._open()
# Dup STDOUT and STDERR in detach mode
if not self.no_detach:
os.dup2(self.stream.fileno(), sys.stdout.fileno())
os.dup2(self.stream.fileno(), sys.stderr.fileno())
# Emit header records (should only do this for subsequent log files)
for header_record in self.header_records:
if self.FILE_NUM in header_record.__dict__:
# Increment log file number
header_record.__dict__[self.FILE_NUM] += 1
header_record.args = header_record.args[0:-1] + (
header_record.__dict__[self.FILE_NUM], )
logging.FileHandler.emit(self, header_record)
def reconnect( self ):
try:
client = cylc_pyro_client.client(
self.cfg.suite,
self.cfg.pphrase,
self.cfg.owner,
self.cfg.host,
self.cfg.pyro_timeout,
self.cfg.port )
self.god = client.get_proxy( 'state_summary' )
self.remote = client.get_proxy( 'remote' )
except:
if self.stop_summary is None:
self.stop_summary = dump.get_stop_state_summary(
self.cfg.suite,
self.cfg.owner,
self.cfg.host)
if any(self.stop_summary):
self.info_bar.set_stop_summary(self.stop_summary)
return False
else:
self.stop_summary = None
self.family_nodes = self.remote.get_family_nodes()
self.graphed_family_nodes = self.remote.get_graphed_family_nodes()
self.families = self.remote.get_families()
self.live_graph_movie, self.live_graph_dir = self.remote.do_live_graph_movie()
if self.live_graph_movie:
try:
mkdir_p( self.live_graph_dir )
except Exception, x:
print >> sys.stderr, x
raise SuiteConfigError, 'ERROR, illegal dir? ' + self.live_graph_dir
self.status = "connected"
self.info_bar.set_status( self.status )
return True
def update(self):
if not self.updater.connected:
if not self.cleared:
gobject.idle_add(self.clear_graph)
self.cleared = True
return False
self.cleared = False
if ( self.last_update_time is not None and
self.last_update_time >= self.updater.last_update_time ):
if self.action_required:
return True
return False
self.updater.set_update(False)
self.task_list = deepcopy(self.updater.task_list)
self.live_graph_movie = self.updater.live_graph_movie
self.live_graph_dir = self.updater.live_graph_dir
states_full = deepcopy(self.updater.state_summary)
fam_states_full = deepcopy(self.updater.fam_state_summary)
self.ancestors = deepcopy(self.updater.ancestors)
self.descendants = deepcopy(self.updater.descendants)
self.all_families = deepcopy(self.updater.all_families)
self.triggering_families = deepcopy(self.updater.triggering_families)
self.global_summary = deepcopy(self.updater.global_summary)
self.updater.set_update(True)
if self.last_update_time is None:
self.first_update = True
if self.live_graph_movie:
try:
mkdir_p( self.live_graph_dir )
except Exception, x:
print >> sys.stderr, x
print >> sys.stderr, "Disabling live graph movie"
self.live_graph_movie = False
def _dump_certificate_and_key_to_dir(self, path, suite):
"""Dump SSL certificate to "ssl.cert" file in "path"."""
try:
from OpenSSL import crypto
except ImportError:
# OpenSSL not installed, so we can't use HTTPS anyway.
return
host = get_hostname()
altnames = ["DNS:*", "DNS:%s" % host,
"IP:%s" % get_local_ip_address(host)]
# Workaround for https://github.com/kennethreitz/requests/issues/2621
altnames.append("DNS:%s" % get_local_ip_address(host))
# Use suite name as the 'common name', but no more than 64 chars.
cert_common_name = suite
if len(suite) > 64:
cert_common_name = suite[:61] + "..."
# Create a private key.
pkey_obj = crypto.PKey()
pkey_obj.generate_key(crypto.TYPE_RSA, 2048)
# Create a self-signed certificate.
cert_obj = crypto.X509()
cert_obj.get_subject().O = "Cylc"
cert_obj.get_subject().CN = cert_common_name
cert_obj.gmtime_adj_notBefore(0)
cert_obj.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) # 10 years.
cert_obj.set_issuer(cert_obj.get_subject())
cert_obj.set_pubkey(pkey_obj)
cert_obj.add_extensions([
crypto.X509Extension(
"subjectAltName", False, ", ".join(altnames)
)
])
cert_obj.sign(pkey_obj, 'sha256')
mkdir_p(path)
# Work in a user-read-write-only directory for guaranteed safety.
from tempfile import mkdtemp
work_dir = mkdtemp()
pkey_file = os.path.join(work_dir, self.SSL_PRIVATE_KEY_FILE_BASE)
cert_file = os.path.join(work_dir, self.SSL_CERTIFICATE_FILE_BASE)
with open(pkey_file, "w") as file_handle:
file_handle.write(
crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey_obj))
with open(cert_file, "w") as file_handle:
file_handle.write(
crypto.dump_certificate(crypto.FILETYPE_PEM, cert_obj))
import stat
os.chmod(pkey_file, stat.S_IRUSR)
os.chmod(cert_file, stat.S_IRUSR)
pkey_dest_file = os.path.join(path, self.SSL_PRIVATE_KEY_FILE_BASE)
cert_dest_file = os.path.join(path, self.SSL_CERTIFICATE_FILE_BASE)
import shutil
shutil.copy(pkey_file, pkey_dest_file)
shutil.copy(cert_file, cert_dest_file)
shutil.rmtree(work_dir)
if cylc.flags.verbose:
print 'Generated suite SSL certificate: %s' % cert_dest_file
print 'Generated suite SSL private key: %s' % pkey_dest_file
def _jobs_submit_prep_by_stdin(self, job_log_root, job_log_dirs):
"""Prepare job files for submit by reading from STDIN.
Job files are uploaded via STDIN in remote mode. Modify job
files' CYLC_DIR for this host. Extract job submission methods
and job submission command templates from each job file.
Return a list, where each element contains something like:
(job_log_dir, batch_sys_name, submit_opts)
"""
items = [[job_log_dir, None, {}] for job_log_dir in job_log_dirs]
items_map = {}
for item in items:
items_map[item[0]] = item
handle = None
batch_sys_name = None
submit_opts = {}
job_log_dir = None
lines = []
# Get job files from STDIN.
# Modify CYLC_DIR in job file, if necessary.
# Get batch system name and batch submit command template from each job
# file.
# Write job file in correct location.
while True: # Note: "for cur_line in sys.stdin:" may hang
cur_line = sys.stdin.readline()
if not cur_line:
if handle is not None:
handle.close()
break
if cur_line.startswith(self.LINE_PREFIX_CYLC_DIR):
old_line = cur_line
cur_line = "%s'%s'\n" % (
self.LINE_PREFIX_CYLC_DIR, os.environ["CYLC_DIR"])
if old_line != cur_line:
lines.append(self.LINE_UPDATE_CYLC_DIR)
elif cur_line.startswith(self.LINE_PREFIX_BATCH_SYS_NAME):
batch_sys_name = cur_line.replace(
self.LINE_PREFIX_BATCH_SYS_NAME, "").strip()
elif cur_line.startswith(self.LINE_PREFIX_BATCH_SUBMIT_CMD_TMPL):
submit_opts["batch_submit_cmd_tmpl"] = cur_line.replace(
self.LINE_PREFIX_BATCH_SUBMIT_CMD_TMPL, "").strip()
elif cur_line.startswith(self.LINE_PREFIX_EXECUTION_TIME_LIMIT):
submit_opts["execution_time_limit"] = float(cur_line.replace(
self.LINE_PREFIX_EXECUTION_TIME_LIMIT, "").strip())
elif cur_line.startswith(self.LINE_PREFIX_JOB_LOG_DIR):
job_log_dir = cur_line.replace(
self.LINE_PREFIX_JOB_LOG_DIR, "").strip()
mkdir_p(os.path.join(job_log_root, job_log_dir))
handle = open(
os.path.join(job_log_root, job_log_dir, "job.tmp"), "wb")
if handle is None:
lines.append(cur_line)
else:
for line in lines + [cur_line]:
handle.write(line)
lines = []
if cur_line.startswith(self.LINE_PREFIX_EOF + job_log_dir):
handle.close()
# Make it executable
os.chmod(handle.name, (
os.stat(handle.name).st_mode |
stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
# Rename from "*/job.tmp" to "*/job"
os.rename(handle.name, handle.name[:-4])
try:
items_map[job_log_dir][1] = batch_sys_name
items_map[job_log_dir][2] = submit_opts
except KeyError:
pass
handle = None
job_log_dir = None
batch_sys_name = None
submit_opts = {}
return items
Much of the information here should ultimately end up in a sensible
site and host configuration file or similar.
"""
# PYRO CONFIGURATION ###################################################
# base port (the lowest allowed socket number)
pyro_base_port = 7766 # (7766 is the Pyro default)
# max number of ports starting from base port
pyro_port_range = 100 # (100 is the Pyro default)
# SUITE REGISTRATION DATABASE LOCATION #################################
# Local registrations, user-specific
local_regdb_path = os.path.join(os.environ["HOME"], ".cylc", "DB")
# CYLC TEMPORARY DIRECTORY #############################################
try:
cylc_tmpdir = os.environ["CYLC_TMPDIR"]
except KeyError:
# use tempfile.mkdtemp() to create a new temp directory
cylc_tmpdir = mkdtemp(prefix="cylc-")
atexit.register(lambda: shutil.rmtree(cylc_tmpdir))
else:
# if CYLC_TMPDIR was set, create the dir if necessary
try:
mkdir_p(cylc_tmpdir)
except Exception, x:
print >>sys.stderr, x
print >>sys.stderr, "ERROR, conf/CylcGlobals.py: illegal temp dir?", cylc_tmpdir
sys.exit(1)
# print "Cylc Temp Dir is:", cylc_tmpdir
def register(self, reg=None, source=None, redirect=False):
"""Register a suite, or renew its registration.
Create suite service directory and symlink to suite source location.
Args:
reg (str): suite name, default basename($PWD).
source (str): directory location of suite.rc file, default $PWD.
redirect (bool): allow reuse of existing name and run directory.
Return:
The registered suite name (which may be computed here).
Raise:
SuiteServiceFileError:
No suite.rc file found in source location.
Illegal name (can look like a relative path, but not absolute).
Another suite already has this name (unless --redirect).
"""
if reg is None:
reg = os.path.basename(os.getcwd())
if os.path.isabs(reg):
raise SuiteServiceFileError(
"ERROR: suite name cannot be an absolute path: %s" % reg)
if source is not None:
if os.path.basename(source) == self.FILE_BASE_SUITE_RC:
source = os.path.dirname(source)
else:
source = os.getcwd()
# suite.rc must exist so we can detect accidentally reversed args.
source = os.path.abspath(source)
if not os.path.isfile(os.path.join(source, self.FILE_BASE_SUITE_RC)):
raise SuiteServiceFileError("ERROR: no suite.rc in %s" % source)
# Create service dir if necessary.
srv_d = self.get_suite_srv_dir(reg)
mkdir_p(srv_d)
# See if suite already has a source or not
try:
orig_source = os.readlink(
os.path.join(srv_d, self.FILE_BASE_SOURCE))
except OSError:
orig_source = None
else:
if not os.path.isabs(orig_source):
orig_source = os.path.normpath(os.path.join(
srv_d, orig_source))
if orig_source is not None and source != orig_source:
if not redirect:
raise SuiteServiceFileError(
"ERROR: the name '%s' already points to %s.\nUse "
"--redirect to re-use an existing name and run "
"directory." % (reg, orig_source))
LOG.warning(
"the name '%(reg)s' points to %(old)s.\nIt will now"
" be redirected to %(new)s.\nFiles in the existing %(reg)s run"
" directory will be overwritten.\n", {
'reg': reg,
'old': orig_source,
'new': source
})
# Remove symlink to the original suite.
os.unlink(os.path.join(srv_d, self.FILE_BASE_SOURCE))
# Create symlink to the suite, if it doesn't already exist.
if orig_source is None or source != orig_source:
target = os.path.join(srv_d, self.FILE_BASE_SOURCE)
if (os.path.abspath(source) == os.path.abspath(
os.path.dirname(srv_d))):
# If source happens to be the run directory,
# create .service/source -> ..
source_str = ".."
else:
source_str = source
os.symlink(source_str, target)
print('REGISTERED %s -> %s' % (reg, source))
return reg
def _jobs_submit_prep_by_stdin(self, job_log_root, job_log_dirs):
"""Prepare job files for submit by reading from STDIN.
Job files are uploaded via STDIN in remote mode. Modify job
files' CYLC_DIR for this host. Extract job submission methods
and job submission command templates from each job file.
Return a list, where each element contains something like:
(job_log_dir, batch_sys_name, submit_opts)
"""
items = [[job_log_dir, None, {}] for job_log_dir in job_log_dirs]
items_map = {}
for item in items:
items_map[item[0]] = item
handle = None
batch_sys_name = None
submit_opts = {}
job_log_dir = None
lines = []
# Get job files from STDIN.
# Modify CYLC_DIR in job file, if necessary.
# Get batch system name and batch submit command template from each job
# file.
# Write job file in correct location.
while True: # Note: "for cur_line in sys.stdin:" may hang
cur_line = sys.stdin.readline()
if not cur_line:
if handle is not None:
handle.close()
break
if cur_line.startswith(self.LINE_PREFIX_CYLC_DIR):
old_line = cur_line
cur_line = "%s'%s'\n" % (
self.LINE_PREFIX_CYLC_DIR, os.environ["CYLC_DIR"])
if old_line != cur_line:
lines.append(self.LINE_UPDATE_CYLC_DIR)
elif cur_line.startswith(self.LINE_PREFIX_BATCH_SYS_NAME):
batch_sys_name = cur_line.replace(
self.LINE_PREFIX_BATCH_SYS_NAME, "").strip()
elif cur_line.startswith(self.LINE_PREFIX_BATCH_SUBMIT_CMD_TMPL):
submit_opts["batch_submit_cmd_tmpl"] = cur_line.replace(
self.LINE_PREFIX_BATCH_SUBMIT_CMD_TMPL, "").strip()
elif cur_line.startswith(self.LINE_PREFIX_EXECUTION_TIME_LIMIT):
submit_opts["execution_time_limit"] = float(cur_line.replace(
self.LINE_PREFIX_EXECUTION_TIME_LIMIT, "").strip())
elif cur_line.startswith(self.LINE_PREFIX_JOB_LOG_DIR):
job_log_dir = cur_line.replace(
self.LINE_PREFIX_JOB_LOG_DIR, "").strip()
mkdir_p(os.path.join(job_log_root, job_log_dir))
handle = open(
os.path.join(job_log_root, job_log_dir, "job.tmp"), "wb")
if handle is None:
lines.append(cur_line)
else:
for line in lines + [cur_line]:
handle.write(line)
lines = []
if cur_line.startswith(self.LINE_PREFIX_EOF + job_log_dir):
handle.close()
# Make it executable
os.chmod(handle.name, (
os.stat(handle.name).st_mode |
stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
# Rename from "*/job.tmp" to "*/job"
os.rename(handle.name, handle.name[:-4])
try:
items_map[job_log_dir][1] = batch_sys_name
items_map[job_log_dir][2] = submit_opts
except KeyError:
pass
handle = None
job_log_dir = None
batch_sys_name = None
submit_opts = {}
return items
