def load_config(self):
"""Load the suite config."""
if self.suiterc:
is_reload = True
collapsed = self.suiterc.closed_families
else:
is_reload = False
collapsed = []
try:
self.suiterc = SuiteConfig(
self.suite,
self.file,
self.template_vars,
is_reload=is_reload,
collapsed=collapsed,
cli_initial_point_string=self.start_point_string,
vis_start_string=self.start_point_string,
vis_stop_string=self.stop_point_string)
except Exception as exc:
msg = "Failed - parsing error?\n\n" + str(exc)
ERR.error(msg)
dia = gtk.MessageDialog(type=gtk.MESSAGE_ERROR,
buttons=gtk.BUTTONS_OK,
message_format=msg)
dia.run()
dia.destroy()
return False
self.inherit = self.suiterc.get_parent_lists()
return True
def load_config(self):
"""Load the suite config."""
if self.suiterc:
is_reload = True
collapsed = self.suiterc.closed_families
else:
is_reload = False
collapsed = []
try:
self.suiterc = SuiteConfig(
self.suite, self.file, self.template_vars,
is_reload=is_reload, collapsed=collapsed,
cli_initial_point_string=self.start_point_string,
vis_start_string=self.start_point_string,
vis_stop_string=self.stop_point_string)
except Exception as exc:
msg = "Failed - parsing error?\n\n" + str(exc)
ERR.error(msg)
if self.interactive:
dia = gtk.MessageDialog(type=gtk.MESSAGE_ERROR,
buttons=gtk.BUTTONS_OK,
message_format=msg)
dia.run()
dia.destroy()
return False
sys.exit(1)
self.inherit = self.suiterc.get_parent_lists()
return True
def send_retry(self, event_message, event_id, max_n_tries,
retry_intvl_secs):
"""CLI external trigger interface."""
max_n_tries = int(max_n_tries or self.__class__.MAX_N_TRIES)
retry_intvl_secs = float(retry_intvl_secs
or self.__class__.RETRY_INTVL_SECS)
sent = False
i_try = 0
while not sent and i_try < max_n_tries:
i_try += 1
try:
self.put(event_message, event_id)
except Exception as exc:
ERR.error(exc)
OUT.info(self.__class__.MSG_SEND_FAILED % (
i_try,
max_n_tries,
))
if i_try >= max_n_tries:
break
OUT.info(self.__class__.MSG_SEND_RETRY %
(retry_intvl_secs, self.timeout))
sleep(retry_intvl_secs)
else:
if i_try > 1:
OUT.info(self.__class__.MSG_SEND_SUCCEEDED %
(i_try, max_n_tries))
sent = True
break
if not sent:
sys.exit('ERROR: send failed')
return sent
def _prep_submit_task_job(self, suite, itask, dry_run):
"""Prepare a task job submission.
Return itask on a good preparation.
"""
if itask.local_job_file_path and not dry_run:
return itask
try:
job_conf = self._prep_submit_task_job_impl(suite, itask)
local_job_file_path = self.task_events_mgr.get_task_job_log(
suite, itask.point, itask.tdef.name, itask.submit_num,
self.JOB_FILE_BASE)
self.job_file_writer.write(local_job_file_path, job_conf)
except Exception, exc:
# Could be a bad command template.
ERR.error(traceback.format_exc())
LOG.error(traceback.format_exc())
self.task_events_mgr.log_task_job_activity(
SuiteProcContext(
self.JOBS_SUBMIT,
'(prepare job file)', err=exc, ret_code=1),
suite, itask.point, itask.tdef.name)
if not dry_run:
self.task_events_mgr.process_message(
itask, CRITICAL, self.task_events_mgr.EVENT_SUBMIT_FAILED)
return
def _run_event_handlers_callback(self, proc_ctx, abort_on_error=False):
"""Callback on completion of a suite event handler."""
if proc_ctx.ret_code:
msg = '%s EVENT HANDLER FAILED' % proc_ctx.cmd_key[1]
LOG.error(str(proc_ctx))
ERR.error(msg)
if abort_on_error:
raise SuiteEventError(msg)
else:
LOG.info(str(proc_ctx))
def _conditional_is_satisfied(self):
try:
res = eval(self.conditional_expression)
except Exception, exc:
err_msg = str(exc)
if str(exc).find("unexpected EOF") != -1:
err_msg += ("\n(?could be unmatched parentheses in the graph "
"string?)")
ERR.error(err_msg)
raise TriggerExpressionError(
'"' + self.raw_conditional_expression + '"')
def _conditional_is_satisfied(self):
try:
res = eval(self.conditional_expression)
except Exception, exc:
err_msg = str(exc)
if str(exc).find("unexpected EOF") != -1:
err_msg += ("\n(?could be unmatched parentheses in the graph "
"string?)")
ERR.error(err_msg)
raise TriggerExpressionError('"' +
self.raw_conditional_expression + '"')
def restart_upgrade(self):
"""Vacuum/upgrade runtime DB on restart."""
# Backward compat, upgrade database with state file if necessary
suite_run_d = os.path.dirname(os.path.dirname(self.pub_path))
old_pri_db_path = os.path.join(
suite_run_d, 'state', CylcSuiteDAO.OLD_DB_FILE_BASE_NAME)
old_pri_db_path_611 = os.path.join(
suite_run_d, CylcSuiteDAO.OLD_DB_FILE_BASE_NAME_611[0])
old_state_file_path = os.path.join(suite_run_d, "state", "state")
if (os.path.exists(old_pri_db_path) and
os.path.exists(old_state_file_path) and
not os.path.exists(self.pri_path)):
# Upgrade pre-6.11.X runtime database + state file
copy(old_pri_db_path, self.pri_path)
pri_dao = self.get_pri_dao()
pri_dao.upgrade_with_state_file(old_state_file_path)
target = os.path.join(suite_run_d, "state.tar.gz")
cmd = ["tar", "-C", suite_run_d, "-czf", target, "state"]
if call(cmd, stdin=open(os.devnull)) == 0:
rmtree(os.path.join(suite_run_d, "state"), ignore_errors=True)
else:
try:
os.unlink(os.path.join(suite_run_d, "state.tar.gz"))
except OSError:
pass
ERR.error("cannot tar-gzip + remove old state/ directory")
# Remove old files as well
try:
os.unlink(os.path.join(suite_run_d, "cylc-suite-env"))
except OSError:
pass
elif (os.path.exists(old_pri_db_path_611) and
not os.path.exists(self.pri_path)):
# Upgrade 6.11.X runtime database
os.rename(old_pri_db_path_611, self.pri_path)
pri_dao = self.get_pri_dao()
pri_dao.upgrade_from_611()
# Remove old files as well
for name in [
CylcSuiteDAO.OLD_DB_FILE_BASE_NAME_611[1],
"cylc-suite-env"]:
try:
os.unlink(os.path.join(suite_run_d, name))
except OSError:
pass
else:
pri_dao = self.get_pri_dao()
pri_dao.upgrade_pickle_to_json()
# Vacuum the primary/private database file
pri_dao.vacuum()
pri_dao.close()
def restart_upgrade(self):
"""Vacuum/upgrade runtime DB on restart."""
# Backward compat, upgrade database with state file if necessary
suite_run_d = os.path.dirname(os.path.dirname(self.pub_path))
old_pri_db_path = os.path.join(
suite_run_d, 'state', CylcSuiteDAO.OLD_DB_FILE_BASE_NAME)
old_pri_db_path_611 = os.path.join(
suite_run_d, CylcSuiteDAO.OLD_DB_FILE_BASE_NAME_611[0])
old_state_file_path = os.path.join(suite_run_d, "state", "state")
if (os.path.exists(old_pri_db_path) and
os.path.exists(old_state_file_path) and
not os.path.exists(self.pri_path)):
# Upgrade pre-6.11.X runtime database + state file
copy(old_pri_db_path, self.pri_path)
pri_dao = self.get_pri_dao()
pri_dao.upgrade_with_state_file(old_state_file_path)
target = os.path.join(suite_run_d, "state.tar.gz")
cmd = ["tar", "-C", suite_run_d, "-czf", target, "state"]
if call(cmd, stdin=open(os.devnull)) == 0:
rmtree(os.path.join(suite_run_d, "state"), ignore_errors=True)
else:
try:
os.unlink(os.path.join(suite_run_d, "state.tar.gz"))
except OSError:
pass
ERR.error("cannot tar-gzip + remove old state/ directory")
# Remove old files as well
try:
os.unlink(os.path.join(suite_run_d, "cylc-suite-env"))
except OSError:
pass
elif (os.path.exists(old_pri_db_path_611) and
not os.path.exists(self.pri_path)):
# Upgrade 6.11.X runtime database
os.rename(old_pri_db_path_611, self.pri_path)
pri_dao = self.get_pri_dao()
pri_dao.upgrade_from_611()
# Remove old files as well
for name in [
CylcSuiteDAO.OLD_DB_FILE_BASE_NAME_611[1],
"cylc-suite-env"]:
try:
os.unlink(os.path.join(suite_run_d, name))
except OSError:
pass
else:
pri_dao = self.get_pri_dao()
pri_dao.upgrade_pickle_to_json()
# Vacuum the primary/private database file
pri_dao.vacuum()
pri_dao.close()
def __init__(self, suite):
# Suite only needed for back-compat with old clients (see below):
self.suite = suite
self.engine = None
self.port = None
# Figure out the ports we are allowed to use.
base_port = glbl_cfg().get(['communication', 'base port'])
max_ports = glbl_cfg().get(
['communication', 'maximum number of ports'])
self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
random.shuffle(self.ok_ports)
comms_options = glbl_cfg().get(['communication', 'options'])
# HTTP Digest Auth uses MD5 - pretty secure in this use case.
# Extending it with extra algorithms is allowed, but won't be
# supported by most browsers. requests and urllib2 are OK though.
self.hash_algorithm = "MD5"
if "SHA1" in comms_options:
# Note 'SHA' rather than 'SHA1'.
self.hash_algorithm = "SHA"
self.srv_files_mgr = SuiteSrvFilesManager()
self.comms_method = glbl_cfg().get(['communication', 'method'])
self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(
{
'cylc':
self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
suite,
content=True),
'anon':
NO_PASSPHRASE
},
algorithm=self.hash_algorithm)
if self.comms_method == 'http':
self.cert = None
self.pkey = None
else: # if self.comms_method in [None, 'https']:
try:
self.cert = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, suite)
self.pkey = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_PEM, suite)
except SuiteServiceFileError:
ERR.error("no HTTPS/OpenSSL support. Aborting...")
raise CylcError("No HTTPS support. "
"Configure user's global.rc to use HTTP.")
self.start()
def _conditional_is_satisfied(self):
"""Evaluate the prerequisite's condition expression.
Does not cache the result.
"""
try:
res = eval(self.conditional_expression)
except Exception, exc:
err_msg = str(exc)
if str(exc).find("unexpected EOF") != -1:
err_msg += ("\n(?could be unmatched parentheses in the graph "
"string?)")
ERR.error(err_msg)
raise TriggerExpressionError(
'"%s"' % self.get_raw_conditional_expression())
def load_config(self):
if self.suiterc:
is_reload = True
collapsed = self.suiterc.closed_families
else:
is_reload = False
collapsed = []
try:
self.suiterc = SuiteConfig(
self.suite, self.file, self.template_vars,
is_reload=is_reload, collapsed=collapsed,
vis_start_string=self.start_point_string,
vis_stop_string=self.stop_point_string)
except Exception, x:
ERR.error("Failed - parsing error?\n" + str(x))
return False
def __init__(self, suite):
# Suite only needed for back-compat with old clients (see below):
self.suite = suite
self.engine = None
self.port = None
# Figure out the ports we are allowed to use.
base_port = glbl_cfg().get(['communication', 'base port'])
max_ports = glbl_cfg().get(
['communication', 'maximum number of ports'])
self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
random.shuffle(self.ok_ports)
comms_options = glbl_cfg().get(['communication', 'options'])
# HTTP Digest Auth uses MD5 - pretty secure in this use case.
# Extending it with extra algorithms is allowed, but won't be
# supported by most browsers. requests and urllib2 are OK though.
self.hash_algorithm = "MD5"
if "SHA1" in comms_options:
# Note 'SHA' rather than 'SHA1'.
self.hash_algorithm = "SHA"
self.srv_files_mgr = SuiteSrvFilesManager()
self.comms_method = glbl_cfg().get(['communication', 'method'])
self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(
{
'cylc': self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
suite, content=True),
'anon': NO_PASSPHRASE
},
algorithm=self.hash_algorithm)
if self.comms_method == 'http':
self.cert = None
self.pkey = None
else: # if self.comms_method in [None, 'https']:
try:
self.cert = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, suite)
self.pkey = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_PEM, suite)
except SuiteServiceFileError:
ERR.error("no HTTPS/OpenSSL support. Aborting...")
raise CylcError("No HTTPS support. "
"Configure user's global.rc to use HTTP.")
self.start()
def load_config(self):
if self.suiterc:
is_reload = True
collapsed = self.suiterc.closed_families
else:
is_reload = False
collapsed = []
try:
self.suiterc = SuiteConfig(
self.suite, self.file, self.template_vars,
is_reload=is_reload, collapsed=collapsed,
cli_initial_point_string=self.start_point_string,
vis_start_string=self.start_point_string,
vis_stop_string=self.stop_point_string)
except Exception, x:
ERR.error("Failed - parsing error?\n" + str(x))
return False
def _conditional_is_satisfied(self):
"""Evaluate the prerequisite's condition expression.
Does not cache the result.
"""
try:
res = eval(self.conditional_expression)
except (SyntaxError, ValueError) as exc:
err_msg = str(exc)
if str(exc).find("unexpected EOF") != -1:
err_msg += ("\n(?could be unmatched parentheses in the graph "
"string?)")
ERR.error(err_msg)
raise TriggerExpressionError(
'"%s"' % self.get_raw_conditional_expression())
return res
def send_retry(self, event_message, event_id,
max_n_tries, retry_intvl_secs):
"""CLI external trigger interface."""
max_n_tries = int(max_n_tries or self.__class__.MAX_N_TRIES)
retry_intvl_secs = float(
retry_intvl_secs or self.__class__.RETRY_INTVL_SECS)
sent = False
i_try = 0
while not sent and i_try < max_n_tries:
i_try += 1
try:
self.put(event_message, event_id)
except Exception as exc:
ERR.error(exc)
OUT.info(self.__class__.MSG_SEND_FAILED % (
i_try,
max_n_tries,
))
if i_try >= max_n_tries:
break
OUT.info(self.__class__.MSG_SEND_RETRY % (
retry_intvl_secs,
self.timeout
))
sleep(retry_intvl_secs)
else:
if i_try > 1:
OUT.info(self.__class__.MSG_SEND_SUCCEEDED % (
i_try,
max_n_tries
))
sent = True
break
if not sent:
sys.exit('ERROR: send failed')
return sent
