def load_config(self): """Load the suite config.""" if self.suiterc: is_reload = True collapsed = self.suiterc.closed_families else: is_reload = False collapsed = [] try: self.suiterc = SuiteConfig( self.suite, self.file, self.template_vars, is_reload=is_reload, collapsed=collapsed, cli_initial_point_string=self.start_point_string, vis_start_string=self.start_point_string, vis_stop_string=self.stop_point_string) except Exception as exc: msg = "Failed - parsing error?\n\n" + str(exc) ERR.error(msg) dia = gtk.MessageDialog(type=gtk.MESSAGE_ERROR, buttons=gtk.BUTTONS_OK, message_format=msg) dia.run() dia.destroy() return False self.inherit = self.suiterc.get_parent_lists() return True
def load_config(self): """Load the suite config.""" if self.suiterc: is_reload = True collapsed = self.suiterc.closed_families else: is_reload = False collapsed = [] try: self.suiterc = SuiteConfig( self.suite, self.file, self.template_vars, is_reload=is_reload, collapsed=collapsed, cli_initial_point_string=self.start_point_string, vis_start_string=self.start_point_string, vis_stop_string=self.stop_point_string) except Exception as exc: msg = "Failed - parsing error?\n\n" + str(exc) ERR.error(msg) if self.interactive: dia = gtk.MessageDialog(type=gtk.MESSAGE_ERROR, buttons=gtk.BUTTONS_OK, message_format=msg) dia.run() dia.destroy() return False sys.exit(1) self.inherit = self.suiterc.get_parent_lists() return True
def send_retry(self, event_message, event_id, max_n_tries, retry_intvl_secs): """CLI external trigger interface.""" max_n_tries = int(max_n_tries or self.__class__.MAX_N_TRIES) retry_intvl_secs = float(retry_intvl_secs or self.__class__.RETRY_INTVL_SECS) sent = False i_try = 0 while not sent and i_try < max_n_tries: i_try += 1 try: self.put(event_message, event_id) except Exception as exc: ERR.error(exc) OUT.info(self.__class__.MSG_SEND_FAILED % ( i_try, max_n_tries, )) if i_try >= max_n_tries: break OUT.info(self.__class__.MSG_SEND_RETRY % (retry_intvl_secs, self.timeout)) sleep(retry_intvl_secs) else: if i_try > 1: OUT.info(self.__class__.MSG_SEND_SUCCEEDED % (i_try, max_n_tries)) sent = True break if not sent: sys.exit('ERROR: send failed') return sent
def _prep_submit_task_job(self, suite, itask, dry_run): """Prepare a task job submission. Return itask on a good preparation. """ if itask.local_job_file_path and not dry_run: return itask try: job_conf = self._prep_submit_task_job_impl(suite, itask) local_job_file_path = self.task_events_mgr.get_task_job_log( suite, itask.point, itask.tdef.name, itask.submit_num, self.JOB_FILE_BASE) self.job_file_writer.write(local_job_file_path, job_conf) except Exception, exc: # Could be a bad command template. ERR.error(traceback.format_exc()) LOG.error(traceback.format_exc()) self.task_events_mgr.log_task_job_activity( SuiteProcContext( self.JOBS_SUBMIT, '(prepare job file)', err=exc, ret_code=1), suite, itask.point, itask.tdef.name) if not dry_run: self.task_events_mgr.process_message( itask, CRITICAL, self.task_events_mgr.EVENT_SUBMIT_FAILED) return
def _run_event_handlers_callback(self, proc_ctx, abort_on_error=False): """Callback on completion of a suite event handler.""" if proc_ctx.ret_code: msg = '%s EVENT HANDLER FAILED' % proc_ctx.cmd_key[1] LOG.error(str(proc_ctx)) ERR.error(msg) if abort_on_error: raise SuiteEventError(msg) else: LOG.info(str(proc_ctx))
def _conditional_is_satisfied(self): try: res = eval(self.conditional_expression) except Exception, exc: err_msg = str(exc) if str(exc).find("unexpected EOF") != -1: err_msg += ("\n(?could be unmatched parentheses in the graph " "string?)") ERR.error(err_msg) raise TriggerExpressionError( '"' + self.raw_conditional_expression + '"')
def _conditional_is_satisfied(self): try: res = eval(self.conditional_expression) except Exception, exc: err_msg = str(exc) if str(exc).find("unexpected EOF") != -1: err_msg += ("\n(?could be unmatched parentheses in the graph " "string?)") ERR.error(err_msg) raise TriggerExpressionError('"' + self.raw_conditional_expression + '"')
def restart_upgrade(self): """Vacuum/upgrade runtime DB on restart.""" # Backward compat, upgrade database with state file if necessary suite_run_d = os.path.dirname(os.path.dirname(self.pub_path)) old_pri_db_path = os.path.join( suite_run_d, 'state', CylcSuiteDAO.OLD_DB_FILE_BASE_NAME) old_pri_db_path_611 = os.path.join( suite_run_d, CylcSuiteDAO.OLD_DB_FILE_BASE_NAME_611[0]) old_state_file_path = os.path.join(suite_run_d, "state", "state") if (os.path.exists(old_pri_db_path) and os.path.exists(old_state_file_path) and not os.path.exists(self.pri_path)): # Upgrade pre-6.11.X runtime database + state file copy(old_pri_db_path, self.pri_path) pri_dao = self.get_pri_dao() pri_dao.upgrade_with_state_file(old_state_file_path) target = os.path.join(suite_run_d, "state.tar.gz") cmd = ["tar", "-C", suite_run_d, "-czf", target, "state"] if call(cmd, stdin=open(os.devnull)) == 0: rmtree(os.path.join(suite_run_d, "state"), ignore_errors=True) else: try: os.unlink(os.path.join(suite_run_d, "state.tar.gz")) except OSError: pass ERR.error("cannot tar-gzip + remove old state/ directory") # Remove old files as well try: os.unlink(os.path.join(suite_run_d, "cylc-suite-env")) except OSError: pass elif (os.path.exists(old_pri_db_path_611) and not os.path.exists(self.pri_path)): # Upgrade 6.11.X runtime database os.rename(old_pri_db_path_611, self.pri_path) pri_dao = self.get_pri_dao() pri_dao.upgrade_from_611() # Remove old files as well for name in [ CylcSuiteDAO.OLD_DB_FILE_BASE_NAME_611[1], "cylc-suite-env"]: try: os.unlink(os.path.join(suite_run_d, name)) except OSError: pass else: pri_dao = self.get_pri_dao() pri_dao.upgrade_pickle_to_json() # Vacuum the primary/private database file pri_dao.vacuum() pri_dao.close()
def restart_upgrade(self): """Vacuum/upgrade runtime DB on restart.""" # Backward compat, upgrade database with state file if necessary suite_run_d = os.path.dirname(os.path.dirname(self.pub_path)) old_pri_db_path = os.path.join( suite_run_d, 'state', CylcSuiteDAO.OLD_DB_FILE_BASE_NAME) old_pri_db_path_611 = os.path.join( suite_run_d, CylcSuiteDAO.OLD_DB_FILE_BASE_NAME_611[0]) old_state_file_path = os.path.join(suite_run_d, "state", "state") if (os.path.exists(old_pri_db_path) and os.path.exists(old_state_file_path) and not os.path.exists(self.pri_path)): # Upgrade pre-6.11.X runtime database + state file copy(old_pri_db_path, self.pri_path) pri_dao = self.get_pri_dao() pri_dao.upgrade_with_state_file(old_state_file_path) target = os.path.join(suite_run_d, "state.tar.gz") cmd = ["tar", "-C", suite_run_d, "-czf", target, "state"] if call(cmd, stdin=open(os.devnull)) == 0: rmtree(os.path.join(suite_run_d, "state"), ignore_errors=True) else: try: os.unlink(os.path.join(suite_run_d, "state.tar.gz")) except OSError: pass ERR.error("cannot tar-gzip + remove old state/ directory") # Remove old files as well try: os.unlink(os.path.join(suite_run_d, "cylc-suite-env")) except OSError: pass elif (os.path.exists(old_pri_db_path_611) and not os.path.exists(self.pri_path)): # Upgrade 6.11.X runtime database os.rename(old_pri_db_path_611, self.pri_path) pri_dao = self.get_pri_dao() pri_dao.upgrade_from_611() # Remove old files as well for name in [ CylcSuiteDAO.OLD_DB_FILE_BASE_NAME_611[1], "cylc-suite-env"]: try: os.unlink(os.path.join(suite_run_d, name)) except OSError: pass else: pri_dao = self.get_pri_dao() pri_dao.upgrade_pickle_to_json() # Vacuum the primary/private database file pri_dao.vacuum() pri_dao.close()
def __init__(self, suite): # Suite only needed for back-compat with old clients (see below): self.suite = suite self.engine = None self.port = None # Figure out the ports we are allowed to use. base_port = glbl_cfg().get(['communication', 'base port']) max_ports = glbl_cfg().get( ['communication', 'maximum number of ports']) self.ok_ports = range(int(base_port), int(base_port) + int(max_ports)) random.shuffle(self.ok_ports) comms_options = glbl_cfg().get(['communication', 'options']) # HTTP Digest Auth uses MD5 - pretty secure in this use case. # Extending it with extra algorithms is allowed, but won't be # supported by most browsers. requests and urllib2 are OK though. self.hash_algorithm = "MD5" if "SHA1" in comms_options: # Note 'SHA' rather than 'SHA1'. self.hash_algorithm = "SHA" self.srv_files_mgr = SuiteSrvFilesManager() self.comms_method = glbl_cfg().get(['communication', 'method']) self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain( { 'cylc': self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, suite, content=True), 'anon': NO_PASSPHRASE }, algorithm=self.hash_algorithm) if self.comms_method == 'http': self.cert = None self.pkey = None else: # if self.comms_method in [None, 'https']: try: self.cert = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, suite) self.pkey = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_PEM, suite) except SuiteServiceFileError: ERR.error("no HTTPS/OpenSSL support. Aborting...") raise CylcError("No HTTPS support. " "Configure user's global.rc to use HTTP.") self.start()
def _conditional_is_satisfied(self): """Evaluate the prerequisite's condition expression. Does not cache the result. """ try: res = eval(self.conditional_expression) except Exception, exc: err_msg = str(exc) if str(exc).find("unexpected EOF") != -1: err_msg += ("\n(?could be unmatched parentheses in the graph " "string?)") ERR.error(err_msg) raise TriggerExpressionError( '"%s"' % self.get_raw_conditional_expression())
def load_config(self): if self.suiterc: is_reload = True collapsed = self.suiterc.closed_families else: is_reload = False collapsed = [] try: self.suiterc = SuiteConfig( self.suite, self.file, self.template_vars, is_reload=is_reload, collapsed=collapsed, vis_start_string=self.start_point_string, vis_stop_string=self.stop_point_string) except Exception, x: ERR.error("Failed - parsing error?\n" + str(x)) return False
def __init__(self, suite): # Suite only needed for back-compat with old clients (see below): self.suite = suite self.engine = None self.port = None # Figure out the ports we are allowed to use. base_port = glbl_cfg().get(['communication', 'base port']) max_ports = glbl_cfg().get( ['communication', 'maximum number of ports']) self.ok_ports = range(int(base_port), int(base_port) + int(max_ports)) random.shuffle(self.ok_ports) comms_options = glbl_cfg().get(['communication', 'options']) # HTTP Digest Auth uses MD5 - pretty secure in this use case. # Extending it with extra algorithms is allowed, but won't be # supported by most browsers. requests and urllib2 are OK though. self.hash_algorithm = "MD5" if "SHA1" in comms_options: # Note 'SHA' rather than 'SHA1'. self.hash_algorithm = "SHA" self.srv_files_mgr = SuiteSrvFilesManager() self.comms_method = glbl_cfg().get(['communication', 'method']) self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain( { 'cylc': self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_PASSPHRASE, suite, content=True), 'anon': NO_PASSPHRASE }, algorithm=self.hash_algorithm) if self.comms_method == 'http': self.cert = None self.pkey = None else: # if self.comms_method in [None, 'https']: try: self.cert = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_CERT, suite) self.pkey = self.srv_files_mgr.get_auth_item( self.srv_files_mgr.FILE_BASE_SSL_PEM, suite) except SuiteServiceFileError: ERR.error("no HTTPS/OpenSSL support. Aborting...") raise CylcError("No HTTPS support. " "Configure user's global.rc to use HTTP.") self.start()
def load_config(self): if self.suiterc: is_reload = True collapsed = self.suiterc.closed_families else: is_reload = False collapsed = [] try: self.suiterc = SuiteConfig( self.suite, self.file, self.template_vars, is_reload=is_reload, collapsed=collapsed, cli_initial_point_string=self.start_point_string, vis_start_string=self.start_point_string, vis_stop_string=self.stop_point_string) except Exception, x: ERR.error("Failed - parsing error?\n" + str(x)) return False
def _conditional_is_satisfied(self): """Evaluate the prerequisite's condition expression. Does not cache the result. """ try: res = eval(self.conditional_expression) except (SyntaxError, ValueError) as exc: err_msg = str(exc) if str(exc).find("unexpected EOF") != -1: err_msg += ("\n(?could be unmatched parentheses in the graph " "string?)") ERR.error(err_msg) raise TriggerExpressionError( '"%s"' % self.get_raw_conditional_expression()) return res
def send_retry(self, event_message, event_id, max_n_tries, retry_intvl_secs): """CLI external trigger interface.""" max_n_tries = int(max_n_tries or self.__class__.MAX_N_TRIES) retry_intvl_secs = float( retry_intvl_secs or self.__class__.RETRY_INTVL_SECS) sent = False i_try = 0 while not sent and i_try < max_n_tries: i_try += 1 try: self.put(event_message, event_id) except Exception as exc: ERR.error(exc) OUT.info(self.__class__.MSG_SEND_FAILED % ( i_try, max_n_tries, )) if i_try >= max_n_tries: break OUT.info(self.__class__.MSG_SEND_RETRY % ( retry_intvl_secs, self.timeout )) sleep(retry_intvl_secs) else: if i_try > 1: OUT.info(self.__class__.MSG_SEND_SUCCEEDED % ( i_try, max_n_tries )) sent = True break if not sent: sys.exit('ERROR: send failed') return sent