def prologue(self): """Checks the options given for settings and takes appropriate action. See _merge_options for the format. - if nagios_report is set, creates a SimpleNagios instance and prints the report. - if ha is set, checks if running on the correct host, set the appropriate nagios message and bail if not. - if locking_filename is set, take a lock. If the lock fails, bork and set the nagios exit accordingly. """ # bail if nagios report is requested self.nagios_reporter = SimpleNagios( _cache=self.options.nagios_check_filename, _report_and_exit=self.options.nagios_report, _threshold=self.options.nagios_check_interval_threshold, _cache_user=self.options.nagios_user, ) # check for HA host if self.options.ha and not proceed_on_ha_service(self.options.ha): self.log.warning( "Not running on the target host %s in the HA setup. Stopping." % (self.options.ha, )) self.nagios_reporter.ok("Not running on the HA master.") sys.exit(NAGIOS_EXIT_OK) if not self.options.disable_locking and not self.options.dry_run: self.lockfile = TimestampedPidLockfile( self.options.locking_filename, threshold=self.options.nagios_check_interval_threshold * 2) lock_or_bork(self.lockfile, self.nagios_reporter) self.log.info("%s has started" % (_script_name(sys.argv[0])))
def prologue(self): """Checks the options given for settings and takes appropriate action. See _merge_options for the format. - if nagios_report is set, creates a SimpleNagios instance and prints the report. - if ha is set, checks if running on the correct host, set the appropriate nagios message and bail if not. - if locking_filename is set, take a lock. If the lock fails, bork and set the nagios exit accordingly. """ # bail if nagios report is requested self.nagios_reporter = SimpleNagios(_cache=self.options.nagios_check_filename, _report_and_exit=self.options.nagios_report, _threshold=self.options.nagios_check_interval_threshold, _cache_user=self.options.nagios_user, ) # check for HA host if self.options.ha and not proceed_on_ha_service(self.options.ha): self.log.warning("Not running on the target host %s in the HA setup. Stopping." % (self.options.ha,)) self.nagios_reporter.ok("Not running on the HA master.") sys.exit(NAGIOS_EXIT_OK) if not self.options.disable_locking and not self.options.dry_run: self.lockfile = TimestampedPidLockfile(self.options.locking_filename, threshold=self.options.nagios_check_interval_threshold * 2) lock_or_bork(self.lockfile, self.nagios_reporter) self.log.info("%s has started" % (_script_name(sys.argv[0])))
def main(args): """Main script.""" options = { 'nagios': ('print out nagion information', None, 'store_true', False, 'n'), 'nagios_check_filename': ('filename of where the nagios check data is stored', str, 'store', NAGIOS_CHECK_FILENAME), 'nagios_check_interval_threshold': ('threshold of nagios checks timing out', None, 'store', NAGIOS_CHECK_INTERVAL_THRESHOLD), 'mail-report': ('mail a report to the hpc-admin list with job list for gracing or inactive users', None, 'store_true', False), 'ha': ('high-availability master IP address', None, 'store', None), 'dry-run': ('do not make any updates whatsoever', None, 'store_true', False), } opts = simple_option(options) nagios_reporter = NagiosReporter(NAGIOS_HEADER, NAGIOS_CHECK_FILENAME, NAGIOS_CHECK_INTERVAL_THRESHOLD) if opts.options.nagios: nagios_reporter.report_and_exit() sys.exit(0) # not reached if not proceed_on_ha_service(opts.options.ha): logger.warning("Not running on the target host in the HA setup. Stopping.") nagios_reporter(NAGIOS_EXIT_WARNING, NagiosResult("Not running on the HA master.")) sys.exit(NAGIOS_EXIT_WARNING) try: vsc_config = VscConfiguration() LdapQuery(vsc_config) grace_users = get_user_with_status('grace') inactive_users = get_user_with_status('inactive') pbs_query = PBSQuery() t = time.ctime() jobs = pbs_query.getjobs() # we just get them all removed_queued = remove_queued_jobs(jobs, grace_users, inactive_users, opts.options.dry_run) removed_running = remove_running_jobs(jobs, inactive_users, opts.options.dry_run) if opts.options.mail_report and not opts.options.dry_run: if len(removed_queued) > 0 or len(removed_running) > 0: mail_report(t, removed_queued, removed_running) except Exception, err: logger.exception("Something went wrong: {err}".format(err=err)) nagios_reporter.cache(NAGIOS_EXIT_CRITICAL, NagiosResult("Script failed, check log file ({logfile})".format(logfile=PBS_CHECK_LOG_FILE))) sys.exit(NAGIOS_EXIT_CRITICAL)
def main(): """Main function""" options = { 'nagios_check_filename': ('filename of where the nagios check data is stored', str, 'store', NAGIOS_CHECK_FILENAME), 'nagios_check_interval_threshold': ('threshold of nagios checks timing out', None, 'store', NAGIOS_CHECK_INTERVAL_THRESHOLD), 'hosts': ('the hosts/clusters that should be contacted for job information', None, 'extend', []), 'location': ('the location for storing the pickle file: gengar, muk', str, 'store', 'gengar'), 'ha': ('high-availability master IP address', None, 'store', None), 'dry-run': ('do not make any updates whatsoever', None, 'store_true', False), } opts = simple_option(options) nag = SimpleNagios(_cache=NAGIOS_CHECK_FILENAME) if opts.options.ha and not proceed_on_ha_service(opts.options.ha): _log.info("Not running on the target host in the HA setup. Stopping.") nag.ok("Not running on the HA master.") else: # parse config file clusters = {} for host in opts.options.hosts: master = opts.configfile_parser.get(host, "master") showq_path = opts.configfile_parser.get(host, "showq_path") mjobctl_path = opts.configfile_parser.get(host, "mjobctl_path") clusters[host] = { 'master': master, 'spath': showq_path, 'mpath': mjobctl_path, } # process the new and previous data released_jobids, stats = process_hold(clusters, dry_run=opts.options.dry_run) # nagios state stats.update(RELEASEJOB_LIMITS) stats['message'] = "released %s jobs in hold" % len(released_jobids) nag._eval_and_exit(**stats) _log.info("Cached nagios state: %s %s" % (nag._final_state[0][1], nag._final_state[1]))
def main(): """The main.""" # Note: debug option is provided by generaloption # Note: other settings, e.g., ofr each cluster will be obtained from the configuration file options = { 'nagios': ('print out nagios information', None, 'store_true', False, 'n'), 'nagios-check-filename': ('filename of where the nagios check data is stored', str, 'store', NAGIOS_CHECK_FILENAME), 'nagios-check-interval-threshold': ('threshold of nagios checks timing out', None, 'store', NAGIOS_CHECK_INTERVAL_THRESHOLD), 'location': ('path to store the gzipped files', None, 'store', QUOTA_LOG_ZIP_PATH), 'ha': ('high-availability master IP address', None, 'store', None), 'dry-run': ('do not make any updates whatsoever', None, 'store_true', False), } opts = simple_option(options) nagios_reporter = NagiosReporter(NAGIOS_HEADER, opts.options.nagios_check_filename, opts.options.nagios_check_interval_threshold) if opts.options.nagios: logger.debug("Producing Nagios report and exiting.") nagios_reporter.report_and_exit() sys.exit(0) # not reached if not proceed_on_ha_service(opts.options.ha): logger.warning("Not running on the target host in the HA setup. Stopping.") nagios_reporter.cache(NAGIOS_EXIT_WARNING, NagiosResult("Not running on the HA master.")) sys.exit(NAGIOS_EXIT_WARNING) lockfile = TimestampedPidLockfile(QUOTA_LOG_LOCK_FILE) lock_or_bork(lockfile, nagios_reporter) logger.info("starting quota_log run") filesystem_error = 0 filesystem_ok = 0 error = False try: gpfs = GpfsOperations() quota = gpfs.list_quota() for key in quota: try: filename = "gpfs_quota_%s_%s.gz" % (time.strftime("%Y%m%d-%H:%M"), key) path = os.path.join(opts.options.location, filename) zipfile = gzip.open(path, 'wb', 9) # Compress to the max zipfile.write(json.dumps(quota[key])) zipfile.close() filesystem_ok += 1 logger.info("Stored quota information for FS %s" % (key)) except Exception, err: logger.exception("Failed storing quota information for FS %s" % (key)) filesystem_error += 1 except Exception, err: logger.exception("Failure obtaining GPFS quota") error = True
def main(): # Collect all info # Note: debug option is provided by generaloption # Note: other settings, e.g., ofr each cluster will be obtained from the configuration file options = { "nagios": ("print out nagion information", None, "store_true", False, "n"), "nagios_check_filename": ( "filename of where the nagios check data is stored", str, "store", NAGIOS_CHECK_FILENAME, ), "nagios_check_interval_threshold": ( "threshold of nagios checks timing out", None, "store", NAGIOS_CHECK_INTERVAL_THRESHOLD, ), "hosts": ("the hosts/clusters that should be contacted for job information", None, "extend", []), "information": ("the sort of information to store: user, vo, project", None, "store", "user"), "location": ("the location for storing the pickle file: gengar, muk", str, "store", "gengar"), "ha": ("high-availability master IP address", None, "store", None), "dry-run": ("do not make any updates whatsoever", None, "store_true", False), } opts = simple_option(options) if opts.options.debug: fancylogger.setLogLevelDebug() nagios_reporter = NagiosReporter(NAGIOS_HEADER, NAGIOS_CHECK_FILENAME, NAGIOS_CHECK_INTERVAL_THRESHOLD) if opts.options.nagios: logger.debug("Producing Nagios report and exiting.") nagios_reporter.report_and_exit() sys.exit(0) # not reached if not proceed_on_ha_service(opts.options.ha): logger.warning("Not running on the target host in the HA setup. Stopping.") nagios_reporter.cache(NAGIOS_EXIT_WARNING, NagiosResult("Not running on the HA master.")) sys.exit(NAGIOS_EXIT_WARNING) lockfile = TimestampedPidLockfile(DSHOWQ_LOCK_FILE) lock_or_bork(lockfile, nagios_reporter) logger.info("starting dshowq run") clusters = {} for host in opts.options.hosts: master = opts.configfile_parser.get(host, "master") showq_path = opts.configfile_parser.get(host, "showq_path") clusters[host] = {"master": master, "path": showq_path} showq = Showq(clusters, cache_pickle=True, dry_run=opts.options.dry_run) (queue_information, reported_hosts, failed_hosts) = showq.get_moab_command_information() timeinfo = time.time() active_users = queue_information.keys() logger.debug("Active users: %s" % (active_users)) logger.debug("Queue information: %s" % (queue_information)) # We need to determine which users should get an updated pickle. This depends on # - the active user set # - the information we want to provide on the cluster(set) where this script runs # At the same time, we need to determine the job information each user gets to see (target_users, target_queue_information, user_map) = determine_target_information( opts.options.information, active_users, queue_information ) nagios_user_count = 0 nagios_no_store = 0 LdapQuery(VscConfiguration()) for user in target_users: if not opts.options.dry_run: try: (path, store) = get_pickle_path(opts.options.location, user) user_queue_information = target_queue_information[user] user_queue_information["timeinfo"] = timeinfo store(user, path, (user_queue_information, user_map[user])) nagios_user_count += 1 except (UserStorageError, FileStoreError, FileMoveError), err: logger.error("Could not store pickle file for user %s" % (user)) nagios_no_store += 1 else: logger.info( "Dry run, not actually storing data for user %s at path %s" % (user, get_pickle_path(opts.options.location, user)[0]) ) logger.debug("Dry run, queue information for user %s is %s" % (user, target_queue_information[user]))
def main(): # Collect all info # Note: debug option is provided by generaloption # Note: other settings, e.g., ofr each cluster will be obtained from the configuration file options = { "nagios": ("print out nagios information", None, "store_true", False, "n"), "nagios_check_filename": ( "filename of where the nagios check data is stored", str, "store", NAGIOS_CHECK_FILENAME, ), "nagios_check_interval_threshold": ( "threshold of nagios checks timing out", None, "store", NAGIOS_CHECK_INTERVAL_THRESHOLD, ), "hosts": ("the hosts/clusters that should be contacted for job information", None, "extend", []), "location": ("the location for storing the pickle file: home, scratch", str, "store", "home"), "ha": ("high-availability master IP address", None, "store", None), "dry-run": ("do not make any updates whatsoever", None, "store_true", False), } opts = simple_option(options) if opts.options.debug: fancylogger.setLogLevelDebug() nagios_reporter = NagiosReporter( NAGIOS_HEADER, opts.options.nagios_check_filename, opts.options.nagios_check_interval_threshold ) if opts.options.nagios: logger.debug("Producing Nagios report and exiting.") nagios_reporter.report_and_exit() sys.exit(0) # not reached if not proceed_on_ha_service(opts.options.ha): logger.warning("Not running on the target host in the HA setup. Stopping.") nagios_reporter.cache(NAGIOS_EXIT_WARNING, NagiosResult("Not running on the HA master.")) sys.exit(NAGIOS_EXIT_WARNING) lockfile = TimestampedPidLockfile(DCHECKJOB_LOCK_FILE) lock_or_bork(lockfile, nagios_reporter) logger.info("Starting dcheckjob") LdapQuery(VscConfiguration()) clusters = {} for host in opts.options.hosts: master = opts.configfile_parser.get(host, "master") checkjob_path = opts.configfile_parser.get(host, "checkjob_path") clusters[host] = {"master": master, "path": checkjob_path} checkjob = Checkjob(clusters, cache_pickle=True, dry_run=True) (job_information, reported_hosts, failed_hosts) = checkjob.get_moab_command_information() timeinfo = time.time() active_users = job_information.keys() logger.debug("Active users: %s" % (active_users)) logger.debug("Checkjob information: %s" % (job_information)) nagios_user_count = 0 nagios_no_store = 0 for user in active_users: if not opts.options.dry_run: try: (path, store) = get_pickle_path(opts.options.location, user) user_queue_information = CheckjobInfo({user: job_information[user]}) store(user, path, (timeinfo, user_queue_information)) nagios_user_count += 1 except (UserStorageError, FileStoreError, FileMoveError), _: logger.error("Could not store pickle file for user %s" % (user)) nagios_no_store += 1 else: logger.info( "Dry run, not actually storing data for user %s at path %s" % (user, get_pickle_path(opts.options.location, user)[0]) ) logger.debug("Dry run, queue information for user %s is %s" % (user, job_information[user]))