def _create_checkpointer(self):
if self.use_kvstore_checkpointer:
checkpointer_name = ':'.join([
self.app, self.config_name,
self.kvstore_checkpointer_collection_name
])
checkpointer_name = ':'.join([
self.app, self.config_name,
self.kvstore_checkpointer_collection_name
])
try:
return checkpointer.KVStoreCheckpointer(
checkpointer_name,
self.session_key,
self.app,
owner='nobody',
scheme=self.server_scheme,
host=self.server_host,
port=self.server_port)
except binding.HTTPError as e:
logging.error('Failed to init kvstore checkpointer: %s.',
traceback.format_exc(e))
raise
else:
return checkpointer.FileCheckpointer(self._checkpoint_dir)
def __init__(self, app, meta):
self._app = app
self._input_name = meta['name']
self._code = meta['code']
self._options = _encode_options(meta['customized_options'])
self._file_path = meta['modinput_file']
self._test_id = meta['test_id']
self._server_uri = meta['server_uri']
self._session_key = meta['session_key']
self._checkpoint_dir = meta['checkpoint_dir']
self._ckpter = ckpt.FileCheckpointer(CKPT_DIR)
self._globalsettings = meta.get('global_settings', None)
self._data_inputs_options = meta['data_inputs_options']
self._interval = meta.get('interval', 30)
self._sourcetype = meta['sourcetype']
def _init_ckpt(self):
if self.ckpt is None:
if 'AOB_TEST' in os.environ:
ckpt_dir = self.context_meta.get('checkpoint_dir', tempfile.mkdtemp())
if not os.path.exists(ckpt_dir):
os.makedirs(ckpt_dir)
self.ckpt = checkpointer.FileCheckpointer(ckpt_dir)
else:
if 'server_uri' not in self.context_meta:
raise ValueError('server_uri not found in input meta.')
if 'session_key' not in self.context_meta:
raise ValueError('session_key not found in input meta.')
dscheme, dhost, dport = sutils.extract_http_scheme_host_port(self.context_meta[
'server_uri'])
self.ckpt = checkpointer.KVStoreCheckpointer(self.app + "_checkpointer",
self.context_meta['session_key'], self.app,
scheme=dscheme, host=dhost, port=dport)
def stream_events(self, inputs, ew):
input_items = {}
input_name = list(inputs.inputs.keys())[0]
input_items = inputs.inputs[input_name]
# Create UTC timezone for conversion
utc = pytz.utc
params = {}
start = 0
zenoss_server = input_items.get("zenoss_server")
username = input_items.get("zenoss_username")
zenoss_realm = input_items.get("zenoss_realm")
no_ssl_cert_check = int(input_items.get("no_ssl_cert_check"))
# Since Disable=1 and Enable=0, negate bool() to keep alignment
ssl_cert_check = not bool(no_ssl_cert_check)
cafile = input_items.get("cafile")
interval = int(input_items.get("interval", HOUR))
start_date = input_items.get("start_date")
index_closed = int(input_items.get("index_closed"))
index_cleared = int(input_items.get("index_cleared"))
index_archived = int(input_items.get("index_archived"))
index_suppressed = int(input_items.get("index_suppressed"))
index_repeats = int(input_items.get("index_repeats"))
archive_threshold = int(input_items.get("archive_threshold"))
checkpoint_delete_threshold = int(
input_items.get("checkpoint_delete_threshold"))
tzone = input_items.get("tzone")
proxy_uri = input_items.get("proxy_uri")
proxy_username = input_items.get("proxy_username")
proxy_realm = input_items.get("proxy_realm")
proxy_password = None
meta_configs = self._input_definition.metadata
# Generate logger with input name
_, input_name = (input_name.split('//', 2))
self.logger = log.Logs().get_logger('{}_input'.format(APP_NAME))
# Log level configuration
self.logger.setLevel('INFO')
if index_closed: params = dict(index_closed=True)
if index_cleared: params = dict(index_closed=True)
if index_suppressed: params = dict(index_suppressed=True)
if index_repeats: params = dict(index_repeats=True)
try:
if tzone:
zenoss_tz = pytz.timezone(tzone)
else:
zenoss_tz = pytz.timezone(str(get_localzone()))
except pytz.UnknownTimeZoneError as e:
self.logger.warn(
"Unknown Timezone {} - Using default UTC".format(e))
zenoss_tz = pytz.timezone("utc")
# Get UTC timestamp
utc_now = datetime.utcnow().replace(tzinfo=utc)
# Convert to Zenoss server timezone
now_local = utc_now.astimezone(zenoss_tz)
# Create local time string
now_str = now_local.strftime(DATE_FORMAT)
# Load checkpoint file
self.chk = checkpointer.FileCheckpointer(
meta_configs['checkpoint_dir'])
if self.chk.get("run_from") is None:
# Initializing keys in checkpoint
self.chk.update("run_from", start_date)
self.chk.update("last_run", None)
self.chk.update("last_cleaned", now_str)
try:
device = input_items.get("device")
except Exception:
device = None
# Get password from storage password
try:
password = self.get_password(zenoss_realm, username,
meta_configs['session_key'])
except Exception as e:
self.logger.error(
"Failed to get password for user %s, realm %s. Verify credential account exists. User who scheduled alert must have Admin privileges. - %s"
% (username, zenoss_realm, e))
sys.exit(1)
if proxy_username is not None:
try:
proxy_password = self.get_password(proxy_realm, proxy_username,
meta_configs['session_key'])
except Exception as e:
self.logger.error(
"Failed to get password for user %s, realm %s. Verify credential account exists. User who scheduled alert must have Admin privileges. - %s"
% (proxy_username, proxy_realm, e))
sys.exit(1)
while True:
run_from = self.chk.get("run_from")
# When none --> get ALL events, otherwise from users' specified date
# Work with datetimes in UTC and then convert to timezone of Zenoss server
utc_dt = utc.localize(datetime.utcnow())
now_local = zenoss_tz.normalize(utc_dt.astimezone(zenoss_tz))
now_epoch = calendar.timegm(now_local.utctimetuple())
cur_time = now_local.strftime(DATE_FORMAT)
# Connect to Zenoss web interface and get events
try:
z = ZenossAPI(zenoss_server, username, password, proxy_uri,
proxy_username, proxy_password, ssl_cert_check,
cafile)
except Exception as e:
log_message = "Zenoss Events: Failed to connect to server %s as user %s - Error: %s" % (
zenoss_server, username, e)
self.logger.error("{}. Exiting.".format(log_message))
sys.exit(1)
# Initializing data
events_dict = {
"run_from": self.chk.get("run_from"),
"last_run": self.chk.get("last_run"),
"last_cleaned": self.chk.get("last_cleaned")
}
# Get Events
events_dict = self.get_and_process_events(z, events_dict, ew,
params, device, start,
run_from, index_closed,
index_cleared,
index_suppressed)
# Update last run timestamp
events_dict['last_run'] = cur_time
# Processed archived events
if index_archived:
# Get last archive read, convert and create epoch timestamp
try:
last_archive_read = self.chk.get('last_archive_read')
if last_archive_read is None:
# Key does not exist in checkpoint
raise Exception
archive_delta = self.calc_epoch_delta(
last_archive_read, DATE_FORMAT, now_epoch, zenoss_tz,
HOUR)
except Exception:
last_archive_read = None
archive_delta = 0
# Read the archived events table if it hasn't been read or
# last read exceeds archive threshold
if archive_delta >= archive_threshold or \
not last_archive_read:
log_message = "Zenoss Events: Processing Archived Events\n" % params
self.logger.info(log_message)
self.get_and_process_events(z,
events_dict,
ew,
params,
device,
start,
run_from,
index_closed,
index_cleared,
index_suppressed,
archive=True)
events_dict['last_archive_read'] = cur_time
# Clean checkpoint file
try:
last_cleaned = events_dict['last_cleaned']
if last_cleaned is None:
# Key does not exist in checkpoint
raise Exception
except Exception:
last_cleaned = cur_time
# Check to see if we need to clean the checkpoint file based on the
# checkpoint delta threshold
last_cleaned_delta = self.calc_epoch_delta(last_cleaned,
DATE_FORMAT, now_epoch,
zenoss_tz, DAY)
keys_toclean = []
# Clean checkpoint file of old archive records
if last_cleaned_delta >= CHECKPOINT_CLEAN_FREQUENCY:
for k in events_dict.keys():
if isinstance(events_dict[k],
dict) and 'last_time' in events_dict[k]:
last_time = events_dict[k]['last_time']
epoch_delta = self.calc_epoch_delta(
last_time, "%Y-%m-%d %H:%M:%S", now_epoch,
zenoss_tz, DAY)
if epoch_delta >= int(checkpoint_delete_threshold):
keys_toclean.append(k)
self.chk.delete(k)
# Update checkpoint file
for key in events_dict.keys():
if key in keys_toclean:
continue
# dict2str to save among checkpoints
value = events_dict[key]
if isinstance(value, dict):
value = json.dumps(value)
self.chk.update(key, value)
time.sleep(float(interval))
def __init__(self):
self._ckpter = ckpt.FileCheckpointer(CKPT_DIR)