def start(self):
cmd = WmiUtils.get_wmic_cmd(
username=self.username,
password=self.password,
target_ip=self.target_ip,
columns=["LogfileName"],
table="Win32_NTEventlogFile",
)
self.logfile_com = ExtCom(
self.log, cmd, debug=True,
shell=False) # shell=False since args must not be parsed again
self.logfile_com.run()
def __call__(self, job):
where_clause = "WHERE Logfile = '{}'".format(job.logfile_name)
if job.last_known_record_number is not None:
where_clause += "AND RecordNumber >= {}".format(
job.last_known_record_number)
cmd = WmiUtils.get_wmic_cmd(
username=job.username,
password=job.password,
target_ip=job.target_ip,
columns=["RecordNumber"],
table="Win32_NTLogEvent",
where_clause=where_clause,
)
job.log(
"Querying maximal entry for {} with last known record number {}"
.format(job, job.last_known_record_number))
job.ext_com = ExtCom(
job.log, cmd, debug=True,
shell=False) # shell=False since args must not be parsed again
job.ext_com.run()
job.current_phase = WmiLogEntryJob.FindMaximumPhase()
return True
def start_next_job(self, job):
cmd = (
"/opt/cluster/bin/ipmitool",
"-H",
job.target_ip,
"-U",
job.username,
"-P",
job.password,
"sel",
"get",
hex(self.current_record_id),
)
self.ext_com = ExtCom(
job.log, cmd, shell=False, debug=global_config['DEBUG']
) # shell=False since args must not be parsed again
self.ext_com.run()
def __call__(self, job):
do_continue = True
if self.ext_com is None:
# initial call, start the job
cmd = (
"/opt/cluster/bin/ipmitool",
"-H",
job.target_ip,
"-U",
job.username,
"-P",
job.password,
"sel",
"list",
)
self.ext_com = ExtCom(
job.log, cmd, shell=False, debug=global_config['DEBUG']
) # shell=False since args must not be parsed again
self.ext_com.run()
else:
# job has been started, check
exit_code = self.ext_com.finished()
if exit_code is not None:
stdout_out, stderr_out = self.ext_com.communicate()
if stderr_out:
job._handle_stderr(
stderr_out,
"finding out maximum ipmi sel record id for {}".
format(job))
if exit_code != 0:
raise RuntimeError(
"IpmiLogJob FindMaximumPhase ipmitool command failed with code {}"
.format(exit_code))
# lines look like this (first is record id in hex):
# 67 | 06/17/2015 | 04:20:03 | Temperature #0x32 | Upper Non-critical going high
stdout_lines = stdout_out.split("\n")
record_ids_string = [
line.split("|", 1)[0].strip() for line in stdout_lines
]
record_ids_no_empty_ones = [
entry for entry in record_ids_string if entry
]
record_ids_int = [
int(entry, base=16)
for entry in record_ids_no_empty_ones
]
max_record_id = max(record_ids_int)
if job.last_known_record_id == max_record_id:
job.log(
"Maximal record id same as last known one ({}), we are done"
.format(max_record_id))
do_continue = False
else:
job.log("Need to gather records from {} to {} for {}".
format(job.last_known_record_id, max_record_id,
job))
job.current_phase = IpmiLogJob.RetrieveTime(
job.last_known_record_id, max_record_id)
return do_continue # do_continue
class RetrieveEvents(object):
def __init__(self, last_known_record_id, max_record_id,
device_time_diff):
self.ext_com = None
self.max_record_id = max_record_id
self.current_record_id = last_known_record_id + 1 if last_known_record_id is not None else 1
# 1 is first record id
self.device_time_diff = device_time_diff
def __call__(self, job):
if self.ext_com is not None:
self.check_current_output(job)
do_continue = self.current_record_id <= self.max_record_id
# start next job if there are ids yet to get and if no job is running
if do_continue and (self.ext_com is None
or self.ext_com.finished() is not None):
self.start_next_job(job)
return do_continue
def __unicode(self):
return "RetrieveEvents(cur_record_id={}, max_record_id={})".format(
self.current_record_id, self.max_record_id)
__repr__ = __unicode
def start_next_job(self, job):
cmd = (
"/opt/cluster/bin/ipmitool",
"-H",
job.target_ip,
"-U",
job.username,
"-P",
job.password,
"sel",
"get",
hex(self.current_record_id),
)
self.ext_com = ExtCom(
job.log, cmd, shell=False, debug=global_config['DEBUG']
) # shell=False since args must not be parsed again
self.ext_com.run()
def check_current_output(self, job):
exit_code = self.ext_com.finished()
if exit_code is not None:
stdout_out, stderr_out = self.ext_com.communicate()
if stderr_out:
job._handle_stderr(
stderr_out, "retrieving sel record {}".format(
self.current_record_id))
if exit_code != 0:
raise RuntimeError(
"IpmiLogJob ipmitool command failed with code {}".
format(exit_code))
# entries look like this:
# SEL Record ID : 0067
# Record Type : 02
# Timestamp : 06/17/2015 04:20:03
# [...]
#
# Sensor ID : Ambient Temp (0x32)
# Entity ID : 12.1
# Sensor Type (Threshold) : Temperature
# sections are separated by empty lines (usually without whitespace in between, but handle anyway)
sections_raw = re.compile("\n[ \t]*\n").split(stdout_out)
sections_db_data = []
timestamp = None
for sec in sections_raw:
if sec: # there might be an empty last entry
parsed = job._parse_section(sec)
if parsed:
sections_db_data.append(parsed)
raw_timestamp = parsed.get(
'Timestamp') # only present in one section
if raw_timestamp is not None:
try:
timestamp_device_time = job._parse_ipmi_datetime_string(
raw_timestamp)
except ValueError:
job.log(
"Unparsable time stamp in ipmi entry: {}"
.format(raw_timestamp))
job.log(traceback.format_exc())
else:
timestamp = timestamp_device_time + self.device_time_diff
# timestamp section is first section (timestamp could be None for somewhat broken entries)
sections_db_data = [{
'Timestamp Local':
str(timestamp) if timestamp is not None else None
}] + sections_db_data
keys_ordered = list(
itertools.chain.from_iterable(
iter(sec.keys()) for sec in sections_db_data))
db_entry = {
'parse_date': django.utils.timezone.now(),
'creation_date': timestamp,
'record_id': self.current_record_id,
'keys_ordered': keys_ordered,
'sections': sections_db_data,
'device_pk': job.target_device.pk,
}
job.db.ipmi_event_log.insert(db_entry)
if global_config['DEBUG']:
job.log("feeding db: {}".format(db_entry))
self.current_record_id += 1
def __call__(self, job):
if self.ext_com is None:
# initial call, start the job
cmd = (
"/opt/cluster/bin/ipmitool",
"-H",
job.target_ip,
"-U",
job.username,
"-P",
job.password,
"sel",
"time",
"get",
)
self.ext_com = ExtCom(
job.log, cmd, shell=False, debug=global_config['DEBUG']
) # shell=False since args must not be parsed again
self.ext_com.run()
else:
exit_code = self.ext_com.finished()
if exit_code is not None:
stdout_out, stderr_out = self.ext_com.communicate()
if stderr_out:
job._handle_stderr(stderr_out,
"ipmi sel time get {}".format(job))
if exit_code != 0:
raise RuntimeError(
"IpmiLogJob RetrieveTime ipmitool command failed with code {}"
.format(exit_code))
try:
device_time = job._parse_ipmi_datetime_string(
stdout_out)
except ValueError as e:
job.log(
"Failed to parse time of job {}: {}".format(
job, e), logging_tools.LOG_LEVEL_ERROR)
job.log(traceback.format_exc(),
logging_tools.LOG_LEVEL_ERROR)
job.log("Assuming that device has local time")
device_time = django.utils.timezone.now()
device_time_diff = django.utils.timezone.now(
) - device_time
job.log("Local time is {}, i.e. diff {}, for {}".format(
device_time,
device_time_diff,
job,
))
job.current_phase = IpmiLogJob.RetrieveEvents(
job.last_known_record_id,
self.max_record_id,
device_time_diff=device_time_diff)
return True # do_continue
class WmiLogFileJob(_WmiJobBase):
"""Job to retrieve list of log files from wmi server"""
def __init__(self, *args, **kwargs):
super(WmiLogFileJob, self).__init__(*args, **kwargs)
self.logfile_com = None
def __unicode__(self):
return u"WmiLogFileJob(dev={}, ip={})".format(self.target_device,
self.target_ip)
__repr__ = __unicode__
def start(self):
cmd = WmiUtils.get_wmic_cmd(
username=self.username,
password=self.password,
target_ip=self.target_ip,
columns=["LogfileName"],
table="Win32_NTEventlogFile",
)
self.logfile_com = ExtCom(
self.log, cmd, debug=True,
shell=False) # shell=False since args must not be parsed again
self.logfile_com.run()
def periodic_check(self):
do_continue = True
if self.logfile_com.finished() is not None:
do_continue = False
stdout_out, stderr_out = self.logfile_com.communicate()
if stderr_out:
self._handle_stderr(
stderr_out,
"scanning for wmi log files for {}".format(self))
if self.logfile_com.finished() != 0:
raise RuntimeError(
"Scanning for wmi log files failed with code {}".format(
self.logfile_com.finished()))
else:
parsed = WmiUtils.parse_wmic_output(stdout_out)
logfiles = []
for entry in parsed:
if 'LogfileName' not in entry:
self.log(
"Invalid entry in log file scanning: {}".format(
entry), logging_tools.LOG_LEVEL_WARN)
else:
logfiles.append(entry['LogfileName'])
# logfiles = ["Application"]
self.log("Detected {} wmi logfiles for {}".format(
len(logfiles), self.target_device))
self.db.wmi_logfile.update_one(
filter={'device_pk': self.target_device.pk},
update={
'$set': {
'device_pk': self.target_device.pk,
'date': django.utils.timezone.now(),
'logfiles': logfiles,
}
},
upsert=True)
return do_continue
def __eq__(self, other):
return isinstance(other, WmiLogFileJob) and super(WmiLogFileJob,
self).__eq__(other)
def __call__(self, job):
do_continue = True
com_finished = self.retrieve_ext_com is not None and self.retrieve_ext_com.finished(
) is not None
is_initial = self.retrieve_ext_com is None
if com_finished:
# handle output
stdout_out, stderr_out = self.retrieve_ext_com.communicate()
if stderr_out:
job._handle_stderr(stderr_out,
"RetrieveEvents for {}".format(job))
if self.retrieve_ext_com.finished() != 0:
raise RuntimeError(
"RetrieveEvents wmi command failed with code {}".
format(self.retrieve_ext_com.finished()))
#print 'code', self.retrieve_ext_com.finished()
#_, tmpfilename = tempfile.mkstemp()
#f = open(tmpfilename, 'w')
#f.write(stdout_out)
#f.flush()
#print 'stdout len', len(stdout_out), tmpfilename
#print ' stderr'
#import pprint
#pprint.pprint(stderr_out)
parsed = job._parse_wmi_logfile_output(stdout_out,
try_handle_lists=True,
log=job.log)
# print 'len', len(parsed)
# `parsed` may be empty for RecordNumber-holes
job.log("Found {} log entries between {} and {} for {}".format(
len(parsed),
self.from_record_number,
self.get_next_upper_limit(self.from_record_number),
job,
))
maximal_record_number = self.get_next_upper_limit(
self.from_record_number)
db_entries = []
date_now = django.utils.timezone.now()
for log_entry in parsed:
record_number = log_entry.get('RecordNumber')
if record_number is None:
job.log(
"Warning: WMI log entry without record number, ignoring:",
logging_tools.LOG_LEVEL_WARN)
job.log("{}".format(log_entry))
time_generated = log_entry.get('TimeGenerated')
if time_generated is not None:
try:
time_generated = self.parse_datetime(
time_generated)
except ValueError as e:
job.log(
"Failed to parse datetime from time generated {}: {}"
.format(time_generated, e))
job.log(traceback.format_exc())
time_generated = None
if time_generated is None:
job.log(
"Warning: WMI log entry without TimeGenerated, ignoring:",
logging_tools.LOG_LEVEL_WARN)
job.log("{}".format(log_entry))
else:
db_entries.append({
'time_generated': time_generated,
'logfile_name': job.logfile_name,
'entry': log_entry,
'record_number': record_number,
'device_pk': job.target_device.pk,
})
# DEBUG check
for entry in db_entries:
if list(
job.db.wmi_event_log.find({
'logfile_name':
entry['logfile_name'],
'record_number':
entry["record_number"],
'device_pk':
entry["device_pk"],
})):
raise RuntimeError("DUPLICATE WMI for " +
unicode(entry))
if db_entries:
# must not feed empty list to mongo
job.db.wmi_event_log.insert_many(db_entries)
job.db.wmi_logfile_maximal_record_number.update_one(
filter={
'device_pk': job.target_device.pk,
'logfile_name': job.logfile_name,
},
update={
'$set': {
'device_pk': job.target_device.pk,
'date': date_now,
'maximal_record_number':
maximal_record_number, # this entry may not actually be present
}
},
upsert=True,
)
self.from_record_number = maximal_record_number
job.log(
"New maximal record number: {} (maximum to reach: {}) for {}"
.format(maximal_record_number, self.to_record_number, job))
self.retrieve_ext_com = None
if com_finished or is_initial:
# check whether to start next run
if self.from_record_number >= self.to_record_number:
do_continue = False
job.log("Reached maximal record number {} (by {}).".format(
self.to_record_number, self.from_record_number))
else:
# start next run
cmd = WmiUtils.get_wmic_cmd(
username=job.username,
password=job.password,
target_ip=job.target_ip,
columns=[
"RecordNumber, Message, Category, CategoryString, ComputerName, EventCode, "
+
"EventIdentifier, InsertionStrings, SourceName, TimeGenerated, TimeWritten, "
+ "EventType, Type, User"
],
table="Win32_NTLogEvent",
where_clause=
"WHERE Logfile = '{}' AND RecordNumber > {} and RecordNumber <= {}"
.format(
job.logfile_name,
self.from_record_number,
self.get_next_upper_limit(self.from_record_number),
))
job.log("querying entries from {} to {} for {}".format(
self.from_record_number,
self.get_next_upper_limit(self.from_record_number),
job,
))
self.retrieve_ext_com = ExtCom(
job.log, cmd, debug=True, shell=False
) # shell=False since args must not be parsed again
self.retrieve_ext_com.run()
return do_continue
class RetrieveEventsPhase(object):
PAGINATION_LIMIT = 10000
# start retrieval jobs and handle output until all entries have been retrieved
def __init__(self, from_number, to_record_number):
# this is increased in the process
self.from_record_number = from_number if from_number is not None else 0
# 1 is the first RecordNumber
self.to_record_number = to_record_number
self.retrieve_ext_com = None
def parse_datetime(self, datetime_string):
# https://technet.microsoft.com/en-us/library/ee198928.aspx
# e.g. 20011224113047.000000-480
parts = datetime_string.split(".")
if len(parts) != 2:
raise ValueError(
"Date does not contain 1 dot: {}".format(datetime_string))
base_date = datetime.datetime.strptime(parts[0], "%Y%m%d%H%M%S")
timezone_parts = parts[1].split('-')
if len(timezone_parts) != 2:
raise ValueError(
"Date does not contain proper timezone info: {}".format(
datetime_string))
return base_date.replace(
tzinfo=dateutil.tz.tzoffset("Custom wmi offset",
int(timezone_parts[1]) * 60))
def get_next_upper_limit(self, lower_limit):
return min(lower_limit + self.__class__.PAGINATION_LIMIT,
self.to_record_number)
def __call__(self, job):
do_continue = True
com_finished = self.retrieve_ext_com is not None and self.retrieve_ext_com.finished(
) is not None
is_initial = self.retrieve_ext_com is None
if com_finished:
# handle output
stdout_out, stderr_out = self.retrieve_ext_com.communicate()
if stderr_out:
job._handle_stderr(stderr_out,
"RetrieveEvents for {}".format(job))
if self.retrieve_ext_com.finished() != 0:
raise RuntimeError(
"RetrieveEvents wmi command failed with code {}".
format(self.retrieve_ext_com.finished()))
#print 'code', self.retrieve_ext_com.finished()
#_, tmpfilename = tempfile.mkstemp()
#f = open(tmpfilename, 'w')
#f.write(stdout_out)
#f.flush()
#print 'stdout len', len(stdout_out), tmpfilename
#print ' stderr'
#import pprint
#pprint.pprint(stderr_out)
parsed = job._parse_wmi_logfile_output(stdout_out,
try_handle_lists=True,
log=job.log)
# print 'len', len(parsed)
# `parsed` may be empty for RecordNumber-holes
job.log("Found {} log entries between {} and {} for {}".format(
len(parsed),
self.from_record_number,
self.get_next_upper_limit(self.from_record_number),
job,
))
maximal_record_number = self.get_next_upper_limit(
self.from_record_number)
db_entries = []
date_now = django.utils.timezone.now()
for log_entry in parsed:
record_number = log_entry.get('RecordNumber')
if record_number is None:
job.log(
"Warning: WMI log entry without record number, ignoring:",
logging_tools.LOG_LEVEL_WARN)
job.log("{}".format(log_entry))
time_generated = log_entry.get('TimeGenerated')
if time_generated is not None:
try:
time_generated = self.parse_datetime(
time_generated)
except ValueError as e:
job.log(
"Failed to parse datetime from time generated {}: {}"
.format(time_generated, e))
job.log(traceback.format_exc())
time_generated = None
if time_generated is None:
job.log(
"Warning: WMI log entry without TimeGenerated, ignoring:",
logging_tools.LOG_LEVEL_WARN)
job.log("{}".format(log_entry))
else:
db_entries.append({
'time_generated': time_generated,
'logfile_name': job.logfile_name,
'entry': log_entry,
'record_number': record_number,
'device_pk': job.target_device.pk,
})
# DEBUG check
for entry in db_entries:
if list(
job.db.wmi_event_log.find({
'logfile_name':
entry['logfile_name'],
'record_number':
entry["record_number"],
'device_pk':
entry["device_pk"],
})):
raise RuntimeError("DUPLICATE WMI for " +
unicode(entry))
if db_entries:
# must not feed empty list to mongo
job.db.wmi_event_log.insert_many(db_entries)
job.db.wmi_logfile_maximal_record_number.update_one(
filter={
'device_pk': job.target_device.pk,
'logfile_name': job.logfile_name,
},
update={
'$set': {
'device_pk': job.target_device.pk,
'date': date_now,
'maximal_record_number':
maximal_record_number, # this entry may not actually be present
}
},
upsert=True,
)
self.from_record_number = maximal_record_number
job.log(
"New maximal record number: {} (maximum to reach: {}) for {}"
.format(maximal_record_number, self.to_record_number, job))
self.retrieve_ext_com = None
if com_finished or is_initial:
# check whether to start next run
if self.from_record_number >= self.to_record_number:
do_continue = False
job.log("Reached maximal record number {} (by {}).".format(
self.to_record_number, self.from_record_number))
else:
# start next run
cmd = WmiUtils.get_wmic_cmd(
username=job.username,
password=job.password,
target_ip=job.target_ip,
columns=[
"RecordNumber, Message, Category, CategoryString, ComputerName, EventCode, "
+
"EventIdentifier, InsertionStrings, SourceName, TimeGenerated, TimeWritten, "
+ "EventType, Type, User"
],
table="Win32_NTLogEvent",
where_clause=
"WHERE Logfile = '{}' AND RecordNumber > {} and RecordNumber <= {}"
.format(
job.logfile_name,
self.from_record_number,
self.get_next_upper_limit(self.from_record_number),
))
job.log("querying entries from {} to {} for {}".format(
self.from_record_number,
self.get_next_upper_limit(self.from_record_number),
job,
))
self.retrieve_ext_com = ExtCom(
job.log, cmd, debug=True, shell=False
) # shell=False since args must not be parsed again
self.retrieve_ext_com.run()
return do_continue