def __call__(self, job):
where_clause = "WHERE Logfile = '{}'".format(job.logfile_name)
if job.last_known_record_number is not None:
where_clause += "AND RecordNumber >= {}".format(
job.last_known_record_number)
cmd = WmiUtils.get_wmic_cmd(
username=job.username,
password=job.password,
target_ip=job.target_ip,
columns=["RecordNumber"],
table="Win32_NTLogEvent",
where_clause=where_clause,
)
job.log(
"Querying maximal entry for {} with last known record number {}"
.format(job, job.last_known_record_number))
job.ext_com = ExtCom(
job.log, cmd, debug=True,
shell=False) # shell=False since args must not be parsed again
job.ext_com.run()
job.current_phase = WmiLogEntryJob.FindMaximumPhase()
return True
def __init(self):
_QueryData = collections.namedtuple("_QueryData",
['columns', 'where_clause'])
query_structures = {
self.NETWORK_ADAPTER_MODEL:
_QueryData(
['Name', 'Speed', 'MACAddress', 'Index'],
"WHERE NetEnabled = TRUE"
if self.discard_disabled_interfaces else "",
),
self.NETWORK_ADAPTER_CONFIGURATION_MODEL:
_QueryData(
['IPAddress', 'IPSubnet', 'MTU', 'Index', 'DefaultIPGateway'],
"")
}
self._ext_coms = {}
for query_structure in query_structures.iteritems():
cmd = WmiUtils.get_wmic_cmd(
username=self.username,
password=self.password,
target_ip=self.device.target_ip,
columns=query_structure[1].columns,
table=query_structure[0],
where_clause=query_structure[1].where_clause)
self.log("starting WMI scan with command: {}".format(cmd))
ext_com = ExtCom(
self.log, cmd,
shell=False) # shell=False since args must not be parsed again
ext_com.run()
self._ext_coms[query_structure[0]] = ext_com
self.start_result = ResultNode(ok="started base_scan")
def start(self):
cmd = WmiUtils.get_wmic_cmd(
username=self.username,
password=self.password,
target_ip=self.target_ip,
columns=["LogfileName"],
table="Win32_NTEventlogFile",
)
self.logfile_com = ExtCom(
self.log, cmd, debug=True,
shell=False) # shell=False since args must not be parsed again
self.logfile_com.run()
def _parse_wmi_logfile_output(cls, stdout_out, log, **kwargs):
parsed = WmiUtils.parse_wmic_output(stdout_out, **kwargs)
# make sure that all record numbers are integer
output = []
for entry in parsed:
rec_num = entry.get("RecordNumber")
if rec_num is not None:
try:
entry["RecordNumber"] = int(rec_num)
except ValueError:
if log is not None:
log("Failed to parse RecordNumber {} of entry {}".
format(rec_num, unicode(entry)))
else:
output.append(entry)
return output
def periodic_check(self):
do_continue = True
if self.logfile_com.finished() is not None:
do_continue = False
stdout_out, stderr_out = self.logfile_com.communicate()
if stderr_out:
self._handle_stderr(
stderr_out,
"scanning for wmi log files for {}".format(self))
if self.logfile_com.finished() != 0:
raise RuntimeError(
"Scanning for wmi log files failed with code {}".format(
self.logfile_com.finished()))
else:
parsed = WmiUtils.parse_wmic_output(stdout_out)
logfiles = []
for entry in parsed:
if 'LogfileName' not in entry:
self.log(
"Invalid entry in log file scanning: {}".format(
entry), logging_tools.LOG_LEVEL_WARN)
else:
logfiles.append(entry['LogfileName'])
# logfiles = ["Application"]
self.log("Detected {} wmi logfiles for {}".format(
len(logfiles), self.target_device))
self.db.wmi_logfile.update_one(
filter={'device_pk': self.target_device.pk},
update={
'$set': {
'device_pk': self.target_device.pk,
'date': django.utils.timezone.now(),
'logfiles': logfiles,
}
},
upsert=True)
return do_continue
def __call__(self, job):
do_continue = True
com_finished = self.retrieve_ext_com is not None and self.retrieve_ext_com.finished(
) is not None
is_initial = self.retrieve_ext_com is None
if com_finished:
# handle output
stdout_out, stderr_out = self.retrieve_ext_com.communicate()
if stderr_out:
job._handle_stderr(stderr_out,
"RetrieveEvents for {}".format(job))
if self.retrieve_ext_com.finished() != 0:
raise RuntimeError(
"RetrieveEvents wmi command failed with code {}".
format(self.retrieve_ext_com.finished()))
#print 'code', self.retrieve_ext_com.finished()
#_, tmpfilename = tempfile.mkstemp()
#f = open(tmpfilename, 'w')
#f.write(stdout_out)
#f.flush()
#print 'stdout len', len(stdout_out), tmpfilename
#print ' stderr'
#import pprint
#pprint.pprint(stderr_out)
parsed = job._parse_wmi_logfile_output(stdout_out,
try_handle_lists=True,
log=job.log)
# print 'len', len(parsed)
# `parsed` may be empty for RecordNumber-holes
job.log("Found {} log entries between {} and {} for {}".format(
len(parsed),
self.from_record_number,
self.get_next_upper_limit(self.from_record_number),
job,
))
maximal_record_number = self.get_next_upper_limit(
self.from_record_number)
db_entries = []
date_now = django.utils.timezone.now()
for log_entry in parsed:
record_number = log_entry.get('RecordNumber')
if record_number is None:
job.log(
"Warning: WMI log entry without record number, ignoring:",
logging_tools.LOG_LEVEL_WARN)
job.log("{}".format(log_entry))
time_generated = log_entry.get('TimeGenerated')
if time_generated is not None:
try:
time_generated = self.parse_datetime(
time_generated)
except ValueError as e:
job.log(
"Failed to parse datetime from time generated {}: {}"
.format(time_generated, e))
job.log(traceback.format_exc())
time_generated = None
if time_generated is None:
job.log(
"Warning: WMI log entry without TimeGenerated, ignoring:",
logging_tools.LOG_LEVEL_WARN)
job.log("{}".format(log_entry))
else:
db_entries.append({
'time_generated': time_generated,
'logfile_name': job.logfile_name,
'entry': log_entry,
'record_number': record_number,
'device_pk': job.target_device.pk,
})
# DEBUG check
for entry in db_entries:
if list(
job.db.wmi_event_log.find({
'logfile_name':
entry['logfile_name'],
'record_number':
entry["record_number"],
'device_pk':
entry["device_pk"],
})):
raise RuntimeError("DUPLICATE WMI for " +
unicode(entry))
if db_entries:
# must not feed empty list to mongo
job.db.wmi_event_log.insert_many(db_entries)
job.db.wmi_logfile_maximal_record_number.update_one(
filter={
'device_pk': job.target_device.pk,
'logfile_name': job.logfile_name,
},
update={
'$set': {
'device_pk': job.target_device.pk,
'date': date_now,
'maximal_record_number':
maximal_record_number, # this entry may not actually be present
}
},
upsert=True,
)
self.from_record_number = maximal_record_number
job.log(
"New maximal record number: {} (maximum to reach: {}) for {}"
.format(maximal_record_number, self.to_record_number, job))
self.retrieve_ext_com = None
if com_finished or is_initial:
# check whether to start next run
if self.from_record_number >= self.to_record_number:
do_continue = False
job.log("Reached maximal record number {} (by {}).".format(
self.to_record_number, self.from_record_number))
else:
# start next run
cmd = WmiUtils.get_wmic_cmd(
username=job.username,
password=job.password,
target_ip=job.target_ip,
columns=[
"RecordNumber, Message, Category, CategoryString, ComputerName, EventCode, "
+
"EventIdentifier, InsertionStrings, SourceName, TimeGenerated, TimeWritten, "
+ "EventType, Type, User"
],
table="Win32_NTLogEvent",
where_clause=
"WHERE Logfile = '{}' AND RecordNumber > {} and RecordNumber <= {}"
.format(
job.logfile_name,
self.from_record_number,
self.get_next_upper_limit(self.from_record_number),
))
job.log("querying entries from {} to {} for {}".format(
self.from_record_number,
self.get_next_upper_limit(self.from_record_number),
job,
))
self.retrieve_ext_com = ExtCom(
job.log, cmd, debug=True, shell=False
) # shell=False since args must not be parsed again
self.retrieve_ext_com.run()
return do_continue
def check_ext_com(self):
if all(ext_com.finished() is not None
for ext_com in self._ext_coms.itervalues()):
outputs = {
ext_com_key: ext_com.communicate()
for ext_com_key, ext_com in self._ext_coms.iteritems()
}
any_err = False
for ext_com_key, ext_com in self._ext_coms.iteritems():
if ext_com.result != 0:
any_err = True
self.log("Error querying {}, output:".format(ext_com_key),
logging_tools.LOG_LEVEL_ERROR)
self.log("Stdout: {}".format(outputs[ext_com_key][0]),
logging_tools.LOG_LEVEL_ERROR)
self.log("Stderr: {}".format(outputs[ext_com_key][1]),
logging_tools.LOG_LEVEL_ERROR)
if outputs[ext_com_key][1]:
self.log(
"Query for {} wrote to stderr: {}".format(
ext_com_key, outputs[ext_com_key][1]),
logging_tools.LOG_LEVEL_WARN)
if not any_err:
network_adapter_data = WmiUtils.parse_wmic_output(
outputs[self.NETWORK_ADAPTER_MODEL][0])
network_adapter_configuration_data = WmiUtils.parse_wmic_output(
outputs[self.NETWORK_ADAPTER_CONFIGURATION_MODEL][0])
ND_SPEED_LUT = netdevice_speed.build_lut()
updated_nds, created_nds, created_ips, existing_ips = [], [], [], []
# iterate by adapter since only adapters are filtered
for adapter in network_adapter_data:
adapter_index = int(adapter['Index'])
adapter_name = adapter['Name']
# corresponding adapter and adapter_configuration have same index according to some sources
# http://blogs.technet.com/b/heyscriptingguy/archive/2011/10/07/use-powershell-to-identify-your-real-network-adapter.aspx
# http://blogs.technet.com/b/heyscriptingguy/archive/2005/06/14/how-can-i-associate-a-network-connection-with-an-ip-address.aspx
adapter_configuration = next(
c for c in network_adapter_configuration_data
if int(c['Index']) == adapter_index)
device_netdevices = netdevice.objects.filter(
device=self.device)
# find existing dev by idx or else by name
present_nds = [
nd for nd in device_netdevices
if nd.wmi_interface_index == adapter_index
]
if not present_nds:
present_nds = [
nd for nd in device_netdevices
if nd.devname == adapter_name
]
if len(present_nds) > 1:
self.log(
"Error: Found multiple netdevices matching specification:"
+ "Index: {}; Name: {}; Net devices: {}".format(
adapter['Index'], adapter['Name'], present_nds)
)
else:
if present_nds: # only one
nd = present_nds[0]
updated_nds.append(nd)
else:
nd = netdevice(
device=self.device,
wmi_interface_index=adapter_index,
force_network_device_type_match=False,
)
created_nds.append(nd)
nd.devname = adapter_name
nd.macaddr = adapter[
'MACAddress'] or "" # must not be None
nd.mtu = adapter_configuration['MTU']
nd.speed = int(adapter['Speed'])
nd.netdevice_speed = ND_SPEED_LUT.get(
int(adapter['Speed']), ND_SPEED_LUT.get(0))
nd.save()
for ip_found in WmiUtils.WmiList.handle(
adapter_configuration['IPAddress']):
try:
ip_found_struct = ipvx_tools.ipv4(ip_found)
except ValueError:
self.log(
"Found IP which is not supported: {}".
format(ip_found),
logging_tools.LOG_LEVEL_WARN)
else:
# find ipv4 subnet
netmasks_found = []
for _nm in WmiUtils.WmiList.handle(
adapter_configuration["IPSubnet"]):
try:
netmasks_found.append(
ipvx_tools.ipv4(_nm))
except ValueError:
pass
if not netmasks_found:
self.log(
"No netmask found among: {}".format(
adapter['IPSubnet']))
else:
netmask_found_struct = netmasks_found[0]
_gws = []
for _gw in WmiUtils.WmiList.handle(
adapter_configuration[
"DefaultIPGateway"]):
try:
_gws.append(ipvx_tools.ipv4(_gw))
except ValueError:
pass
gw_found_struct = _gws[0] if _gws else None
cur_nw = network.objects.get_or_create_network(
network_addr=ip_found_struct
& netmask_found_struct,
netmask=netmask_found_struct,
gateway=gw_found_struct,
context="WMI",
)
try:
nip = net_ip.objects.get(netdevice=nd,
ip=ip_found)
existing_ips.append(nip)
except net_ip.DoesNotExist:
try:
nip = net_ip(
netdevice=nd,
ip=ip_found,
network=cur_nw,
)
nip.save()
created_ips.append(nip)
except ValidationError as e:
self.log(
"Failed to create ip {} for netdevice {}: {}"
.format(ip_found, nd, e),
logging_tools.LOG_LEVEL_ERROR)
self.log(traceback.format_exc(e))
self.log("Created {}, updated {}, created {}, found {}".format(
logging_tools.get_plural("net device", len(created_ips)),
logging_tools.get_plural("net device", len(updated_nds)),
logging_tools.get_plural("ip", len(created_ips)),
logging_tools.get_plural("existing ip", len(existing_ips)),
))
self.finish()