def __call__(self, job): where_clause = "WHERE Logfile = '{}'".format(job.logfile_name) if job.last_known_record_number is not None: where_clause += "AND RecordNumber >= {}".format( job.last_known_record_number) cmd = WmiUtils.get_wmic_cmd( username=job.username, password=job.password, target_ip=job.target_ip, columns=["RecordNumber"], table="Win32_NTLogEvent", where_clause=where_clause, ) job.log( "Querying maximal entry for {} with last known record number {}" .format(job, job.last_known_record_number)) job.ext_com = ExtCom( job.log, cmd, debug=True, shell=False) # shell=False since args must not be parsed again job.ext_com.run() job.current_phase = WmiLogEntryJob.FindMaximumPhase() return True
def __init(self): _QueryData = collections.namedtuple("_QueryData", ['columns', 'where_clause']) query_structures = { self.NETWORK_ADAPTER_MODEL: _QueryData( ['Name', 'Speed', 'MACAddress', 'Index'], "WHERE NetEnabled = TRUE" if self.discard_disabled_interfaces else "", ), self.NETWORK_ADAPTER_CONFIGURATION_MODEL: _QueryData( ['IPAddress', 'IPSubnet', 'MTU', 'Index', 'DefaultIPGateway'], "") } self._ext_coms = {} for query_structure in query_structures.iteritems(): cmd = WmiUtils.get_wmic_cmd( username=self.username, password=self.password, target_ip=self.device.target_ip, columns=query_structure[1].columns, table=query_structure[0], where_clause=query_structure[1].where_clause) self.log("starting WMI scan with command: {}".format(cmd)) ext_com = ExtCom( self.log, cmd, shell=False) # shell=False since args must not be parsed again ext_com.run() self._ext_coms[query_structure[0]] = ext_com self.start_result = ResultNode(ok="started base_scan")
def start(self): cmd = WmiUtils.get_wmic_cmd( username=self.username, password=self.password, target_ip=self.target_ip, columns=["LogfileName"], table="Win32_NTEventlogFile", ) self.logfile_com = ExtCom( self.log, cmd, debug=True, shell=False) # shell=False since args must not be parsed again self.logfile_com.run()
def _parse_wmi_logfile_output(cls, stdout_out, log, **kwargs): parsed = WmiUtils.parse_wmic_output(stdout_out, **kwargs) # make sure that all record numbers are integer output = [] for entry in parsed: rec_num = entry.get("RecordNumber") if rec_num is not None: try: entry["RecordNumber"] = int(rec_num) except ValueError: if log is not None: log("Failed to parse RecordNumber {} of entry {}". format(rec_num, unicode(entry))) else: output.append(entry) return output
def periodic_check(self): do_continue = True if self.logfile_com.finished() is not None: do_continue = False stdout_out, stderr_out = self.logfile_com.communicate() if stderr_out: self._handle_stderr( stderr_out, "scanning for wmi log files for {}".format(self)) if self.logfile_com.finished() != 0: raise RuntimeError( "Scanning for wmi log files failed with code {}".format( self.logfile_com.finished())) else: parsed = WmiUtils.parse_wmic_output(stdout_out) logfiles = [] for entry in parsed: if 'LogfileName' not in entry: self.log( "Invalid entry in log file scanning: {}".format( entry), logging_tools.LOG_LEVEL_WARN) else: logfiles.append(entry['LogfileName']) # logfiles = ["Application"] self.log("Detected {} wmi logfiles for {}".format( len(logfiles), self.target_device)) self.db.wmi_logfile.update_one( filter={'device_pk': self.target_device.pk}, update={ '$set': { 'device_pk': self.target_device.pk, 'date': django.utils.timezone.now(), 'logfiles': logfiles, } }, upsert=True) return do_continue
def __call__(self, job): do_continue = True com_finished = self.retrieve_ext_com is not None and self.retrieve_ext_com.finished( ) is not None is_initial = self.retrieve_ext_com is None if com_finished: # handle output stdout_out, stderr_out = self.retrieve_ext_com.communicate() if stderr_out: job._handle_stderr(stderr_out, "RetrieveEvents for {}".format(job)) if self.retrieve_ext_com.finished() != 0: raise RuntimeError( "RetrieveEvents wmi command failed with code {}". format(self.retrieve_ext_com.finished())) #print 'code', self.retrieve_ext_com.finished() #_, tmpfilename = tempfile.mkstemp() #f = open(tmpfilename, 'w') #f.write(stdout_out) #f.flush() #print 'stdout len', len(stdout_out), tmpfilename #print ' stderr' #import pprint #pprint.pprint(stderr_out) parsed = job._parse_wmi_logfile_output(stdout_out, try_handle_lists=True, log=job.log) # print 'len', len(parsed) # `parsed` may be empty for RecordNumber-holes job.log("Found {} log entries between {} and {} for {}".format( len(parsed), self.from_record_number, self.get_next_upper_limit(self.from_record_number), job, )) maximal_record_number = self.get_next_upper_limit( self.from_record_number) db_entries = [] date_now = django.utils.timezone.now() for log_entry in parsed: record_number = log_entry.get('RecordNumber') if record_number is None: job.log( "Warning: WMI log entry without record number, ignoring:", logging_tools.LOG_LEVEL_WARN) job.log("{}".format(log_entry)) time_generated = log_entry.get('TimeGenerated') if time_generated is not None: try: time_generated = self.parse_datetime( time_generated) except ValueError as e: job.log( "Failed to parse datetime from time generated {}: {}" .format(time_generated, e)) job.log(traceback.format_exc()) time_generated = None if time_generated is None: job.log( "Warning: WMI log entry without TimeGenerated, ignoring:", logging_tools.LOG_LEVEL_WARN) job.log("{}".format(log_entry)) else: db_entries.append({ 'time_generated': time_generated, 'logfile_name': job.logfile_name, 'entry': log_entry, 'record_number': record_number, 'device_pk': job.target_device.pk, }) # DEBUG check for entry in db_entries: if list( job.db.wmi_event_log.find({ 'logfile_name': entry['logfile_name'], 'record_number': entry["record_number"], 'device_pk': entry["device_pk"], })): raise RuntimeError("DUPLICATE WMI for " + unicode(entry)) if db_entries: # must not feed empty list to mongo job.db.wmi_event_log.insert_many(db_entries) job.db.wmi_logfile_maximal_record_number.update_one( filter={ 'device_pk': job.target_device.pk, 'logfile_name': job.logfile_name, }, update={ '$set': { 'device_pk': job.target_device.pk, 'date': date_now, 'maximal_record_number': maximal_record_number, # this entry may not actually be present } }, upsert=True, ) self.from_record_number = maximal_record_number job.log( "New maximal record number: {} (maximum to reach: {}) for {}" .format(maximal_record_number, self.to_record_number, job)) self.retrieve_ext_com = None if com_finished or is_initial: # check whether to start next run if self.from_record_number >= self.to_record_number: do_continue = False job.log("Reached maximal record number {} (by {}).".format( self.to_record_number, self.from_record_number)) else: # start next run cmd = WmiUtils.get_wmic_cmd( username=job.username, password=job.password, target_ip=job.target_ip, columns=[ "RecordNumber, Message, Category, CategoryString, ComputerName, EventCode, " + "EventIdentifier, InsertionStrings, SourceName, TimeGenerated, TimeWritten, " + "EventType, Type, User" ], table="Win32_NTLogEvent", where_clause= "WHERE Logfile = '{}' AND RecordNumber > {} and RecordNumber <= {}" .format( job.logfile_name, self.from_record_number, self.get_next_upper_limit(self.from_record_number), )) job.log("querying entries from {} to {} for {}".format( self.from_record_number, self.get_next_upper_limit(self.from_record_number), job, )) self.retrieve_ext_com = ExtCom( job.log, cmd, debug=True, shell=False ) # shell=False since args must not be parsed again self.retrieve_ext_com.run() return do_continue
def check_ext_com(self): if all(ext_com.finished() is not None for ext_com in self._ext_coms.itervalues()): outputs = { ext_com_key: ext_com.communicate() for ext_com_key, ext_com in self._ext_coms.iteritems() } any_err = False for ext_com_key, ext_com in self._ext_coms.iteritems(): if ext_com.result != 0: any_err = True self.log("Error querying {}, output:".format(ext_com_key), logging_tools.LOG_LEVEL_ERROR) self.log("Stdout: {}".format(outputs[ext_com_key][0]), logging_tools.LOG_LEVEL_ERROR) self.log("Stderr: {}".format(outputs[ext_com_key][1]), logging_tools.LOG_LEVEL_ERROR) if outputs[ext_com_key][1]: self.log( "Query for {} wrote to stderr: {}".format( ext_com_key, outputs[ext_com_key][1]), logging_tools.LOG_LEVEL_WARN) if not any_err: network_adapter_data = WmiUtils.parse_wmic_output( outputs[self.NETWORK_ADAPTER_MODEL][0]) network_adapter_configuration_data = WmiUtils.parse_wmic_output( outputs[self.NETWORK_ADAPTER_CONFIGURATION_MODEL][0]) ND_SPEED_LUT = netdevice_speed.build_lut() updated_nds, created_nds, created_ips, existing_ips = [], [], [], [] # iterate by adapter since only adapters are filtered for adapter in network_adapter_data: adapter_index = int(adapter['Index']) adapter_name = adapter['Name'] # corresponding adapter and adapter_configuration have same index according to some sources # http://blogs.technet.com/b/heyscriptingguy/archive/2011/10/07/use-powershell-to-identify-your-real-network-adapter.aspx # http://blogs.technet.com/b/heyscriptingguy/archive/2005/06/14/how-can-i-associate-a-network-connection-with-an-ip-address.aspx adapter_configuration = next( c for c in network_adapter_configuration_data if int(c['Index']) == adapter_index) device_netdevices = netdevice.objects.filter( device=self.device) # find existing dev by idx or else by name present_nds = [ nd for nd in device_netdevices if nd.wmi_interface_index == adapter_index ] if not present_nds: present_nds = [ nd for nd in device_netdevices if nd.devname == adapter_name ] if len(present_nds) > 1: self.log( "Error: Found multiple netdevices matching specification:" + "Index: {}; Name: {}; Net devices: {}".format( adapter['Index'], adapter['Name'], present_nds) ) else: if present_nds: # only one nd = present_nds[0] updated_nds.append(nd) else: nd = netdevice( device=self.device, wmi_interface_index=adapter_index, force_network_device_type_match=False, ) created_nds.append(nd) nd.devname = adapter_name nd.macaddr = adapter[ 'MACAddress'] or "" # must not be None nd.mtu = adapter_configuration['MTU'] nd.speed = int(adapter['Speed']) nd.netdevice_speed = ND_SPEED_LUT.get( int(adapter['Speed']), ND_SPEED_LUT.get(0)) nd.save() for ip_found in WmiUtils.WmiList.handle( adapter_configuration['IPAddress']): try: ip_found_struct = ipvx_tools.ipv4(ip_found) except ValueError: self.log( "Found IP which is not supported: {}". format(ip_found), logging_tools.LOG_LEVEL_WARN) else: # find ipv4 subnet netmasks_found = [] for _nm in WmiUtils.WmiList.handle( adapter_configuration["IPSubnet"]): try: netmasks_found.append( ipvx_tools.ipv4(_nm)) except ValueError: pass if not netmasks_found: self.log( "No netmask found among: {}".format( adapter['IPSubnet'])) else: netmask_found_struct = netmasks_found[0] _gws = [] for _gw in WmiUtils.WmiList.handle( adapter_configuration[ "DefaultIPGateway"]): try: _gws.append(ipvx_tools.ipv4(_gw)) except ValueError: pass gw_found_struct = _gws[0] if _gws else None cur_nw = network.objects.get_or_create_network( network_addr=ip_found_struct & netmask_found_struct, netmask=netmask_found_struct, gateway=gw_found_struct, context="WMI", ) try: nip = net_ip.objects.get(netdevice=nd, ip=ip_found) existing_ips.append(nip) except net_ip.DoesNotExist: try: nip = net_ip( netdevice=nd, ip=ip_found, network=cur_nw, ) nip.save() created_ips.append(nip) except ValidationError as e: self.log( "Failed to create ip {} for netdevice {}: {}" .format(ip_found, nd, e), logging_tools.LOG_LEVEL_ERROR) self.log(traceback.format_exc(e)) self.log("Created {}, updated {}, created {}, found {}".format( logging_tools.get_plural("net device", len(created_ips)), logging_tools.get_plural("net device", len(updated_nds)), logging_tools.get_plural("ip", len(created_ips)), logging_tools.get_plural("existing ip", len(existing_ips)), )) self.finish()