def run_scans_and_notify(self): # Check sites are in the config if len(self.wp_sites) == 0: log.error( "No sites configured, please provide wp_sites in config file or use arguments --url URL [URL...] or --urls File path" ) return ((-1, [])) new_reports = self.run_scans_wrapper(self.wp_sites) # Print results and finish self.print_scanned_sites_results(new_reports) # Second scans if needed if len(self.scanner.prescanned_sites_warn) > 0: new_reports += self.re_run_scans( self.scanner.prescanned_sites_warn) self.print_scanned_sites_results(new_reports) if not any([r['status'] == 'ERROR' for r in new_reports if r]): log.info("Scans finished successfully.") return ((0, new_reports)) else: log.info("Scans finished with errors.") return ((-1, new_reports))
def __init__(self, files=None, string=None): self.files = files # Init config parser self.parser = configparser.ConfigParser() # Load default configuration self.parser.read_dict({'wpwatcher': self.DEFAULT_CONFIG}) if string: self.parser.read_string(string) if (not self.files or len(self.files) == 0) and not string: self.files = self.find_config_files() if self.files: read_files = self.parser.read(self.files) if len(read_files) < len(self.files): log.error( "Could not read config " + str(list(set(self.files) - set(read_files))) + ". Make sure the file exists, the format is OK and you have correct access right." ) exit(-1) # No config file notice else: log.info( "No config file loaded and could not find default config `~/.wpwatcher/wpwatcher.conf`, `~/wpwatcher.conf` or `./wpwatcher.conf`" )
def _wpscan(self, *args): # WPScan arguments cmd = self.wpscan_executable + list(args) # Log wpscan command without api token log.debug("Running WPScan command: %s" % ' '.join(safe_log_wpscan_args(cmd))) # Run wpscan process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Append process to current process list and launch self.processes.append(process) wpscan_output, stderr = process.communicate() self.processes.remove(process) try: wpscan_output = wpscan_output.decode("utf-8") except UnicodeDecodeError: wpscan_output = wpscan_output.decode("latin1") # Error when wpscan failed, except exit code 5: means the target has at least one vulnerability. # See https://github.com/wpscanteam/CMSScanner/blob/master/lib/cms_scanner/exit_code.rb if process.returncode in [0, 5]: # WPScan comamnd success log.debug("WPScan raw output:\n" + wpscan_output) # Log error else: err_string, full_err_string = self.get_full_err_string( cmd, process.returncode, wpscan_output, stderr) log.error(err_string) log.debug(full_err_string) return ((process.returncode, wpscan_output))
def interrupt(self, sig=None, frame=None) -> None: # type: ignore [no-untyped-def] """Interrupt the program and exit. """ log.error("Interrupting...") # If called inside ThreadPoolExecutor, raise Exeception if not isinstance( threading.current_thread(), threading._MainThread): # type: ignore [attr-defined] raise InterruptedError() self.interrupt_scans() # Give a 5 seconds timeout to buggy WPScan jobs to finish or ignore them try: timeout(5, self._executor.shutdown, kwargs=dict(wait=True)) except TimeoutError: pass # Display results log.info(repr(self.new_reports)) log.info("Scans interrupted.") # and quit sys.exit(-1)
def run_scans_and_notify(self): # Check sites are in the config if len(self.conf['wp_sites'])==0: log.error("No sites configured, please provide wp_sites in config file or use arguments --url URL [URL...] or --urls File path") return((-1, self.wp_reports)) log.info("Starting scans on %s configured sites"%(len(self.conf['wp_sites']))) new_reports=[] self.executor=concurrent.futures.ThreadPoolExecutor(max_workers=self.conf['asynch_workers']) # Sumbit all scans jobs and start scanning for s in self.conf['wp_sites']: self.futures.append(self.executor.submit(self.scan_site, s)) # Loops while scans are running and read results for f in self.futures: try: new_reports.append(f.result()) # Handle interruption from inside threads when using --ff except (InterruptedError): self.wait_and_finish_interrupt() # Print results and finish self.print_scanned_sites_results() if not any ([r['status']=='ERROR' for r in new_reports if r]): log.info("Scans finished successfully.") return((0, self.wp_reports)) else: log.info("Scans finished with errors.") return((-1, self.wp_reports))
def update_wpscan(self): # Update wpscan database log.info("Updating WPScan") exit_code, _ = self._wpscan("--update") if exit_code != 0: log.error("Error updating WPScan") exit(-1)
def _notify(self, wp_site, wp_report, last_wp_report): # Send the report to if len(self.email_errors_to) > 0 and wp_report['status'] == 'ERROR': to = ','.join(self.email_errors_to) else: to = ','.join(wp_site['email_to'] + self.email_to) if not to: log.info( "Not sending WPWatcher %s email report because no email is configured for site %s" % (wp_report['status'], wp_report['site'])) return while mail_lock.locked(): time.sleep(0.01) try: with mail_lock: self.send_report(wp_report, to) return True # Handle send mail error except (smtplib.SMTPException, ConnectionRefusedError, TimeoutError): log.error("Unable to send mail report for site " + wp_site['url'] + "\n" + traceback.format_exc()) wp_report['errors'].append("Unable to send mail report" + "\n" + traceback.format_exc()) raise RuntimeError("Unable to send mail report")
def interrupt(self, sig=None, frame=None): '''Interrupt sequence''' log.error("Interrupting...") # If called inside ThreadPoolExecutor, raise Exeception if not isinstance(threading.current_thread(), threading._MainThread): raise InterruptedError() # Cancel all scans self.cancel_pending_futures() # future scans # Wait all scans finished self.scanner.cancel_scans() # running scans # Give a 5 seconds timeout to buggy WPScan jobs to finish or ignore them try: timeout(5, self.executor.shutdown, kwargs=dict(wait=True)) except TimeoutError: pass # Recover reports from futures results new_reports = [] for f in self.futures: if f.done(): try: new_reports.append(f.result()) except Exception: pass # Display results and quit self.print_scanned_sites_results(new_reports) log.info("Scans interrupted.") sys.exit(-1)
def run_scans(self) -> Tuple[int, ReportCollection]: """ Run WPScan on defined websites and send notifications. :Returns: `tuple (exit code, reports)` """ # Check sites are in the config if len(self.wp_sites) == 0: log.error( "No sites configured, please provide wp_sites in config file or use arguments --url URL [URL...] or --urls File path" ) return (-1, self.new_reports) self.wp_reports.open() try: self._run_scans(self.wp_sites) # Handle interruption from inside threads when using --ff except InterruptedError: self.interrupt() finally: self.wp_reports.close() # Print results and finish log.info(repr(self.new_reports)) if not any([r["status"] == "ERROR" for r in self.new_reports if r]): log.info("Scans finished successfully.") return (0, self.new_reports) else: log.info("Scans finished with errors.") return (-1, self.new_reports)
def __init__(self, files=None, string=None): self.files = files if files else [] # Init config parser self.parser = configparser.ConfigParser() # Load default configuration self.parser.read_dict({'wpwatcher': self.DEFAULT_CONFIG}) if string: self.parser.read_string(string) elif not self.files or len(self.files) == 0: self.files = self.find_config_files() if not self.files: log.info( "Could not find default config: `~/.wpwatcher/wpwatcher.conf`, `~/wpwatcher.conf` or `./wpwatcher.conf`" ) if self.files: for f in self.files: try: with open(f, 'r') as fp: self.parser.read_file(fp) except (OSError): log.error( "Could not read config %s. Make sure the file exists and you have correct access right." % (f)) raise # No config file notice else: log.info("No config file loaded")
def fail(self, reason: str) -> None: """ Mark the scan as failed. """ log.error(reason) if self["error"]: self["error"] += "\n\n" self["error"] += reason
def getbool(conf, key): try: return conf.getboolean('wpwatcher', key) except Exception as err: log.error( "Could not read boolean value in config for: '{}' and string '{}'. Must be Yes/No. Error: {}" .format(key, conf.get('wpwatcher', key), str(err))) raise
def __init__(self, conf): # Create (lazy) wpscan link self.wpscan = WPScanWrapper(conf['wpscan_path']) # Init mail link self.mail = WPWatcherNotification(conf) # Storing the Event object to wait and cancel the waiting self.api_wait = threading.Event() # Toogle if aborting so other errors doesnt get triggerred and exit faster self.interrupting = False # List of urls scanend self.scanned_sites = [] # Save required config options self.api_limit_wait = conf['api_limit_wait'] self.follow_redirect = conf['follow_redirect'] self.wpscan_output_folder = conf['wpscan_output_folder'] self.wpscan_args = conf['wpscan_args'] self.fail_fast = conf['fail_fast'] self.false_positive_strings = conf['false_positive_strings'] self.daemon = conf['daemon'] self.daemon_loop_sleep = conf['daemon_loop_sleep'] self.prescan_without_api_token = conf['prescan_without_api_token'] # Scan timeout self.scan_timeout = conf['scan_timeout'] # Setup prescan options self.prescanned_sites_warn = [] self.api_token = None if self.prescan_without_api_token: log.info("Prescan without API token...") if not self.check_api_token_not_installed(): exit(-1) self.api_token = self.retreive_api_token(self.wpscan_args) if not self.api_token: log.error( "No --api-token in WPScan arguments, please set --api-token in config file wpscan_args values or use --wpargs [...] to allow WPWatcher to handle WPScan API token" ) exit(-1) api_token_index = self.wpscan_args.index("--api-token") + 1 del self.wpscan_args[api_token_index] del self.wpscan_args[api_token_index - 1] # Init wpscan output folder if conf['wpscan_output_folder']: os.makedirs(conf['wpscan_output_folder'], exist_ok=True) os.makedirs(os.path.join(conf['wpscan_output_folder'], 'error/'), exist_ok=True) os.makedirs(os.path.join(conf['wpscan_output_folder'], 'alert/'), exist_ok=True) os.makedirs(os.path.join(conf['wpscan_output_folder'], 'warning/'), exist_ok=True) os.makedirs(os.path.join(conf['wpscan_output_folder'], 'info/'), exist_ok=True)
def getjson(conf, key): string_val = conf.get('wpwatcher', key) try: loaded = json.loads(string_val) return loaded if loaded else [] except Exception as err: log.error( "Could not read config JSON value for: '%s' and string: '%s'. Error: %s" % (key, conf.get('wpwatcher', key), str(err))) raise
def __init__(self, wp_reports_filepath="", daemon=False): self.no_local_storage=wp_reports_filepath=='null' if not wp_reports_filepath : wp_reports_filepath=self.find_wp_reports_file(create=True,daemon=daemon) self.filepath=wp_reports_filepath self._data=self.build_wp_reports(self.filepath) try: self.update_and_write_wp_reports(self._data) except: log.error("Could not write wp_reports database: {}. Use '--reports null' to ignore local Json database".format(self.filepath)) raise
def format_site(self, wp_site): if 'url' not in wp_site : log.error("Invalid site %s"%wp_site) wp_site={'url':''} else: # Format sites with scheme indication p_url=list(urlparse(wp_site['url'])) if p_url[0]=="": wp_site['url']='http://'+wp_site['url'] # Read the wp_site dict and assing default values if needed if 'email_to' not in wp_site or wp_site['email_to'] is None: wp_site['email_to']=[] if 'false_positive_strings' not in wp_site or wp_site['false_positive_strings'] is None: wp_site['false_positive_strings']=[] if 'wpscan_args' not in wp_site or wp_site['wpscan_args'] is None: wp_site['wpscan_args']=[] return wp_site
def cancel_scans(self): self.interrupting = True # Send ^C to all WPScan not finished for p in self.wpscan.processes: p.send_signal(signal.SIGINT) # Wait for all processes to finish , kill after timeout try: timeout(INTERRUPT_TIMEOUT, self.wait_all_wpscan_process) except TimeoutError: log.error("Interrupt timeout reached, killing WPScan processes") for p in self.wpscan.processes: p.kill() # Unlock api wait self.api_wait.set()
def __init__(self, conf): # Copy config dict as is. Copy not to edit initial dict self.conf=copy.deepcopy(conf) # (Re)init logger with config init_log(verbose=self.conf['verbose'], quiet=self.conf['quiet'], logfile=self.conf['log_file']) # Dump config log.info("WordPress sites and configuration:{}".format(self.dump_config())) # Create wpscan link self.wpscan=WPScanWrapper(path=self.conf['wpscan_path']) # Check if WPScan exists if not self.wpscan.is_wpscan_installed(): log.error("There is an issue with your WPScan installation or WPScan not installed. Make sure wpscan in you PATH or configure full path to executable in config files. If you're using RVM, the path should point to the WPScan wrapper like /usr/local/rvm/gems/ruby-2.6.0/wrappers/wpscan. Fix wpscan on your system. See https://wpscan.org for installation steps.") exit(-1) # Update wpscan database self.wpscan.update_wpscan() # Try delete temp files. if os.path.isdir('/tmp/wpscan'): try: shutil.rmtree('/tmp/wpscan') log.info("Deleted temp WPScan files in /tmp/wpscan/") except (FileNotFoundError, OSError, Exception) : log.info("Could not delete temp WPScan files in /tmp/wpscan/. Error:\n%s"%(traceback.format_exc())) # Read DB self.wp_reports=self.build_wp_reports() # Try if local Json databse is accessible try: self.update_and_write_wp_reports(self.wp_reports) except: log.error("Could not write wp_reports database: {}. Use '--reports null' to ignore local Json database".format(self.conf['wp_reports'])) raise # Init wpscan output folder if self.conf['wpscan_output_folder'] : os.makedirs(self.conf['wpscan_output_folder'], exist_ok=True) # Asynchronous executor, will be created when calling run_scans_and_notify self.executor=None # List of conccurent futures self.futures=[] # List of urls scanend self.scanned_sites=[] # Toogle if aborting so other errors doesnt get triggerred and exit faster self.interrupting=False # Register the signals to be caught ^C , SIGTERM (kill) , service restart , will trigger interrupt() signal.signal(signal.SIGINT, self.interrupt) signal.signal(signal.SIGTERM, self.interrupt) # Storing the Event object to wait and cancel the waiting self.api_wait=threading.Event()
def _lazy_init(self): # Check if WPScan exists exit_code, version_info = self._wpscan("--version", "--format", "json") if exit_code != 0: log.error( "There is an issue with your WPScan installation or WPScan not installed. Make sure wpscan in you PATH or configure full path to executable in config files. If you're using RVM, the path should point to the WPScan wrapper like /usr/local/rvm/gems/ruby-2.6.0/wrappers/wpscan. Fix wpscan on your system. See https://wpscan.org for installation steps." ) exit(-1) version_info = json.loads(version_info) if not version_info['last_db_update'] or datetime.now( ) - datetime.strptime(version_info['last_db_update'].split(".")[0], "%Y-%m-%dT%H:%M:%S") > UPDATE_DB_INTERVAL: self.update_wpscan() self.init_check_done = True
def build_wp_reports(self, filepath): wp_reports=[] if self.no_local_storage: return wp_reports if os.path.isfile(filepath): try: with open(filepath, 'r') as reportsfile: wp_reports=json.load(reportsfile) log.info("Load wp_reports database: %s"%filepath) except Exception: log.error("Could not read wp_reports database: {}. Use '--reports null' to ignore local Json database".format(filepath)) raise else: log.info("The database file %s do not exist. It will be created."%(filepath)) return wp_reports
def open(self) -> None: """ Acquire the file lock for the DB file. """ try: self._wp_report_file_lock.acquire(timeout=1) except Timeout as err: raise RuntimeError(f"Could not use the database file '{self.filepath}' because another instance of WPWatcher is using it. ") from err log.debug(f"Acquired DB lock file '{self.filepath}.lock'") try: self.write() except: log.error( f"Could not write wp_reports database: {self.filepath}. Use '--reports null' to ignore local Json database." ) raise
def build_wp_reports(self): wp_reports=[] if self.conf['wp_reports']!='null': if not self.conf['wp_reports']: self.conf['wp_reports']=self.find_wp_reports_file(create=True) if self.conf['wp_reports']: if os.path.isfile(self.conf['wp_reports']): try: with open(self.conf['wp_reports'], 'r') as reportsfile: wp_reports=json.load(reportsfile) log.info("Load wp_reports database: %s"%self.conf['wp_reports']) except Exception: log.error("Could not read wp_reports database: {}. Use '--reports null' to ignore local Json database".format(self.conf['wp_reports'])) raise else: log.info("The database file %s do not exist. It will be created."%(self.conf['wp_reports'])) return wp_reports
def handle_wpscan_err_follow_redirect(self, wp_site, wp_report): url = re.findall( r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', wp_report["wpscan_output"].split( "The URL supplied redirects to")[1]) if len(url) == 1: wp_site['url'] = url[0].strip() log.info("Following redirection to %s" % wp_site['url']) new_report = self.scan_site(wp_site) return ((new_report, new_report != None)) else: err_str = "Could not parse the URL to follow in WPScan output after words 'The URL supplied redirects to'" log.error(err_str) wp_report['errors'].append(err_str) return ((wp_report, False))
def _wpscan_site(self, wp_site, wp_report): # WPScan arguments wpscan_arguments = self.wpscan_args + wp_site['wpscan_args'] + [ '--url', wp_site['url'] ] # Output log.info("Scanning site %s" % wp_site['url']) # Launch WPScan (wpscan_exit_code, wp_report["wpscan_output"]) = self.wpscan.wpscan(*wpscan_arguments) # Exit code 0: all ok. Exit code 5: Vulnerable. Other exit code are considered as errors if wpscan_exit_code in [0, 5]: # Call parse_result from parser.py log.debug("Parsing WPScan output") try: wp_report['infos'], wp_report['warnings'], wp_report[ 'alerts'] = parse_results( wp_report['wpscan_output'], self.false_positive_strings + wp_site['false_positive_strings'] + ['No WPVulnDB API Token given']) wp_report['errors'] = [] # clear errors if any except Exception as err: err_str = "Could not parse WPScan output for site %s\n%s" % ( wp_site['url'], traceback.format_exc()) log.error(err_str) raise RuntimeError(err_str) from err else: return wp_report # Handle scan errors ----- # Quick return if interrupting and/or if user cacelled scans if self.interrupting or wpscan_exit_code in [2, -2, -9]: return None # Other errors codes : -9, -2, 127, etc: # or wpscan_exit_code not in [1,3,4] # If WPScan error, add the error to the reports # This types if errors will be written into the Json database file exit codes 1,3,4 err_str = "WPScan failed with exit code %s. \nWPScan arguments: %s. \nWPScan output: \n%s" % ( (wpscan_exit_code, safe_log_wpscan_args(wpscan_arguments), wp_report['wpscan_output'])) raise RuntimeError(err_str)
def wpscan_site(self, wp_site, wp_report): # Launch WPScan try: wp_report_new = timeout(self.scan_timeout.total_seconds(), self._wpscan_site, args=(wp_site, wp_report)) if wp_report_new: wp_report.update(wp_report_new) else: return None except TimeoutError: wp_report['errors'].append( "Timeout scanning site after %s seconds" % self.scan_timeout.total_seconds()) log.error("Timeout scanning site %s after %s seconds." % (wp_site['url'], self.scan_timeout.total_seconds())) # Terminate self.terminate_scan(wp_site, wp_report) self.check_fail_fast() return wp_report
def wpscan_site(self, wp_site, wp_report): '''Timeout wrapper arround `WPWatcherScanner._wpscan_site()` Launch WPScan. Returns filled wp_report or None ''' try: wp_report_new= timeout(self.scan_timeout.total_seconds(), self._wpscan_site, args=(wp_site, wp_report) ) if wp_report_new: wp_report.update(wp_report_new) else : return None except TimeoutError: wp_report['error']+="Timeout scanning site after %s seconds.\nSetup scan_timeout in config file to allow more time"%self.scan_timeout.total_seconds() log.error("Timeout scanning site %s after %s seconds. Setup scan_timeout in config file to allow more time"%(wp_site['url'], self.scan_timeout.total_seconds())) # Kill process for p in self.wpscan.processes: if ( wp_site['url'] in p.args ) and not p.returncode: log.info('Killing WPScan process %s'%(safe_log_wpscan_args(p.args))) p.kill() self.check_fail_fast() return wp_report
def format_site(wp_site): if 'url' not in wp_site: log.error("Invalid site %s" % wp_site) wp_site = {'url': ''} else: #Strip URL string wp_site['url'] = wp_site['url'].strip() # Format sites with scheme indication p_url = list(urlparse(wp_site['url'])) if p_url[0] == "": wp_site['url'] = 'http://' + wp_site['url'] # Read the wp_site dict and assing default values if needed optionals = ['email_to', 'false_positive_strings', 'wpscan_args'] for op in optionals: if op not in wp_site or wp_site[op] is None: wp_site[op] = [] return wp_site
def _build_db(self, filepath: str) -> ReportCollection: """Load reports database and return the complete structure""" wp_reports = ReportCollection() if self.no_local_storage: return wp_reports if os.path.isfile(filepath): try: with open(filepath, "r") as reportsfile: wp_reports.extend( ScanReport(item) for item in json.load(reportsfile) ) log.info(f"Load wp_reports database: {filepath}") except Exception: log.error( f"Could not read wp_reports database: {filepath}. Use '--reports null' to ignore local Json database" ) raise else: log.info(f"The database file {filepath} do not exist. It will be created.") return wp_reports
def check_api_token_not_installed(): if 'WPSCAN_API_TOKEN' in os.environ: log.error( "WPSCAN_API_TOKEN environnement varible is set, please remove it to allow WPWatcher to handle WPScan API token" ) return False files = ['.wpscan/scan.json', '.wpscan/scan.yml'] env = ['HOME', 'XDG_CONFIG_HOME', 'APPDATA', 'PWD'] for wpscan_config_file in WPWatcherConfig.find_files(env, files): with open(wpscan_config_file, 'r') as wpscancfg: if any([ 'api_token' in line and line.strip()[0] != "#" for line in wpscancfg.readlines() ]): log.error( 'API token is set in the config file %s, please remove it to allow WPWatcher to handle WPScan API token' % (wpscan_config_file)) return False return True
def _wpscan_site(self, wp_site, wp_report): '''Handled WPScan scanning , parsing, errors and reporting. Returns filled wp_report, None if interrupted or killed. Can raise RuntimeError if WPScan failed''' # WPScan arguments wpscan_arguments=self.wpscan_args+wp_site['wpscan_args']+['--url', wp_site['url']] # Output log.info("Scanning site %s"%wp_site['url'] ) # Launch WPScan (wpscan_exit_code, wp_report["wpscan_output"]) = self.wpscan.wpscan(*wpscan_arguments) log.debug("Parsing WPScan output") try: # Call parse_results_from_string from wpscan_out_parse module results = parse_results_from_string(wp_report['wpscan_output'] , self.false_positive_strings + wp_site['false_positive_strings'] + ['No WPVulnDB API Token given'] ) wp_report['infos'], wp_report['warnings'] , wp_report['alerts'], wp_report['summary'] = results['infos'], results['warnings'], results['alerts'], results['summary'] if results['error']: wp_report['error']+=results['error'] except Exception as err: err_str="Could not parse WPScan output for site %s\n%s"%(wp_site['url'],traceback.format_exc()) log.error(err_str) raise RuntimeError(err_str) from err # Exit code 0: all ok. Exit code 5: Vulnerable. Other exit code are considered as errors if wpscan_exit_code in [0,5]: return wp_report # Quick return if interrupting and/or if user cacelled scans if self.interrupting or wpscan_exit_code in [2, -2, -9]: return None # Other errors codes : 127, etc, simply raise error err_str="WPScan failed with exit code %s. \nArguments: %s. \nOutput: \n%s"%((wpscan_exit_code, safe_log_wpscan_args(wpscan_arguments), re.sub(r'(\x1b|\[[0-9][0-9]?m)','', wp_report['wpscan_output']) )) raise RuntimeError(err_str)