def _rebuild_rew_reports(self) -> None:
"Recover reports from futures results"
self.new_reports = ReportCollection()
for f in self._futures:
if f.done():
try:
self.new_reports.append(f.result())
except Exception:
pass
def __init__(self, conf: Config):
"""
Arguments:
- `conf`: the configuration dict. Required
"""
# (Re)init logger with config
_init_log(verbose=conf["verbose"],
quiet=conf["quiet"],
logfile=conf["log_file"])
self._delete_tmp_wpscan_files()
# Init DB interface
self.wp_reports: DataBase = DataBase(filepath=conf["wp_reports"],
daemon=conf['daemon'])
# Update config before passing it to WPWatcherScanner
conf.update({"wp_reports": self.wp_reports.filepath})
# Init scanner
self.scanner: Scanner = Scanner(conf)
# Save sites
conf["wp_sites"] = [Site(site_conf) for site_conf in conf["wp_sites"]]
self.wp_sites: List[Site] = conf["wp_sites"]
# Asynchronous executor
self._executor: concurrent.futures.ThreadPoolExecutor = (
concurrent.futures.ThreadPoolExecutor(
max_workers=conf["asynch_workers"]))
# List of conccurent futures
self._futures: List[concurrent.futures.Future] = [
] # type: ignore [type-arg]
# Register the signals to be caught ^C , SIGTERM (kill) , service restart , will trigger interrupt()
signal.signal(signal.SIGINT, self.interrupt)
signal.signal(signal.SIGTERM, self.interrupt)
self.new_reports = ReportCollection()
"New reports, cleared and filled when running `run_scans`."
self.all_reports = ReportCollection()
"All reports an instance of `WPWatcher` have generated using `run_scans`."
# Dump config
log.debug(f"Configuration:{repr(conf)}")
def __init__(self, filepath: Optional[str] = None, daemon: bool = False):
if not filepath:
filepath = self._find_db_file(daemon=daemon)
self.no_local_storage: bool = filepath == "null"
"True if the DB is disabled"
self.filepath = filepath
self._data = ReportCollection()
self._data.extend(self._build_db(self.filepath))
# Writing into the database file is thread safe
self._wp_report_lock: threading.Lock = threading.Lock()
# Only once instance of WPWatcher can use a database file at a time.
self._wp_report_file_lock: FileLock = FileLock(f"{self.filepath}.lock")
def _build_db(self, filepath: str) -> ReportCollection:
"""Load reports database and return the complete structure"""
wp_reports = ReportCollection()
if self.no_local_storage:
return wp_reports
if os.path.isfile(filepath):
try:
with open(filepath, "r") as reportsfile:
wp_reports.extend(
ScanReport(item) for item in json.load(reportsfile)
)
log.info(f"Load wp_reports database: {filepath}")
except Exception:
log.error(
f"Could not read wp_reports database: {filepath}. Use '--reports null' to ignore local Json database"
)
raise
else:
log.info(f"The database file {filepath} do not exist. It will be created.")
return wp_reports
def show(urlpart: str,
filepath: Optional[str] = None,
daemon: bool = False) -> None:
"""Inspect a report in database"""
db = DataBase(filepath, daemon=daemon)
matching_reports = [r for r in db._data if urlpart in r["site"]]
eq_reports = [r for r in db._data if urlpart == r["site"]]
if len(eq_reports):
print(format_results(eq_reports[0], format="cli"))
elif len(matching_reports) == 1:
print(format_results(matching_reports[0], format="cli"))
elif len(matching_reports) > 1:
print("The following sites match your search: \n")
print(repr(ReportCollection(matching_reports)))
print("\nPlease be more specific. \n")
else:
print("No report found")
exit(1)
exit(0)
class DataBase:
"""
Interface to JSON database file.
Write all reports in a thread safe way.
"""
def __repr__(self) -> str:
return repr(self._data)
def __init__(self, filepath: Optional[str] = None, daemon: bool = False):
if not filepath:
filepath = self._find_db_file(daemon=daemon)
self.no_local_storage: bool = filepath == "null"
"True if the DB is disabled"
self.filepath = filepath
self._data = ReportCollection()
self._data.extend(self._build_db(self.filepath))
# Writing into the database file is thread safe
self._wp_report_lock: threading.Lock = threading.Lock()
# Only once instance of WPWatcher can use a database file at a time.
self._wp_report_file_lock: FileLock = FileLock(f"{self.filepath}.lock")
def open(self) -> None:
"""
Acquire the file lock for the DB file.
"""
try:
self._wp_report_file_lock.acquire(timeout=1)
except Timeout as err:
raise RuntimeError(f"Could not use the database file '{self.filepath}' because another instance of WPWatcher is using it. ") from err
log.debug(f"Acquired DB lock file '{self.filepath}.lock'")
try:
self.write()
except:
log.error(
f"Could not write wp_reports database: {self.filepath}. Use '--reports null' to ignore local Json database."
)
raise
def close(self) -> None:
"""
Release the file lock.
"""
self._wp_report_file_lock.release()
log.debug(f"Released DB lock file '{self.filepath}.lock'")
@staticmethod
def _find_db_file(daemon: bool = False) -> str:
files = [DEFAULT_REPORTS] if not daemon else [DEFAULT_REPORTS_DAEMON]
env = ["HOME", "PWD", "XDG_CONFIG_HOME", "APPDATA"]
return Config.find_files(env, files, "[]", create=True)[0]
# Read wp_reports database
def _build_db(self, filepath: str) -> ReportCollection:
"""Load reports database and return the complete structure"""
wp_reports = ReportCollection()
if self.no_local_storage:
return wp_reports
if os.path.isfile(filepath):
try:
with open(filepath, "r") as reportsfile:
wp_reports.extend(
ScanReport(item) for item in json.load(reportsfile)
)
log.info(f"Load wp_reports database: {filepath}")
except Exception:
log.error(
f"Could not read wp_reports database: {filepath}. Use '--reports null' to ignore local Json database"
)
raise
else:
log.info(f"The database file {filepath} do not exist. It will be created.")
return wp_reports
def write(
self, wp_reports: Optional[Iterable[ScanReport]] = None
) -> bool:
"""
Write the reports to the database.
Returns `True` if the reports have been successfully written.
"""
if not self._wp_report_file_lock.is_locked:
raise RuntimeError("The file lock must be acquired before writing data. ")
if not wp_reports:
wp_reports = self._data
for newr in wp_reports:
new = True
for r in self._data:
if r["site"] == newr["site"]:
self._data[self._data.index(r)] = newr
new = False
break
if new:
self._data.append(newr)
# Write to file if not null
if not self.no_local_storage:
# Write method thread safe
while self._wp_report_lock.locked():
time.sleep(0.01)
self._wp_report_lock.acquire()
with open(self.filepath, "w") as reportsfile:
json.dump(self._data, reportsfile, indent=4)
self._wp_report_lock.release()
return True
else:
return False
def find(self, wp_report: ScanReport) -> Optional[ScanReport]:
"""
Find the pre-existing report if any.
"""
last_wp_reports = [r for r in self._data if r["site"] == wp_report["site"]]
last_wp_report: Optional[ScanReport]
if len(last_wp_reports) > 0:
last_wp_report = last_wp_reports[0]
else:
last_wp_report = None
return last_wp_report
class WPWatcher:
"""WPWatcher object
Usage exemple:
.. python::
from wpwatcher.config import Config
from wpwatcher.core import WPWatcher
config = Config.fromenv()
config.update({ 'send_infos': True,
'wp_sites': [ {'url':'exemple1.com'},
{'url':'exemple2.com'} ],
'wpscan_args': ['--format', 'json', '--stealthy']
})
watcher = WPWatcher(config)
exit_code, reports = watcher.run_scans()
for r in reports:
print("%s\t\t%s"%( r['site'], r['status'] ))
"""
# WPWatcher must use a configuration dict
def __init__(self, conf: Config):
"""
Arguments:
- `conf`: the configuration dict. Required
"""
# (Re)init logger with config
_init_log(verbose=conf["verbose"],
quiet=conf["quiet"],
logfile=conf["log_file"])
self._delete_tmp_wpscan_files()
# Init DB interface
self.wp_reports: DataBase = DataBase(filepath=conf["wp_reports"],
daemon=conf['daemon'])
# Update config before passing it to WPWatcherScanner
conf.update({"wp_reports": self.wp_reports.filepath})
# Init scanner
self.scanner: Scanner = Scanner(conf)
# Save sites
conf["wp_sites"] = [Site(site_conf) for site_conf in conf["wp_sites"]]
self.wp_sites: List[Site] = conf["wp_sites"]
# Asynchronous executor
self._executor: concurrent.futures.ThreadPoolExecutor = (
concurrent.futures.ThreadPoolExecutor(
max_workers=conf["asynch_workers"]))
# List of conccurent futures
self._futures: List[concurrent.futures.Future] = [
] # type: ignore [type-arg]
# Register the signals to be caught ^C , SIGTERM (kill) , service restart , will trigger interrupt()
signal.signal(signal.SIGINT, self.interrupt)
signal.signal(signal.SIGTERM, self.interrupt)
self.new_reports = ReportCollection()
"New reports, cleared and filled when running `run_scans`."
self.all_reports = ReportCollection()
"All reports an instance of `WPWatcher` have generated using `run_scans`."
# Dump config
log.debug(f"Configuration:{repr(conf)}")
@staticmethod
def _delete_tmp_wpscan_files() -> None:
"""Delete temp wpcan files"""
# Try delete temp files.
if os.path.isdir("/tmp/wpscan"):
try:
shutil.rmtree("/tmp/wpscan")
log.info("Deleted temp WPScan files in /tmp/wpscan/")
except (FileNotFoundError, OSError, Exception):
log.info(
f"Could not delete temp WPScan files in /tmp/wpscan/\n{traceback.format_exc()}"
)
def _cancel_pending_futures(self) -> None:
"""Cancel all asynchronous jobs"""
for f in self._futures:
if not f.done():
f.cancel()
def interrupt_scans(self) -> None:
"""
Interrupt the scans and append finished scan reports to self.new_reports
"""
# Cancel all scans
self._cancel_pending_futures() # future scans
self.scanner.interrupt() # running scans
self._rebuild_rew_reports()
def _rebuild_rew_reports(self) -> None:
"Recover reports from futures results"
self.new_reports = ReportCollection()
for f in self._futures:
if f.done():
try:
self.new_reports.append(f.result())
except Exception:
pass
def interrupt(self,
sig=None,
frame=None) -> None: # type: ignore [no-untyped-def]
"""Interrupt the program and exit. """
log.error("Interrupting...")
# If called inside ThreadPoolExecutor, raise Exeception
if not isinstance(
threading.current_thread(),
threading._MainThread): # type: ignore [attr-defined]
raise InterruptedError()
self.interrupt_scans()
# Give a 5 seconds timeout to buggy WPScan jobs to finish or ignore them
try:
timeout(5, self._executor.shutdown, kwargs=dict(wait=True))
except TimeoutError:
pass
# Display results
log.info(repr(self.new_reports))
log.info("Scans interrupted.")
# and quit
sys.exit(-1)
def _log_db_reports_infos(self) -> None:
if len(self.new_reports) > 0 and repr(
self.new_reports) != "No scan report to show":
if self.wp_reports.filepath != "null":
log.info(
f"Updated reports in database: {self.wp_reports.filepath}")
else:
log.info("Local database disabled, no reports updated.")
def _scan_site(self, wp_site: Site) -> Optional[ScanReport]:
"""
Helper method to wrap the scanning process of `WPWatcherScanner.scan_site` and add the following:
- Find the last report in the database and launch the scan
- Write it in DB after scan.
- Print progress bar
This function can be called asynchronously.
"""
last_wp_report = self.wp_reports.find(ScanReport(site=wp_site["url"]))
# Launch scanner
wp_report = self.scanner.scan_site(wp_site, last_wp_report)
# Save report in global instance database and to file when a site has been scanned
if wp_report:
self.wp_reports.write([wp_report])
else:
log.info(f"No report saved for site {wp_site['url']}")
# Print progress
print_progress_bar(len(self.scanner.scanned_sites), len(self.wp_sites))
return wp_report
def _run_scans(self, wp_sites: List[Site]) -> ReportCollection:
"""
Helper method to deal with :
- executor, concurent futures
- Trigger self.interrupt() on InterruptedError (raised if fail fast enabled)
- Append result to `self.new_reports` list.
"""
log.info(f"Starting scans on {len(wp_sites)} configured sites")
# reset new reports and scanned sites list.
self._futures.clear()
self.new_reports.clear()
self.scanner.scanned_sites.clear()
for wp_site in wp_sites:
self._futures.append(
self._executor.submit(self._scan_site, wp_site))
for f in self._futures:
try:
self.new_reports.append(f.result())
except concurrent.futures.CancelledError:
pass
# Ensure everything is down
self._cancel_pending_futures()
return self.new_reports
def run_scans(self) -> Tuple[int, ReportCollection]:
"""
Run WPScan on defined websites and send notifications.
:Returns: `tuple (exit code, reports)`
"""
# Check sites are in the config
if len(self.wp_sites) == 0:
log.error(
"No sites configured, please provide wp_sites in config file or use arguments --url URL [URL...] or --urls File path"
)
return (-1, self.new_reports)
self.wp_reports.open()
try:
self._run_scans(self.wp_sites)
# Handle interruption from inside threads when using --ff
except InterruptedError:
self.interrupt()
finally:
self.wp_reports.close()
# Print results and finish
log.info(repr(self.new_reports))
if not any([r["status"] == "ERROR" for r in self.new_reports if r]):
log.info("Scans finished successfully.")
return (0, self.new_reports)
else:
log.info("Scans finished with errors.")
return (-1, self.new_reports)
def test_wp_reports_read_write(self):
SPECIFIC_WP_REPORTS_FILE_CONFIG = DEFAULT_CONFIG + "\nwp_reports=%s"
# Compare with config and no config
db = DataBase()
paths_found = db._find_db_file()
db2 = DataBase(
filepath=Config.fromstring(SPECIFIC_WP_REPORTS_FILE_CONFIG %
(paths_found))['wp_reports'])
self.assertEqual(
db._data, db2._data,
"WP reports database are different even if files are the same")
# Test Reports database
reports = [
ScanReport({
"site":
"exemple.com",
"status":
"WARNING",
"datetime":
"2020-04-08T16-05-16",
"last_email":
None,
"error":
'',
"infos": ["[+]", "blablabla"],
"warnings": [
"[+] WordPress version 5.2.2 identified (Insecure, released on 2019-06-18).\n| Found By: Emoji Settings (Passive Detection)\n",
"[!] No WPVulnDB API Token given, as a result vulnerability data has not been output.\n[!] You can get a free API token with 50 daily requests by registering at https://wpvulndb.com/users/sign_up"
],
"alerts": [],
"fixed": []
}),
ScanReport({
"site": "exemple2.com",
"status": "INFO",
"datetime": "2020-04-08T16-05-16",
"last_email": None,
"error": '',
"infos": ["[+]", "blablabla"],
"warnings": [],
"alerts": [],
"fixed": []
})
]
db = DataBase()
db.open()
db.write(reports)
db.close()
# Test internal _data gets updated after write() method
for r in reports:
self.assertIn(
r, db._data,
"The report do not seem to have been saved into WPWatcher.wp_report list"
)
# Test write method
wrote_db = ReportCollection(
ScanReport(item) for item in db._build_db(db.filepath))
with open(db.filepath, 'r') as dbf:
wrote_db_alt = ReportCollection(
ScanReport(item) for item in json.load(dbf))
for r in reports:
self.assertIn(
r, list(wrote_db),
"The report do not seem to have been saved into db file")
self.assertIn(
r, list(wrote_db_alt),
"The report do not seem to have been saved into db file (directly read with json.load)"
)
self.assertIsNotNone(
db.find(ScanReport(site=r['site'])),
"The report do not seem to have been saved into db, cannot find it using find(). "
)
self.assertEqual(
list(db._data), list(wrote_db_alt),
"The database file wrote (directly read with json.load) differ from in memory database"
)
self.assertEqual(
list(db._data), list(wrote_db),
"The database file wrote differ from in memory database")