def startAutoscan(self):
"""Start an automatic scan. Will try to launch all undone tools."""
mongoInstance = MongoCalendar.getInstance()
workers = mongoInstance.getWorkers(
{"excludedDatabases": {
"$nin": [mongoInstance.calendarName]
}})
workers = [w for w in workers]
if len(workers) == 0:
tk.messagebox.showwarning(
"No selected worker found",
"Check worker treeview to see if there are workers registered and double click on the disabled one to enable them"
)
return
if self.settings.db_settings.get("include_all_domains", False):
answer = tk.messagebox.askyesno(
"Autoscan warning",
"The current settings will add every domain found in attack's scope. Are you sure ?"
)
if not answer:
return
from core.Components.AutoScanMaster import sendStartAutoScan
self.btn_autoscan.configure(text="Stop Scanning",
command=self.stopAutoscan)
tasks = sendStartAutoScan(MongoCalendar.getInstance().calendarName)
self.running_auto_scans = tasks
def sendStartAutoScan(calendarName):
mongoInstance = MongoCalendar.getInstance()
mongoInstance.connectToDb(calendarName)
workers = mongoInstance.getWorkers(
{"excludedDatabases": {
"$nin": [mongoInstance.calendarName]
}})
launchedTasks = []
for worker in workers:
worker = worker["name"]
if worker != "localhost":
queueName = str(mongoInstance.calendarName)+"&" + \
worker
app.control.add_consumer(queue=queueName,
reply=True,
exchange="celery",
exchange_type="direct",
routing_key="transient",
destination=[worker])
from AutoScanWorker import startAutoScan
result_async = startAutoScan.apply_async(
args=[calendarName, worker],
queue=queueName,
retry=False,
serializer="json")
launchedTasks.append(result_async)
# Append to running tasks this celery result and the corresponding tool id
return launchedTasks
def tabSwitch(self, event):
"""Called when the user click on the tab menu to switch tab. Add a behaviour before the tab switches.
Args:
event : hold informations to identify which tab was clicked.
"""
tabName = self.nbk.tab(self.nbk.select(), "text").strip()
self.searchBar.quit()
if tabName == "Commands":
self.commandsTreevw.initUI()
mongoInstance = MongoCalendar.getInstance()
if not mongoInstance.hasACalendarOpen():
opened = self.promptCalendarName()
if opened is None:
return
if tabName == "Scan":
if mongoInstance.calendarName is not None:
if mongoInstance.calendarName != "":
if os.name != 'nt': # Disable on windows
# if self.scanManager is None:
# self.scanManager = ScanManager(mongoInstance.calendarName, self.settings)
self.scanManager.initUI(self.scanViewFrame)
else:
lbl = ttk.Label(self.scanViewFrame, text="Disabled on windows because celery does not support it.")
lbl.pack()
elif tabName == "Settings":
self.settings.reloadUI()
else:
for module in self.modules:
if tabName.strip().lower() == module["name"].strip().lower():
module["object"].open()
event.widget.winfo_children()[event.widget.index("current")].update()
def openCalendar(self, filename=""):
"""
Open the given database name. Loads it in treeview.
Args:
filename: the pentest database name to load in application. If "" is given (default), will refresh the already opened database if there is one.
"""
print("Start monitoring")
calendarName = None
mongoInstance = MongoCalendar.getInstance()
if filename == "" and mongoInstance.hasACalendarOpen():
calendarName = mongoInstance.calendarName
elif filename != "":
calendarName = filename.split(".")[0].split("/")[-1]
if calendarName is not None:
if self.scanManager is not None:
self.scanManager.stop()
if self.notifications_timers is not None:
self.notifications_timers.cancel()
self.notifications_timers = None
mongoInstance.connectToDb(calendarName)
for widget in self.viewframe.winfo_children():
widget.destroy()
for module in self.modules:
module["object"].initUI(module["view"], self.nbk, self.treevw)
self.statusbar.reset()
self.treevw.refresh()
if os.name != "nt": # On windows celery 4.X is not managed
self.scanManager = ScanManager(self.nbk, self.treevw, mongoInstance.calendarName, self.settings)
self.notifications_timers = threading.Timer(
0.5, self.readNotifications)
self.notifications_timers.start()
def generateReportPowerpoint(self):
"""
Export a calendar defects to a pptx formatted file.
"""
if self.ent_client.get().strip() == "":
tk.messagebox.showerror(
"Missing required field", "The client's name input must be filled.")
return
if self.ent_contract.get().strip() == "":
tk.messagebox.showerror(
"Missing required field", "The contract's name input must be filled.")
return
mongoInstance = MongoCalendar.getInstance()
toExport = mongoInstance.calendarName
if toExport != "":
modele_pptx = str(self.val_ppt.get())
timestr = datetime.now().strftime("%Y%m%d-%H%M%S")
basename = self.ent_client.get().strip() + " - "+self.ent_contract.get().strip()
out_name = str(timestr)+" - "+basename
dialog = ChildDialogProgress(
self.parent, "Powerpoint Report", "Creating report "+str(out_name) + ". Please wait.", 200, progress_mode="determinate")
PowerpointExport.createReport(self.getDefectsAsDict(), modele_pptx, out_name, client=self.ent_client.get(
).strip(), contract=self.ent_contract.get().strip(), root=self.parent, progressbar=dialog)
dialog.destroy()
tkinter.messagebox.showinfo(
"Success", "The document was generated in ./exports/"+str(out_name))
def _getParent(self):
"""
Return the mongo ObjectId _id of the first parent of this object.
Returns:
Returns the parent's ObjectId _id".
"""
if self.parent is not None:
return self.parent
try:
if IPAddress(self.ip).is_private():
return None
except AddrFormatError:
return None
except ValueError:
return None
ip_real = performLookUp(self.ip)
if ip_real is not None:
mongoInstance = MongoCalendar.getInstance()
ip_in_db = mongoInstance.find("ips", {"ip": ip_real}, False)
if ip_in_db is None:
return None
self.parent = ip_in_db["_id"]
self.update({"parent": self.parent})
return ip_in_db["_id"]
return None
def getPorts(self):
"""Returns ip assigned ports as a list of mongo fetched defects dict
Returns:
list of port raw mongo data dictionnaries
"""
mongoInstance = MongoCalendar.getInstance()
return mongoInstance.find("ports", {"ip": self.ip})
def addAllTool(self, command_name, wave_name, scope):
"""
Kind of recursive operation as it will call the same function in its children ports.
Add the appropriate tools (level check and wave's commands check) for this ip.
Also add for all registered ports the appropriate tools.
Args:
command_name: The command that we want to create all the tools for.
wave_name: the wave name from where we want to load tools
scope: a scope object allowing to launch this command. Opt
"""
# retrieve the command level
mongoInstance = MongoCalendar.getInstance()
command = mongoInstance.findInDb(mongoInstance.calendarName,
"commands", {"name": command_name}, False)
if command["lvl"] == "ip":
# finally add tool
newTool = Tool()
newTool.initialize(command_name, wave_name,
"", self.ip, "", "", "ip")
newTool.addInDb()
return
# Do the same thing for all children ports.
ports = mongoInstance.find("ports", {"ip": self.ip})
for port in ports:
p = Port(port)
p.addAllTool(command_name, wave_name, scope)
def addInDb(self):
"""
Add this scope in database.
Returns: a tuple with :
* bool for success
* mongo ObjectId : already existing object if duplicate, create object id otherwise
"""
base = self.getDbKey()
# Checking unicity
mongoInstance = MongoCalendar.getInstance()
existing = mongoInstance.find("scopes", base, False)
if existing is not None:
return False, existing["_id"]
# Inserting scope
parent = self.getParent()
res_insert = mongoInstance.insert("scopes", base, parent)
ret = res_insert.inserted_id
self._id = ret
# adding the appropriate tools for this scope.
wave = mongoInstance.find("waves", {"wave": self.wave}, False)
commands = wave["wave_commands"]
for commName in commands:
if commName.strip() != "":
self.addAllTool(commName)
# Testing this scope against every ips
ips = Ip.fetchObjects({})
for ip in ips:
if self._id not in ip.in_scopes:
if ip.fitInScope(self.scope):
ip.addScopeFitting(self.getId())
return True, ret
def autoScanv2(databaseName, workerName):
"""
Search tools to launch within defined conditions and attempts to launch them this celery worker.
Gives a visual feedback on stdout
Args:
databaseName: The database to search tools in
endless: a boolean that indicates if the autoscan will be endless or if it will stop at the moment it does not found anymore launchable tools.
useReprinter: a boolean that indicates if the array outpur will be entirely reprinted or if it will be overwritten.
"""
mongoInstance = MongoCalendar.getInstance()
mongoInstance.connectToDb(databaseName)
time_compatible_waves_id = Wave.searchForAddressCompatibleWithTime()
worker = Worker(workerName)
while True:
# Extract commands with compatible time and not yet done
launchableTools, waiting = findLaunchableToolsOnWorker(
worker, databaseName)
# Sort by command priority
launchableTools.sort(
key=lambda tup: (tup["errored"], int(tup["priority"])))
# print(str(launchableTools))
dispatchLaunchableToolsv2(launchableTools, worker)
time.sleep(3)
def getNotDoneTools(cls, waveName):
"""Returns a set of tool mongo ID that are not done yet.
"""
notDoneTools = set()
mongoInstance = MongoCalendar.getInstance()
tools = mongoInstance.find(
"tools", {
"wave": waveName,
"ip": "",
"scanner_ip": "None",
"dated": "None",
"datef": "None"
})
for tool in tools:
notDoneTools.add(tool["_id"])
scopes = Scope.fetchObjects({"wave": waveName})
for scope in scopes:
scopeId = scope.getId()
ips = Ip.getIpsInScope(scopeId)
for ip in ips:
tools = mongoInstance.find(
"tools", {
"wave": waveName,
"ip": ip.ip,
"scanner_ip": "None",
"dated": "None",
"datef": "None"
})
for tool in tools:
notDoneTools.add(tool["_id"])
return notDoneTools
def getScopes(self):
"""Return wave assigned scopes as a list of mongo fetched scopes dict
Returns:
list of defect raw mongo data dictionnaries
"""
mongoInstance = MongoCalendar.getInstance()
return mongoInstance.find("scopes", {"wave": self.wave})
def update(self, pipeline_set=None):
"""Update this object in database.
Args:
pipeline_set: (Opt.) A dictionnary with custom values. If None (default) use model attributes.
"""
oldPort = Port.fetchObject({"_id": ObjectId(self._id)})
if oldPort is None:
return
oldService = oldPort.service
mongoInstance = MongoCalendar.getInstance()
if oldService != self.service:
mongoInstance.delete("tools", {
"lvl": "port", "ip": self.ip, "port": self.port, "proto": self.proto}, many=True)
port_commands = mongoInstance.findInDb(
"pollenisator", "commands", {"lvl": "port"})
for port_command in port_commands:
allowed_services = port_command["ports"].split(",")
for i, elem in enumerate(allowed_services):
if not(elem.strip().startswith("tcp/") or elem.strip().startswith("udp/")):
allowed_services[i] = "tcp/"+str(elem)
if self.proto+"/"+str(self.service) in allowed_services:
waves = mongoInstance.find("waves", {"wave_commands": {"$elemMatch": {
"$eq": port_command["name"].strip()}}})
for wave in waves:
tool_m = Tool().initialize(port_command["name"], wave["wave"], "",
self.ip, self.port, self.proto, "port")
tool_m.addInDb()
# Update variable instance. (this avoid to refetch the whole command in database)
if pipeline_set is None:
mongoInstance.update("ports", {"_id": ObjectId(self._id)}, {
"$set": {"service": self.service, "product":self.product, "notes": self.notes, "tags": self.tags, "infos": self.infos}})
else:
mongoInstance.update("ports", {"_id": ObjectId(self._id)}, {
"$set": pipeline_set})
def getDefects(self):
"""Return port assigned defects as a list of mongo fetched defects dict
Returns:
list of defect raw mongo data dictionnaries
"""
mongoInstance = MongoCalendar.getInstance()
return mongoInstance.find("defects", {"ip": self.ip, "port": self.port, "proto": self.proto})
def autoScan(databaseName, endless, useReprinter=False):
"""
Search tools to launch within defined conditions and attempts to launch them on celery workers.
Gives a visual feedback on stdout
Args:
databaseName: The database to search tools in
endless: a boolean that indicates if the autoscan will be endless or if it will stop at the moment it does not found anymore launchable tools.
useReprinter: a boolean that indicates if the array outpur will be entirely reprinted or if it will be overwritten.
"""
mongoInstance = MongoCalendar.getInstance()
mongoInstance.connectToDb(databaseName)
my_monitor = Monitor(databaseName)
Utils.resetUnfinishedTools()
time_compatible_waves_id = Wave.searchForAddressCompatibleWithTime()
killer = GracefulKiller()
if not endless:
killer.kill_now = len(time_compatible_waves_id) <= 0
print("No wave compatible")
else:
killer.kill_now = False
if useReprinter:
reprinter = Reprinter()
else:
reprinter = None
max_tabulation = _getMaxColumnLen()
while not killer.kill_now:
# Extract commands with compatible time and not yet done
launchableTools, waiting = findLaunchableTools()
# Sort by command priority
launchableTools.sort(key=lambda tup: int(tup["priority"]))
dispatchLaunchableTools(my_monitor, launchableTools)
printStatus(max_tabulation, waiting, reprinter)
time.sleep(3)
my_monitor.stop()
def removeInactiveWorkers(self):
"""
Remove inactive workers every 30s
"""
print("Time to remove inactive workers !")
mongoInstance = MongoCalendar.getInstance()
mongoInstance.removeInactiveWorkers()
def update(self, pipeline_set=None):
"""Update this object in database.
Args:
pipeline_set: (Opt.) A dictionnary with custom values. If None (default) use model attributes.
"""
mongoInstance = MongoCalendar.getInstance()
if pipeline_set is None:
mongoInstance.update("defects", {"_id": ObjectId(self._id)}, {
"$set": {
"ip": self.ip,
"title": self.title,
"port": self.port,
"proto": self.proto,
"notes": self.notes,
"ease": self.ease,
"impact": self.impact,
"risk": self.risk,
"redactor": self.redactor,
"type": list(self.mtype),
"proofs": self.proofs,
"infos": self.infos,
"index": self.index
}
})
else:
mongoInstance.update("defects", {"_id": ObjectId(self._id)},
{"$set": pipeline_set})
def addAllTool(self, command_name):
"""
Add the appropriate tools (level check and wave's commands check) for this scope.
Args:
command_name: The command that we want to create all the tools for.
"""
mongoInstance = MongoCalendar.getInstance()
command = mongoInstance.findInDb("pollenisator", "commands",
{"name": command_name}, False)
if command["lvl"] == "network":
newTool = Tool()
newTool.initialize(command["name"], self.wave, self.scope, "", "",
"", "network")
newTool.addInDb()
return
if command["lvl"] == "domain":
if not Utils.isNetworkIp(self.scope):
newTool = Tool()
newTool.initialize(command["name"], self.wave, self.scope, "",
"", "", "domain")
newTool.addInDb()
return
ips = self.getIpsFitting()
for ip in ips:
i = Ip(ip)
i.addAllTool(command_name, self.wave, self.scope)
def getTools(self):
"""Return ip assigned tools as a list of mongo fetched defects dict
Returns:
list of tool raw mongo data dictionnaries
"""
mongoInstance = MongoCalendar.getInstance()
return mongoInstance.find("tools", {"lvl": "ip", "ip": self.ip})
def getIpsFitting(self):
"""Returns a list of ip mongo dict fitting this scope
Returns:
A list ip IP dictionnary from mongo db
"""
mongoInstance = MongoCalendar.getInstance()
ips = mongoInstance.find("ips", )
ips_fitting = []
isdomain = self.isDomain()
for ip in ips:
if isdomain:
my_ip = Utils.performLookUp(self.scope)
my_domain = self.scope
ip_isdomain = not Utils.isIp(ip["ip"])
if ip_isdomain:
if my_domain == ip["ip"]:
ips_fitting.append(ip)
if Scope.isSubDomain(my_domain, ip["ip"]):
ips_fitting.append(ip)
else:
if my_ip == ip["ip"]:
ips_fitting.append(ip)
else:
if Ip.checkIpScope(self.scope, ip["ip"]):
ips_fitting.append(ip)
return ips_fitting
def getDefects(self):
"""Returns ip assigned tools as a list of mongo fetched defects dict
Returns:
list of defect raw mongo data dictionnaries
"""
mongoInstance = MongoCalendar.getInstance()
return mongoInstance.find("defects", {"ip": self.ip, "$or": [{"port": {"$exists": False}}, {"port": None}]})
def delete(self):
"""
Delete the Scope represented by this model in database.
Also delete the tools associated with this scope
Also remove this scope from ips in_scopes attributes
"""
# deleting tool with scope lvl
tools = Tool.fetchObjects({
"scope":
self.scope,
"wave":
self.wave,
"$or": [{
"lvl": "network"
}, {
"lvl": "domain"
}]
})
for tool in tools:
tool.delete()
# Deleting this scope against every ips
ips = Ip.getIpsInScope(self._id)
for ip in ips:
ip.removeScopeFitting(self._id)
mongoInstance = MongoCalendar.getInstance()
mongoInstance.delete("scopes", {"_id": self._id})
parent_wave = mongoInstance.find("waves", {"wave": self.wave}, False)
if parent_wave is None:
return
mongoInstance.notify(mongoInstance.calendarName, "waves",
parent_wave["_id"], "update", "")
def moveItemDown(self, _event=None):
"""
Swap the selected treeview item with the one down below it.
Args:
_event: not used but mandatory
Returns:
returns "break" to stop the interrupt the event thus preventing cursor to move down
"""
selected = self.treevw.selection()[0]
len_max = len(self.treevw.get_children())
currentIndice = len_max-1
children = self.treevw.get_children()
for i, child in enumerate(children):
if child == selected:
currentIndice = i
break
if currentIndice < len_max-1:
self.treevw.move(selected, '', currentIndice+1)
mongoInstance = MongoCalendar.getInstance()
selected = children[currentIndice]
moved_by_side_effect = children[currentIndice+1]
mongoInstance.update(Defect.coll_name,
{"_id": ObjectId(selected)}, {"$set": {"index":str(currentIndice+1)}})
mongoInstance.update(Defect.coll_name,
{"_id": ObjectId(moved_by_side_effect)}, {"$set": {"index":str(currentIndice)}})
return "break"
def launchCallback(self, _event=None):
"""
Callback for the launch tool button. Will queue this tool to a celery worker. #TODO move to ToolController
Will try to launch respecting limits first. If it does not work, it will asks the user to force launch.
Args:
_event: Automatically generated with a button Callback, not used.
"""
res = self.safeLaunchCallback()
if not res:
dialog = ChildDialogQuestion(
self.appliViewFrame, "Safe queue failed",
"This tool cannot be launched because no worker add space for its thread.\nDo you want to launch it anyway?"
)
self.appliViewFrame.wait_window(dialog.app)
answer = dialog.rvalue
if answer == "Yes":
mongoInstance = MongoCalendar.getInstance()
res = self.mainApp.scanManager.monitor.launchTask(
mongoInstance.calendarName, self.controller.model, "",
False)
if res:
self.controller.update()
self.form.clear()
for widget in self.appliViewFrame.winfo_children():
widget.destroy()
self.openModifyWindow()
def save(self):
"""
Save all the settings (local, database and global)
"""
mongoInstance = MongoCalendar.getInstance()
for k, v in self.global_settings.items():
if mongoInstance.findInDb("pollenisator", "settings", {"key": k},
False) is None:
mongoInstance.insertInDb("pollenisator", "settings", {
"key": k,
"value": v
})
else:
mongoInstance.updateInDb("pollenisator", "settings",
{"key": k}, {"$set": {
"value": v
}})
for k, v in self.db_settings.items():
if mongoInstance.find("settings", {"key": k}, False) is None:
mongoInstance.insert("settings", {"key": k, "value": v})
else:
mongoInstance.update("settings", {"key": k},
{"$set": {
"value": v
}})
self.saveLocalSettings()
self.reloadUI()
def addOnlineWorker(self, worker_hostname):
"""
Register a celery worker on the worker's list. Also deletes old queues and messages
Args:
worker_hostname: the worker hostname to register on worker's list
"""
mongoInstance = MongoCalendar.getInstance()
agg_queues = mongoInstance.aggregateFromDb(
"broker_pollenisator", "messages.routing", [{
"$group": {
"_id": "$queue"
}
}, {
"$match": {
"_id": {
"$regex": "^.*&" + worker_hostname + "&.*$"
}
}
}])
mongoInstance.deleteFromDb(
"broker_pollenisator", "messages.routing",
{"queue": {
"$regex": "^.*&" + worker_hostname + "&.*$"
}}, True)
for agg_queue in agg_queues:
Utils.execute(
"celery -A slave purge -f -Q '" + agg_queue["_id"] + "'", None,
False)
self.workerRegisterCommands(worker_hostname)
def readNotifications(self):
"""
Read notifications from database every 0.5 or so second. Notifications are used to exchange informations between applications.
"""
mongoInstance = MongoCalendar.getInstance()
for old_notif in self.old_notifs:
mongoInstance.deleteFromDb("pollenisator", "notifications", {"_id": old_notif})
self.old_notifs = []
try:
notifications = mongoInstance.findInDb("pollenisator", "notifications", {"$or":[{"db":str(mongoInstance.calendarName)}, {"db":"pollenisator"}]})
for notification in notifications:
# print("Notification received "+str(notification["db"])+"/"+str(notification["collection"])+" iid="+str(notification["iid"])+" action="+str(notification["action"]))
self.old_notifs.append(notification["_id"])
if notification["db"] == "pollenisator":
if notification["collection"] == "workers":
self.scanManager.notify(notification["iid"], notification["action"])
elif "commands" in notification["collection"]:
self.commandsTreevw.notify(notification["db"], notification["collection"], notification["iid"], notification["action"], notification.get("parent", ""))
else:
self.treevw.notify(notification["db"], notification["collection"],
notification["iid"], notification["action"], notification.get("parent", ""))
for module in self.modules:
if callable(getattr(module["object"], "notify", None)):
module["object"].notify(notification["db"], notification["collection"],
notification["iid"], notification["action"], notification.get("parent", ""))
except Exception as e:
print(str(e))
self.notifications_timers = threading.Timer(
0.5, self.readNotifications)
self.notifications_timers.start()
def updateWorkerLastHeartbeat(self, worker_hostname):
"""Update the given worker last hearthbeat
Args:
worker_hostname: the worker name to be refreshed
"""
mongoInstance = MongoCalendar.getInstance()
mongoInstance.updateWorkerLastHeartbeat(worker_hostname)
def importCommands(self, name=None):
"""
Import a pollenisator archive file gunzip to database.
Args:
name: The filename of the gunzip command table exported previously
Returns:
None if name is None and filedialog is closed
True if commands successfully are imported
False otherwise.
"""
filename = ""
if name is None:
f = tkinter.filedialog.askopenfilename(defaultextension=".gzip")
if f is None: # asksaveasfile return `None` if dialog closed with "cancel".
return
filename = str(f)
else:
filename = name
try:
mongoInstance = MongoCalendar.getInstance()
mongoInstance.importCommands(filename)
self.commandsTreevw.refresh()
except IOError:
tkinter.messagebox.showerror(
"Import commands", "Import failed. "+str(filename)+" was not found or is not a file.")
return False
tkinter.messagebox.showinfo(
"Import commands", "Import of "+filename+" completed")
return True
def removeWorker(self, worker_hostname):
"""Remove a worker from database
Args:
worker_hostname: the worker name to be removed
"""
mongoInstance = MongoCalendar.getInstance()
mongoInstance.removeWorker(worker_hostname)
