def startAutoscan(self): """Start an automatic scan. Will try to launch all undone tools.""" mongoInstance = MongoCalendar.getInstance() workers = mongoInstance.getWorkers( {"excludedDatabases": { "$nin": [mongoInstance.calendarName] }}) workers = [w for w in workers] if len(workers) == 0: tk.messagebox.showwarning( "No selected worker found", "Check worker treeview to see if there are workers registered and double click on the disabled one to enable them" ) return if self.settings.db_settings.get("include_all_domains", False): answer = tk.messagebox.askyesno( "Autoscan warning", "The current settings will add every domain found in attack's scope. Are you sure ?" ) if not answer: return from core.Components.AutoScanMaster import sendStartAutoScan self.btn_autoscan.configure(text="Stop Scanning", command=self.stopAutoscan) tasks = sendStartAutoScan(MongoCalendar.getInstance().calendarName) self.running_auto_scans = tasks
def sendStartAutoScan(calendarName): mongoInstance = MongoCalendar.getInstance() mongoInstance.connectToDb(calendarName) workers = mongoInstance.getWorkers( {"excludedDatabases": { "$nin": [mongoInstance.calendarName] }}) launchedTasks = [] for worker in workers: worker = worker["name"] if worker != "localhost": queueName = str(mongoInstance.calendarName)+"&" + \ worker app.control.add_consumer(queue=queueName, reply=True, exchange="celery", exchange_type="direct", routing_key="transient", destination=[worker]) from AutoScanWorker import startAutoScan result_async = startAutoScan.apply_async( args=[calendarName, worker], queue=queueName, retry=False, serializer="json") launchedTasks.append(result_async) # Append to running tasks this celery result and the corresponding tool id return launchedTasks
def tabSwitch(self, event): """Called when the user click on the tab menu to switch tab. Add a behaviour before the tab switches. Args: event : hold informations to identify which tab was clicked. """ tabName = self.nbk.tab(self.nbk.select(), "text").strip() self.searchBar.quit() if tabName == "Commands": self.commandsTreevw.initUI() mongoInstance = MongoCalendar.getInstance() if not mongoInstance.hasACalendarOpen(): opened = self.promptCalendarName() if opened is None: return if tabName == "Scan": if mongoInstance.calendarName is not None: if mongoInstance.calendarName != "": if os.name != 'nt': # Disable on windows # if self.scanManager is None: # self.scanManager = ScanManager(mongoInstance.calendarName, self.settings) self.scanManager.initUI(self.scanViewFrame) else: lbl = ttk.Label(self.scanViewFrame, text="Disabled on windows because celery does not support it.") lbl.pack() elif tabName == "Settings": self.settings.reloadUI() else: for module in self.modules: if tabName.strip().lower() == module["name"].strip().lower(): module["object"].open() event.widget.winfo_children()[event.widget.index("current")].update()
def openCalendar(self, filename=""): """ Open the given database name. Loads it in treeview. Args: filename: the pentest database name to load in application. If "" is given (default), will refresh the already opened database if there is one. """ print("Start monitoring") calendarName = None mongoInstance = MongoCalendar.getInstance() if filename == "" and mongoInstance.hasACalendarOpen(): calendarName = mongoInstance.calendarName elif filename != "": calendarName = filename.split(".")[0].split("/")[-1] if calendarName is not None: if self.scanManager is not None: self.scanManager.stop() if self.notifications_timers is not None: self.notifications_timers.cancel() self.notifications_timers = None mongoInstance.connectToDb(calendarName) for widget in self.viewframe.winfo_children(): widget.destroy() for module in self.modules: module["object"].initUI(module["view"], self.nbk, self.treevw) self.statusbar.reset() self.treevw.refresh() if os.name != "nt": # On windows celery 4.X is not managed self.scanManager = ScanManager(self.nbk, self.treevw, mongoInstance.calendarName, self.settings) self.notifications_timers = threading.Timer( 0.5, self.readNotifications) self.notifications_timers.start()
def generateReportPowerpoint(self): """ Export a calendar defects to a pptx formatted file. """ if self.ent_client.get().strip() == "": tk.messagebox.showerror( "Missing required field", "The client's name input must be filled.") return if self.ent_contract.get().strip() == "": tk.messagebox.showerror( "Missing required field", "The contract's name input must be filled.") return mongoInstance = MongoCalendar.getInstance() toExport = mongoInstance.calendarName if toExport != "": modele_pptx = str(self.val_ppt.get()) timestr = datetime.now().strftime("%Y%m%d-%H%M%S") basename = self.ent_client.get().strip() + " - "+self.ent_contract.get().strip() out_name = str(timestr)+" - "+basename dialog = ChildDialogProgress( self.parent, "Powerpoint Report", "Creating report "+str(out_name) + ". Please wait.", 200, progress_mode="determinate") PowerpointExport.createReport(self.getDefectsAsDict(), modele_pptx, out_name, client=self.ent_client.get( ).strip(), contract=self.ent_contract.get().strip(), root=self.parent, progressbar=dialog) dialog.destroy() tkinter.messagebox.showinfo( "Success", "The document was generated in ./exports/"+str(out_name))
def _getParent(self): """ Return the mongo ObjectId _id of the first parent of this object. Returns: Returns the parent's ObjectId _id". """ if self.parent is not None: return self.parent try: if IPAddress(self.ip).is_private(): return None except AddrFormatError: return None except ValueError: return None ip_real = performLookUp(self.ip) if ip_real is not None: mongoInstance = MongoCalendar.getInstance() ip_in_db = mongoInstance.find("ips", {"ip": ip_real}, False) if ip_in_db is None: return None self.parent = ip_in_db["_id"] self.update({"parent": self.parent}) return ip_in_db["_id"] return None
def getPorts(self): """Returns ip assigned ports as a list of mongo fetched defects dict Returns: list of port raw mongo data dictionnaries """ mongoInstance = MongoCalendar.getInstance() return mongoInstance.find("ports", {"ip": self.ip})
def addAllTool(self, command_name, wave_name, scope): """ Kind of recursive operation as it will call the same function in its children ports. Add the appropriate tools (level check and wave's commands check) for this ip. Also add for all registered ports the appropriate tools. Args: command_name: The command that we want to create all the tools for. wave_name: the wave name from where we want to load tools scope: a scope object allowing to launch this command. Opt """ # retrieve the command level mongoInstance = MongoCalendar.getInstance() command = mongoInstance.findInDb(mongoInstance.calendarName, "commands", {"name": command_name}, False) if command["lvl"] == "ip": # finally add tool newTool = Tool() newTool.initialize(command_name, wave_name, "", self.ip, "", "", "ip") newTool.addInDb() return # Do the same thing for all children ports. ports = mongoInstance.find("ports", {"ip": self.ip}) for port in ports: p = Port(port) p.addAllTool(command_name, wave_name, scope)
def addInDb(self): """ Add this scope in database. Returns: a tuple with : * bool for success * mongo ObjectId : already existing object if duplicate, create object id otherwise """ base = self.getDbKey() # Checking unicity mongoInstance = MongoCalendar.getInstance() existing = mongoInstance.find("scopes", base, False) if existing is not None: return False, existing["_id"] # Inserting scope parent = self.getParent() res_insert = mongoInstance.insert("scopes", base, parent) ret = res_insert.inserted_id self._id = ret # adding the appropriate tools for this scope. wave = mongoInstance.find("waves", {"wave": self.wave}, False) commands = wave["wave_commands"] for commName in commands: if commName.strip() != "": self.addAllTool(commName) # Testing this scope against every ips ips = Ip.fetchObjects({}) for ip in ips: if self._id not in ip.in_scopes: if ip.fitInScope(self.scope): ip.addScopeFitting(self.getId()) return True, ret
def autoScanv2(databaseName, workerName): """ Search tools to launch within defined conditions and attempts to launch them this celery worker. Gives a visual feedback on stdout Args: databaseName: The database to search tools in endless: a boolean that indicates if the autoscan will be endless or if it will stop at the moment it does not found anymore launchable tools. useReprinter: a boolean that indicates if the array outpur will be entirely reprinted or if it will be overwritten. """ mongoInstance = MongoCalendar.getInstance() mongoInstance.connectToDb(databaseName) time_compatible_waves_id = Wave.searchForAddressCompatibleWithTime() worker = Worker(workerName) while True: # Extract commands with compatible time and not yet done launchableTools, waiting = findLaunchableToolsOnWorker( worker, databaseName) # Sort by command priority launchableTools.sort( key=lambda tup: (tup["errored"], int(tup["priority"]))) # print(str(launchableTools)) dispatchLaunchableToolsv2(launchableTools, worker) time.sleep(3)
def getNotDoneTools(cls, waveName): """Returns a set of tool mongo ID that are not done yet. """ notDoneTools = set() mongoInstance = MongoCalendar.getInstance() tools = mongoInstance.find( "tools", { "wave": waveName, "ip": "", "scanner_ip": "None", "dated": "None", "datef": "None" }) for tool in tools: notDoneTools.add(tool["_id"]) scopes = Scope.fetchObjects({"wave": waveName}) for scope in scopes: scopeId = scope.getId() ips = Ip.getIpsInScope(scopeId) for ip in ips: tools = mongoInstance.find( "tools", { "wave": waveName, "ip": ip.ip, "scanner_ip": "None", "dated": "None", "datef": "None" }) for tool in tools: notDoneTools.add(tool["_id"]) return notDoneTools
def getScopes(self): """Return wave assigned scopes as a list of mongo fetched scopes dict Returns: list of defect raw mongo data dictionnaries """ mongoInstance = MongoCalendar.getInstance() return mongoInstance.find("scopes", {"wave": self.wave})
def update(self, pipeline_set=None): """Update this object in database. Args: pipeline_set: (Opt.) A dictionnary with custom values. If None (default) use model attributes. """ oldPort = Port.fetchObject({"_id": ObjectId(self._id)}) if oldPort is None: return oldService = oldPort.service mongoInstance = MongoCalendar.getInstance() if oldService != self.service: mongoInstance.delete("tools", { "lvl": "port", "ip": self.ip, "port": self.port, "proto": self.proto}, many=True) port_commands = mongoInstance.findInDb( "pollenisator", "commands", {"lvl": "port"}) for port_command in port_commands: allowed_services = port_command["ports"].split(",") for i, elem in enumerate(allowed_services): if not(elem.strip().startswith("tcp/") or elem.strip().startswith("udp/")): allowed_services[i] = "tcp/"+str(elem) if self.proto+"/"+str(self.service) in allowed_services: waves = mongoInstance.find("waves", {"wave_commands": {"$elemMatch": { "$eq": port_command["name"].strip()}}}) for wave in waves: tool_m = Tool().initialize(port_command["name"], wave["wave"], "", self.ip, self.port, self.proto, "port") tool_m.addInDb() # Update variable instance. (this avoid to refetch the whole command in database) if pipeline_set is None: mongoInstance.update("ports", {"_id": ObjectId(self._id)}, { "$set": {"service": self.service, "product":self.product, "notes": self.notes, "tags": self.tags, "infos": self.infos}}) else: mongoInstance.update("ports", {"_id": ObjectId(self._id)}, { "$set": pipeline_set})
def getDefects(self): """Return port assigned defects as a list of mongo fetched defects dict Returns: list of defect raw mongo data dictionnaries """ mongoInstance = MongoCalendar.getInstance() return mongoInstance.find("defects", {"ip": self.ip, "port": self.port, "proto": self.proto})
def autoScan(databaseName, endless, useReprinter=False): """ Search tools to launch within defined conditions and attempts to launch them on celery workers. Gives a visual feedback on stdout Args: databaseName: The database to search tools in endless: a boolean that indicates if the autoscan will be endless or if it will stop at the moment it does not found anymore launchable tools. useReprinter: a boolean that indicates if the array outpur will be entirely reprinted or if it will be overwritten. """ mongoInstance = MongoCalendar.getInstance() mongoInstance.connectToDb(databaseName) my_monitor = Monitor(databaseName) Utils.resetUnfinishedTools() time_compatible_waves_id = Wave.searchForAddressCompatibleWithTime() killer = GracefulKiller() if not endless: killer.kill_now = len(time_compatible_waves_id) <= 0 print("No wave compatible") else: killer.kill_now = False if useReprinter: reprinter = Reprinter() else: reprinter = None max_tabulation = _getMaxColumnLen() while not killer.kill_now: # Extract commands with compatible time and not yet done launchableTools, waiting = findLaunchableTools() # Sort by command priority launchableTools.sort(key=lambda tup: int(tup["priority"])) dispatchLaunchableTools(my_monitor, launchableTools) printStatus(max_tabulation, waiting, reprinter) time.sleep(3) my_monitor.stop()
def removeInactiveWorkers(self): """ Remove inactive workers every 30s """ print("Time to remove inactive workers !") mongoInstance = MongoCalendar.getInstance() mongoInstance.removeInactiveWorkers()
def update(self, pipeline_set=None): """Update this object in database. Args: pipeline_set: (Opt.) A dictionnary with custom values. If None (default) use model attributes. """ mongoInstance = MongoCalendar.getInstance() if pipeline_set is None: mongoInstance.update("defects", {"_id": ObjectId(self._id)}, { "$set": { "ip": self.ip, "title": self.title, "port": self.port, "proto": self.proto, "notes": self.notes, "ease": self.ease, "impact": self.impact, "risk": self.risk, "redactor": self.redactor, "type": list(self.mtype), "proofs": self.proofs, "infos": self.infos, "index": self.index } }) else: mongoInstance.update("defects", {"_id": ObjectId(self._id)}, {"$set": pipeline_set})
def addAllTool(self, command_name): """ Add the appropriate tools (level check and wave's commands check) for this scope. Args: command_name: The command that we want to create all the tools for. """ mongoInstance = MongoCalendar.getInstance() command = mongoInstance.findInDb("pollenisator", "commands", {"name": command_name}, False) if command["lvl"] == "network": newTool = Tool() newTool.initialize(command["name"], self.wave, self.scope, "", "", "", "network") newTool.addInDb() return if command["lvl"] == "domain": if not Utils.isNetworkIp(self.scope): newTool = Tool() newTool.initialize(command["name"], self.wave, self.scope, "", "", "", "domain") newTool.addInDb() return ips = self.getIpsFitting() for ip in ips: i = Ip(ip) i.addAllTool(command_name, self.wave, self.scope)
def getTools(self): """Return ip assigned tools as a list of mongo fetched defects dict Returns: list of tool raw mongo data dictionnaries """ mongoInstance = MongoCalendar.getInstance() return mongoInstance.find("tools", {"lvl": "ip", "ip": self.ip})
def getIpsFitting(self): """Returns a list of ip mongo dict fitting this scope Returns: A list ip IP dictionnary from mongo db """ mongoInstance = MongoCalendar.getInstance() ips = mongoInstance.find("ips", ) ips_fitting = [] isdomain = self.isDomain() for ip in ips: if isdomain: my_ip = Utils.performLookUp(self.scope) my_domain = self.scope ip_isdomain = not Utils.isIp(ip["ip"]) if ip_isdomain: if my_domain == ip["ip"]: ips_fitting.append(ip) if Scope.isSubDomain(my_domain, ip["ip"]): ips_fitting.append(ip) else: if my_ip == ip["ip"]: ips_fitting.append(ip) else: if Ip.checkIpScope(self.scope, ip["ip"]): ips_fitting.append(ip) return ips_fitting
def getDefects(self): """Returns ip assigned tools as a list of mongo fetched defects dict Returns: list of defect raw mongo data dictionnaries """ mongoInstance = MongoCalendar.getInstance() return mongoInstance.find("defects", {"ip": self.ip, "$or": [{"port": {"$exists": False}}, {"port": None}]})
def delete(self): """ Delete the Scope represented by this model in database. Also delete the tools associated with this scope Also remove this scope from ips in_scopes attributes """ # deleting tool with scope lvl tools = Tool.fetchObjects({ "scope": self.scope, "wave": self.wave, "$or": [{ "lvl": "network" }, { "lvl": "domain" }] }) for tool in tools: tool.delete() # Deleting this scope against every ips ips = Ip.getIpsInScope(self._id) for ip in ips: ip.removeScopeFitting(self._id) mongoInstance = MongoCalendar.getInstance() mongoInstance.delete("scopes", {"_id": self._id}) parent_wave = mongoInstance.find("waves", {"wave": self.wave}, False) if parent_wave is None: return mongoInstance.notify(mongoInstance.calendarName, "waves", parent_wave["_id"], "update", "")
def moveItemDown(self, _event=None): """ Swap the selected treeview item with the one down below it. Args: _event: not used but mandatory Returns: returns "break" to stop the interrupt the event thus preventing cursor to move down """ selected = self.treevw.selection()[0] len_max = len(self.treevw.get_children()) currentIndice = len_max-1 children = self.treevw.get_children() for i, child in enumerate(children): if child == selected: currentIndice = i break if currentIndice < len_max-1: self.treevw.move(selected, '', currentIndice+1) mongoInstance = MongoCalendar.getInstance() selected = children[currentIndice] moved_by_side_effect = children[currentIndice+1] mongoInstance.update(Defect.coll_name, {"_id": ObjectId(selected)}, {"$set": {"index":str(currentIndice+1)}}) mongoInstance.update(Defect.coll_name, {"_id": ObjectId(moved_by_side_effect)}, {"$set": {"index":str(currentIndice)}}) return "break"
def launchCallback(self, _event=None): """ Callback for the launch tool button. Will queue this tool to a celery worker. #TODO move to ToolController Will try to launch respecting limits first. If it does not work, it will asks the user to force launch. Args: _event: Automatically generated with a button Callback, not used. """ res = self.safeLaunchCallback() if not res: dialog = ChildDialogQuestion( self.appliViewFrame, "Safe queue failed", "This tool cannot be launched because no worker add space for its thread.\nDo you want to launch it anyway?" ) self.appliViewFrame.wait_window(dialog.app) answer = dialog.rvalue if answer == "Yes": mongoInstance = MongoCalendar.getInstance() res = self.mainApp.scanManager.monitor.launchTask( mongoInstance.calendarName, self.controller.model, "", False) if res: self.controller.update() self.form.clear() for widget in self.appliViewFrame.winfo_children(): widget.destroy() self.openModifyWindow()
def save(self): """ Save all the settings (local, database and global) """ mongoInstance = MongoCalendar.getInstance() for k, v in self.global_settings.items(): if mongoInstance.findInDb("pollenisator", "settings", {"key": k}, False) is None: mongoInstance.insertInDb("pollenisator", "settings", { "key": k, "value": v }) else: mongoInstance.updateInDb("pollenisator", "settings", {"key": k}, {"$set": { "value": v }}) for k, v in self.db_settings.items(): if mongoInstance.find("settings", {"key": k}, False) is None: mongoInstance.insert("settings", {"key": k, "value": v}) else: mongoInstance.update("settings", {"key": k}, {"$set": { "value": v }}) self.saveLocalSettings() self.reloadUI()
def addOnlineWorker(self, worker_hostname): """ Register a celery worker on the worker's list. Also deletes old queues and messages Args: worker_hostname: the worker hostname to register on worker's list """ mongoInstance = MongoCalendar.getInstance() agg_queues = mongoInstance.aggregateFromDb( "broker_pollenisator", "messages.routing", [{ "$group": { "_id": "$queue" } }, { "$match": { "_id": { "$regex": "^.*&" + worker_hostname + "&.*$" } } }]) mongoInstance.deleteFromDb( "broker_pollenisator", "messages.routing", {"queue": { "$regex": "^.*&" + worker_hostname + "&.*$" }}, True) for agg_queue in agg_queues: Utils.execute( "celery -A slave purge -f -Q '" + agg_queue["_id"] + "'", None, False) self.workerRegisterCommands(worker_hostname)
def readNotifications(self): """ Read notifications from database every 0.5 or so second. Notifications are used to exchange informations between applications. """ mongoInstance = MongoCalendar.getInstance() for old_notif in self.old_notifs: mongoInstance.deleteFromDb("pollenisator", "notifications", {"_id": old_notif}) self.old_notifs = [] try: notifications = mongoInstance.findInDb("pollenisator", "notifications", {"$or":[{"db":str(mongoInstance.calendarName)}, {"db":"pollenisator"}]}) for notification in notifications: # print("Notification received "+str(notification["db"])+"/"+str(notification["collection"])+" iid="+str(notification["iid"])+" action="+str(notification["action"])) self.old_notifs.append(notification["_id"]) if notification["db"] == "pollenisator": if notification["collection"] == "workers": self.scanManager.notify(notification["iid"], notification["action"]) elif "commands" in notification["collection"]: self.commandsTreevw.notify(notification["db"], notification["collection"], notification["iid"], notification["action"], notification.get("parent", "")) else: self.treevw.notify(notification["db"], notification["collection"], notification["iid"], notification["action"], notification.get("parent", "")) for module in self.modules: if callable(getattr(module["object"], "notify", None)): module["object"].notify(notification["db"], notification["collection"], notification["iid"], notification["action"], notification.get("parent", "")) except Exception as e: print(str(e)) self.notifications_timers = threading.Timer( 0.5, self.readNotifications) self.notifications_timers.start()
def updateWorkerLastHeartbeat(self, worker_hostname): """Update the given worker last hearthbeat Args: worker_hostname: the worker name to be refreshed """ mongoInstance = MongoCalendar.getInstance() mongoInstance.updateWorkerLastHeartbeat(worker_hostname)
def importCommands(self, name=None): """ Import a pollenisator archive file gunzip to database. Args: name: The filename of the gunzip command table exported previously Returns: None if name is None and filedialog is closed True if commands successfully are imported False otherwise. """ filename = "" if name is None: f = tkinter.filedialog.askopenfilename(defaultextension=".gzip") if f is None: # asksaveasfile return `None` if dialog closed with "cancel". return filename = str(f) else: filename = name try: mongoInstance = MongoCalendar.getInstance() mongoInstance.importCommands(filename) self.commandsTreevw.refresh() except IOError: tkinter.messagebox.showerror( "Import commands", "Import failed. "+str(filename)+" was not found or is not a file.") return False tkinter.messagebox.showinfo( "Import commands", "Import of "+filename+" completed") return True
def removeWorker(self, worker_hostname): """Remove a worker from database Args: worker_hostname: the worker name to be removed """ mongoInstance = MongoCalendar.getInstance() mongoInstance.removeWorker(worker_hostname)