def _rejectError(self, syscall, path, flags, error): """Print a warning that a syscall failed and invalidate the Event.""" # Don't log failed syscalls, but inform the reader if debugEnabled(): print("Info: system call %s(%s, %d) from Application %s:%d " "failed with error %d, and will not be logged." % (syscall, path, flags, self.actor.desktopid, self.actor.pid, error), file=sys.stderr) self.markInvalid()
def purgeFDReferences(self): """Remove FD references that weren't solved yet from the FileStore.""" count = 0 dels = set() delInodes = set() for name in self.nameStore: if name.startswith("@fdref"): dels.add(name) for f in self.nameStore.get(name): delInodes.add(f.inode) for name in dels: del self.nameStore[name] for inode in delInodes: count += 1 del self.inodeStore[inode] if debugEnabled(): print("Info: purged %d unresolved file descriptor references." % count)
def simulateAllEvents(self): """Simulate all events to instantiate Files in the FileStore.""" if not self._sorted: self.sort() fileStore = FileStore.get() fileFactory = FileFactory.get() # First, parse for Zeitgeist designation events in order to instantiate # the designation cache. if debugEnabled(): print("Instantiating Zeitgeist acts of designation...") for event in self.store: if event.isInvalid(): continue if event.getSource() == EventSource.zeitgeist: # The event grants 5 minutes of designation both ways. self.desigcache.addItem(event, start=event.time - 5*60*1000, duration=10*60*1000) # The current Event is an act of designation for future Events # related to the same Application and Files. Save it. elif event.getFileFlags() & EventFileFlags.designationcache: self.desigcache.addItem(event) if debugEnabled(): print("Done. Starting simulation...") # Then, dispatch each event to the appropriate handler for event in self.store: if event.isInvalid(): continue # Designation events are already processed. if event.getFileFlags() & EventFileFlags.designationcache: continue if debugEnabled(): print("Simulating Event %s from %s at time %s." % ( event.evflags, event.actor.uid(), time2Str(event.time))) for data in event.data_app: if data[2] == FD_OPEN: event.actor.openFD(data[0], data[1], event.time) elif data[2] == FD_CLOSE: event.actor.closeFD(data[0], event.time) if event.getFileFlags() & EventFileFlags.destroy: res = self.simulateDestroy(event, fileFactory, fileStore) elif event.getFileFlags() & EventFileFlags.create: res = self.simulateCreate(event, fileFactory, fileStore) elif event.getFileFlags() & EventFileFlags.overwrite: res = self.simulateCreate(event, fileFactory, fileStore) # We received a list of files that were created if isinstance(res, list): pass # We received instructions to hot-patch the event list else: raise NotImplementedError # TODO elif event.getFileFlags() & (EventFileFlags.read | EventFileFlags.write): self.simulateAccess(event, fileFactory, fileStore) # Keep me last, or use elif guards: I WILL change your event flags! elif event.getFileFlags() & EventFileFlags.move or \ event.getFileFlags() & EventFileFlags.copy: res = self.simulateCopy(event, fileFactory, fileStore, keepOld=event.getFileFlags() & EventFileFlags.copy) # Filter out invalid file descriptor references before computing stats. fileStore.purgeFDReferences()
def simulateCopy(self, event: Event, fileFactory: FileFactory, fileStore: FileStore, keepOld: bool=True): """Simulate a file copy or move Event, based on :keepOld:.""" newFiles = [] ctype = 'copy' if keepOld else 'move' baseFlags = event.evflags def _delFile(event: Event, f, read: bool=False): event.evflags = (baseFlags | EventFileFlags.write | EventFileFlags.destroy) if read: event.evflags |= EventFileFlags.read res = self.desigcache.checkForDesignation(event, [f]) fileFactory.deleteFile(f, event.actor, event.time, res[0][1]) event.evflags = baseFlags def _addRead(event: Event, f): event.evflags = (baseFlags | EventFileFlags.read) res = self.desigcache.checkForDesignation(event, [f]) f.addAccess(actor=event.actor, flags=res[0][1], time=event.time) event.evflags = baseFlags def _createCopy(event: Event, oldFile, newPath): # Create a file on the new path which is identical to the old File. event.evflags = (baseFlags | EventFileFlags.write | EventFileFlags.create | EventFileFlags.overwrite) newFile = self.__doCreateFile(newPath, oldFile.ftype, event, fileFactory, fileStore) event.evflags = baseFlags # Update the files' links oldFile.addFollower(newFile.inode, event.time, ctype) newFile.setPredecessor(oldFile.inode, event.time, ctype) fileStore.updateFile(oldFile) fileStore.updateFile(newFile) return newFile # Get each file, set its starting time and type, and update the store subjects = list((old.path, new.path) for (old, new) in event.getData()) for (old, new) in subjects: # Not legal. 'a' and 'a' are the same file. if old == new: # TODO DBG? continue # Get the old file. It must exist, or the simulation is invalid. oldFile = fileFactory.getFile(old, event.time) if not oldFile: raise ValueError("Attempting to move/copy from a file that " "does not exist: %s at time %d" % ( old, event.time)) if debugEnabled(): print("Info: %s '%s' to '%s' at time %s, by actor %s." % ( ctype, old, new, time2Str(event.time), event.actor.uid())) # Check if the target is a directory, or a regular file. If it does # not exist, it's a regular file. newFile = fileFactory.getFileIfExists(new, event.time) newIsFolder = newFile and newFile.isFolder() # If the target is a directory, we will copy/move inside it. sourceIsFolder = oldFile.isFolder() targetPath = new if not newIsFolder else \ new + "/" + oldFile.getFileName() # If mv/cp'ing a folder to an existing path, restrictions apply. if sourceIsFolder: # oldFile is a/, newFile is b/, targetFile is b/a/ targetFile = fileFactory.getFileIfExists(targetPath, event.time) targetIsFolder = targetFile and targetFile.isFolder() # cannot overwrite non-directory 'b' with directory 'a' # cannot overwrite non-directory 'b/a' with directory 'a' if targetFile and not targetIsFolder: # TODO DBG? continue # mv: cannot move 'a' to 'b/a': Directory not empty elif targetIsFolder and ctype == "move": children = fileStore.getChildren(targetFile, event.time) if len(children) == 0: _delFile(event, targetFile) else: # TODO DBG? continue # mv or cp would make the target directory here. Our code later # on in this function will create a copy of the old file, which # means the new folder will be made, with a creation access # from the actor that performs the copy event we are analysing. elif not targetFile and ctype == "copy": pass # When the source is a file, just delete the new target path. else: targetFile = fileFactory.getFileIfExists(targetPath, event.time) if targetFile: _delFile(event, targetFile) # Collect the children of the source folder. children = fileStore.getChildren(oldFile, event.time) if \ sourceIsFolder else [] # Make the target file, and link the old and target files. _createCopy(event, oldFile, targetPath) if ctype == "move": _delFile(event, oldFile, read=True) else: _addRead(event, oldFile) # Move or copy the children. for child in children: childRelPath = child.path[len(oldFile.path)+1:] childTargetPath = targetPath + "/" + childRelPath childNewFile = None # Let the Python purists hang me for that. Iterators will catch # appended elements on a mutable list and this is easier to # read than other solutions that don't modify the list while it # is iterated over. subjects.append((child.path, childTargetPath)) newFiles.append(targetFile) return newFiles
def loadDb(self, store: ApplicationStore = None, checkInitialised: bool = False): """Load the PreloadLogger database. Go through the directory and create all the relevant app instances and events. Can be made to insert all found apps into an ApplicationStore, or to exit if some Application instances are not properly initialised. """ count = 0 # Counter of fetched files, for stats actors = set() # Apps that logged anything at all empties = 0 # Matching files without content (logger crash) invalids = 0 # Files with corrupted content (logger crash) nosyscalls = [] # Logs with zero syscalls logged (not a bug) nosyscallactors = set() # Apps that logged zero syscalls instanceCount = 0 # Count of distinct app instances in the dataset hasErrors = False # Whether some uninitialised apps were found invalidApps = set() # List of desktop IDs that could not be init'd eventCount = 0 # List all log files that match the PreloadLogger syntax for file in os.listdir(self.path): # Ignore files that don't match if not PreloadLoggerLoader.pattern.match(file): continue count += 1 # Process log files that match the PreloadLogger name pattern try: f = open(self.path + "/" + file, 'rb') except (IOError) as e: print("Error: could not open file %s: %s" % (file, str(e)), file=sys.stderr) else: with f: if os.fstat(f.fileno()).st_size == 0: print("Info: file '%s' is empty. Skipping." % file) continue # Parse the first line to get the identity of the app, # but sometimes the header ends up on the second line # in some logs... So, parse until we find a match, and # remember the line index of the header idx = 0 headerLocation = 0 result = None for binary in f: try: line = binary.decode('utf-8') except (UnicodeDecodeError) as e: print("Error: %s has a non utf-8 line: %s " % (file, str(e)), file=sys.stderr) idx += 1 continue result = PreloadLoggerLoader.header.match(line) if result: headerLocation = idx break idx += 1 # Files with a missing or corrupted header are invalid if result is None: print("%s is missing a header" % file, file=sys.stderr) invalids += 1 continue # Parse the header line, make sure it has the right length. g = result.groups() if (len(g) != 3): print("%s has wrong group count: " % file, result.group(), file=sys.stderr) invalids += 1 continue # Filter interpreters, and rewrite them to get the identity # of the app they launched instead. items = space.split(g[2]) interpreterid = None # Python if (pyre.match(g[0])): interpreterid = g[0] g = self.parsePython(g, items) # print("PYTHON APP: %s" % g[2]) # Bash if (bashre.match(g[0])): interpreterid = g[0] g = self.parseBash(g, items) # print("BASH APP: %s" % g[2]) # Java if (javare.match(g[0])): interpreterid = g[0] g = self.parseJava(g, items) # print("JAVA APP: %s" % g[2]) # Perl if (perlre.match(g[0])): interpreterid = g[0] g = self.parsePerl(g, items) # print("PERL APP: %s" % g[2]) # Mono if (monore.match(g[0])): interpreterid = g[0] g = self.parseMono(g, items) # print("MONO APP: %s" % g[2]) # PHP if (phpre.match(g[0])): interpreterid = g[0] g = self.parsePHP(g, items) # print("PHP APP: %s" % g[2]) # Get first and last event to calculate the timestamps. tstart = float("inf") tend = 0 skipCache = None lineIdx = 0 f.seek(0, 0) for binary in f: # Ignore the header. if lineIdx == headerLocation: lineIdx += 1 skipCache = None continue # Decode line. try: line = binary.decode('utf-8') except (UnicodeDecodeError) as e: print("Error: %s has a non utf-8 line: %s " % (file, str(e)), file=sys.stderr) lineIdx += 1 skipCache = None continue # Previous line did not end and was skipped, merge it. if skipCache: line = skipCache + line skipCache = None # Line continues... if line.endswith('\\\n'): skipCache = line lineIdx += 1 continue line = line.rstrip("\n").lstrip(" ") # Line is a parameter to the last system call logged if line.startswith(' '): lineIdx += 1 continue # Check that line is a syntactically valid system call result = PreloadLoggerLoader.syscall.match(line) if result is None: if debugEnabled(): print("%s has a corrupted line (match): %s" % (file, line), file=sys.stderr) lineIdx += 1 continue # Update the timestamp (convert to ZG millisec format) h = result.groups() tstart = int(h[0]) * 1000 break # TODO, first non-header line + tail code. lastLine = tail(f) result = None if lastLine: result = PreloadLoggerLoader.syscall.match(lastLine) if result is None: if debugEnabled(): print("%s's last line is corrupted: %s" % (file, lastLine), file=sys.stderr) else: # Update the timestamp (convert to ZG millisec format) h = result.groups() tend = int(h[0]) * 1000 # Check if the timestamps have been set if tend == 0: nosyscalls.append(g) nosyscallactors.add(g[0]) continue # Sometimes, short logs have event ordering problems... We # can try to ignore these problems as all events are indi- # vidually timestamped anyway. if tstart > tend: tend, start = tstart, tend # TODO: process deletions and remove corresponding files # Make the application try: app = Application(desktopid=g[0], pid=int(g[1]), tstart=tstart, tend=tend, interpreterid=interpreterid) app.setCommandLine(g[2]) except (ValueError) as e: print("MISSING: %s" % g[0], file=sys.stderr) hasErrors = True invalidApps.add(g[0]) continue # Ignore study artefacts! if app.isStudyApp(): continue # Add command-line event event = Event(actor=app, time=tstart, cmdlineStr=g[2]) app.addEvent(event) # Add system call events skipCache = None lineIdx = 0 currentCall = None prevTimestamp = 0 timeDelta = 0 f.seek(0, 0) for binary in f: # Ignore the header. if lineIdx == headerLocation: lineIdx += 1 skipCache = None continue # Decode line. try: line = binary.decode('utf-8') except (UnicodeDecodeError) as e: print("Error: %s has a non utf-8 line: %s " % (file, str(e)), file=sys.stderr) lineIdx += 1 skipCache = None continue # Previous line did not end and was skipped, merge it. if skipCache: line = skipCache + line skipCache = None # Line continues... if line.endswith('\\\n'): skipCache = line lineIdx += 1 continue line = line[:-1] # Remove ending "\n" # Line is a parameter to the last system call logged if line.startswith(' '): if currentCall: currentCall = (currentCall[0], currentCall[1] + '\n' + line) elif debugEnabled(): print("%s has a corrupted line (no call): %s" % (file, line), file=sys.stderr) lineIdx += 1 continue # Check that line is a syntactically valid system call result = PreloadLoggerLoader.syscall.match(line) if result is None: if debugEnabled(): print("%s has a corrupted line (match): %s" % (file, line), file=sys.stderr) lineIdx += 1 continue # Update the timestamp (convert to ZG millisec format) h = result.groups() timestamp = int(h[0]) * 1000 # Append the system call to our syscall list. Note that # we do something odd with the timestamp: because PL # only logs at second precision, a lot of system calls # have the same timestamp, which causes the EventStore # to sort them in the wrong order. So, every time we # have a timestamp identical to the previous one, we # increase a counter that sorts them. This works under # the assumption that there are at most 1000 events per # second. if timestamp == prevTimestamp: timeDelta += 1 else: timeDelta = 0 # Process the last system call into an Event, and clear # up the syscalls list to keep RAM free! if currentCall: event = Event(actor=app, time=currentCall[0], syscallStr=currentCall[1]) app.addEvent(event) eventCount += 1 # Create the new syscalls list. currentCall = (timestamp + timeDelta, h[1]) prevTimestamp = timestamp lineIdx += 1 # Add the found process id to our list of actors, using the # app identity that was resolved by the Application ctor actors.add(app.desktopid) if checkInitialised and not app.isInitialised(): print("MISSING: %s" % g[0], file=sys.stderr) hasErrors = True # Insert into the ApplicationStore if one is available if store is not None: store.insert(app) instanceCount += 1 if checkInitialised and hasErrors: if invalidApps: print("Invalid apps:", file=sys.stderr) for a in sorted(invalidApps): print("\t%s" % a, file=sys.stderr) sys.exit(-1) # print("Apps that logged valid files:") # for act in sorted(actors): # print(act) # print("\nApps that logged files without a single system call:") # for act in sorted(nosyscallactors): # print(act) self.appCount = len(actors) self.instCount = count - empties - invalids - len(nosyscalls) self.eventCount = eventCount self.validEventRatio = 100 - 100 * (invalids + empties + len(nosyscalls)) / (count) print("Finished loading DB.\n%d files seen, %d valid from %d apps, " "%d empty files, " "%d logs with 0 syscalls from %d apps, " "%d invalid.\nIn " "total, %.02f%% files processed." % (count, self.instCount, self.appCount, empties, len(nosyscalls), len(nosyscallactors), invalids, self.validEventRatio)) print("Instance count: %d" % instanceCount)
def loadDb(self, store: ApplicationStore = None): """Browse the SQLite db and create all the relevant app instances.""" # Load up our events from the Zeitgeist database self.cur = self.con.cursor() self.cur.execute('SELECT * \ FROM event_view \ WHERE id IN (SELECT DISTINCT id \ FROM event_view \ WHERE subj_uri LIKE "activity://%")') # Merge all event subjects based on their event id, and find their pids eventsMerged = dict() data = self.cur.fetchone() while data: pid = 0 if "pid://" in data[EV_SUBJ_URI]: m = re.search('(?<=pid://)\d+', data[EV_SUBJ_URI]) pid = int(m.group(0)) if m else 0 ev = eventsMerged.get(data[EV_ID]) if not ev: ev = SqlEvent(id=data[EV_ID], pid=pid, timestamp=data[EV_TIMESTAMP], interpretation=self.getInterpretation( data[EV_INTERPRETATION]), manifestation=self.getManifestation( data[EV_MANIFESTATION]), origin_uri=data[EV_EVENT_ORIGIN_URI], actor_uri=data[EV_ACTOR_URI]) elif pid and ev.pid: assert ev.pid == pid, ("Error: multiple events record a pid " " event %d, and they disagree on the " "pid to record (%d != %d)." % ( data[EV_ID], ev.pid, pid)) elif pid and not ev.pid: ev.pid = pid subj = SqlEventSubject(uri=data[EV_SUBJ_URI], interpretation=self.getInterpretation( data[EV_SUBJ_INTERPRETATION]), manifestation=self.getManifestation( data[EV_SUBJ_MANIFESTATION]), origin_uri=data[EV_SUBJ_ORIGIN_URI], mimetype=self.getMimeType( data[EV_SUBJ_MIMETYPE]), text=data[EV_SUBJ_TEXT], storage_uri=data[EV_SUBJ_STORAGE], current_uri=data[EV_SUBJ_CURRENT_URI]) ev.addSubject(subj) eventsMerged[data[EV_ID]] = ev data = self.cur.fetchone() # Now, sort the events per app PID so we can build apps nopids = [] # Matching events without a PID eventsPerPid = dict() # Storage for our events count = len(eventsMerged) # Counter of fetched events, for stats instanceCount = 0 # Count of distinct app instances in the dataset actors = set() for event in eventsMerged.items(): pid = event[1].pid if not pid: nopids.append(event[1]) else: try: eventsPerPid[pid].append(event[1]) except KeyError as e: eventsPerPid[pid] = [event[1]] del eventsMerged # no longer needed # For each PID, we'll now identify the successive Application instances for (pkey, pevent) in eventsPerPid.items(): pevent = sorted(pevent, key=lambda x: x.timestamp) currentId = '' # currently matched Desktop Id currentApp = None # currently matched Application apps = [] # temp storage for found Applications for ev in pevent: (evId, __) = Application.getDesktopIdFromDesktopUri( ev.actor_uri) if evId != currentId: if debugEnabled(): print ("New application:", evId, currentId, ev) currentId = evId currentApp = Application(desktopid=evId, pid=int(pkey), tstart=ev.timestamp, tend=ev.timestamp) actors.add(currentApp.desktopid) apps.append(currentApp) else: currentApp.setTimeOfStart(min(ev.timestamp, currentApp.getTimeOfStart())) currentApp.setTimeOfEnd(max(ev.timestamp, currentApp.getTimeOfEnd())) # Ignore study artefacts! if not currentApp.isStudyApp(): event = Event(actor=currentApp, time=ev.timestamp, zgEvent=ev) currentApp.addEvent(event) # Insert into the ApplicationStore if one was given to us instanceCount += len(apps) if store is not None: for app in apps: # Ignore study artefacts! if not app.isStudyApp(): store.insert(app) else: instanceCount -= 1 # We discount this app instance self.appCount = len(actors) self.instCount = instanceCount self.eventCount = count self.validEventRatio = 100-100*len(nopids) / count print("Finished loading DB.\n%d events seen, %d normal, %d without a " "PID.\nIn total, %.02f%% events accepted." % ( count, count-len(nopids), len(nopids), self.validEventRatio)) print("Instance count: %d" % self.instCount)
def _runAttackRound(self, attack: Attack, policy: Policy, acListInst: dict, lookUps: dict, allowedCache: dict): """Run an attack round with a set source and time.""" fileStore = FileStore.get() appStore = ApplicationStore.get() userConf = UserConfigLoader.get() userHome = userConf.getHomeDir() seen = set() # Already seen targets. spreadTimes = dict() # Times from which the attack can spread. toSpread = deque() toSpread.append(attack.source) spreadTimes[attack.source] = attack.time # Statistics counters. appSet = set() userAppSet = set() fileCount = 0 docCount = 0 if debugEnabled(): tprnt("Launching attack on %s at time %s %s app memory." % (attack.source if isinstance(attack.source, File) else attack.source.uid(), time2Str(attack.time), "with" if attack.appMemory else "without")) def _allowed(policy, f, acc): k = (policy, f, acc) if k not in allowedCache: v = (policy.fileOwnedByApp(f, acc) or policy.allowedByPolicy(f, acc.actor) or policy.accessAllowedByPolicy(f, acc)) allowedCache[k] = v return v else: return allowedCache[k] # As long as there are reachable targets, loop. while toSpread: current = toSpread.popleft() currentTime = spreadTimes[current] # When the attack spreads to a File. if isinstance(current, File): fileCount += 1 if current.isUserDocument(userHome): docCount += 1 if debugEnabled(): tprnt("File added @%d: %s" % (currentTime, current)) # Add followers. for f in current.follow: if f.time > currentTime: follower = fileStore.getFile(f.inode) if follower not in seen: toSpread.append(follower) seen.add(follower) spreadTimes[follower] = f.time # Add future accesses. for acc in current.accesses: if acc.time > currentTime and \ acc.actor.desktopid not in appSet and \ _allowed(policy, current, acc): toSpread.append(acc.actor) spreadTimes[acc.actor] = acc.time # When the attack spreads to an app instance. elif isinstance(current, Application): if debugEnabled(): tprnt("App added @%d: %s" % (currentTime, current.uid())) # Add files accessed by the app. for (accFile, acc) in acListInst.get(current.uid()) or []: if acc.time > currentTime and \ accFile not in seen and \ _allowed(policy, accFile, acc): toSpread.append(accFile) seen.add(accFile) spreadTimes[accFile] = acc.time # Add future versions of the app. if attack.appMemory and current.desktopid not in appSet: for app in appStore.lookupDesktopId(current.desktopid): if app.tstart > currentTime: toSpread.append(app) spreadTimes[app] = app.tstart # We do this last to use appSet as a cache for already seen # apps, so we append all future instances once and for all to # the spread list. appSet.add(current.desktopid) if current.isUserlandApp(): userAppSet.add(current.desktopid) else: print("Error: attack simulator attempting to parse an unknown" " object (%s)" % type(current), file=sys.stderr) return (appSet, userAppSet, fileCount, docCount)
def main(argv): __opt_inode_query = None __opt_post_analysis = None __opt_quick_pol = None # Parse command-line parameters try: (opts, args) = getopt.getopt(argv, "hta:cedf:o:q:sk:rpgGi:u:x", [ "help", "attacks", "post-analysis=", "check-missing", "check-excluded-files", "debug", "frequency", "inode", "extensions", "related-files", "output=", "output-fs=", "score", "quick-pol=", "skip=", "user", "clusters", "print-clusters", "graph", "graph-clusters", "disable-plotting" ]) except (getopt.GetoptError): print(USAGE_STRING) sys.exit(2) else: for opt, arg in opts: if opt in ('-h', '--help'): print(USAGE_STRING + "\n\n\n\n") print("--attacks:\n\tSimulates attacks and reports " "on proportions of infected files and apps.\n") print("--check-excluded-files:\n\tPrints the lists of files " "accessed by apps that also wrote to excluded\n\tfiles," " then aborts execution of the program.\n") print("--check-missing:\n\tChecks whether some Desktop IDs " "for apps in the user's directory are\n\tmissing. If so," " aborts execution of the program.\n") print("--clusters:\n\tPrints clusters of files with " "information flows to one another. Requires\n\tthe " "--score option.\n") print("--debug:\n\tPrints additional debug information in " "various code paths to help debug\n\tthe program.\n") print("--disable-plotting:\n\tDo not plot cluster graphs. See " "the --graph option.\n") print("--extensions:\n\tPrints file extensions and MIME type " "associations for this user.\n") print("--frequency:\n\tSets the frequency used by the " "frequent-itemsets algorithm in the\n\t--related-files " "post-analysis. Requires the --related-files option.\n") print("--graph:\n\tFind communities in file/app " "accesses using graph theory methods.\n") print("--help:\n\tPrints this help information and exits.\n") print("--output=<DIR>:\n\tSaves a copy of the simulated " "files, and some information on events\n\trelated to " "them, in a folder created at the <DIR> path.\n") print("--post-analysis=<DIR,DIR,DIR>:\n\t" "Uses the value pointed to" " by --output in order to produce graphs and\n\t" "statistics.\n") print("--quick-pol=Policy:\n\tReplace the default policies " "with this one single Policy.\n") print("--related-files:\n\tMines for files that are frequently" " accessed together by apps. Produces\n\toutput files in" " scoring mode, and an analysis output in post-analysis" "\n\tmode. See also --frequency.\n") print("--score:\n\tCalculates the usability and security " "scores of a number of file access\n\tcontrol policies" ", replayed over the simulated accesses. Prints results" "\n\tand saves them to the output directory.\n") print( "--skip=<Policy,Policy,'graphs'>:\n\tSkip the scoring of " "policies in the lists. If the list contains the word" "\n\t'graphs', skips the general graph computation.\n") sys.exit() elif opt in ('-c', '--check-missing'): __setCheckMissing(True) elif opt in ('-e', '--check-excluded-files'): __setCheckExcludedFiles(True) elif opt in ('-x', '--extensions'): __setPrintExtensions(True) elif opt in ('-d', '--debug'): __setDebug(True) elif opt in ('-r', '--related-files'): __setRelatedFiles(True) elif opt in ('-s', '--score'): __setScore(True) elif opt in ('-p', '--print-clusters', '--clusters'): __setPrintClusters(True) elif opt in ('-g', '--graph-clusters', '--graph'): __setGraph(True) elif opt in ('-t', '--attacks'): __setAttacks(True) elif opt in ('-G', '--disable-plotting'): __setPlottingDisabled(True) elif opt in ('-f', '--frequency'): if not arg: print(USAGE_STRING) sys.exit(2) __setFrequency(arg[1:] if arg[0] == '=' else arg) elif opt in ('-o', '--output-fs', '--output'): if not arg: print(USAGE_STRING) sys.exit(2) __setOutputFs(arg[1:] if arg[0] == '=' else arg) elif opt in ('-u', '--user'): if not arg: print(USAGE_STRING) sys.exit(2) __setUser(arg[1:] if arg[0] == '=' else arg) elif opt in ('-i', '--inode'): if not arg: print(USAGE_STRING) sys.exit(2) try: __opt_inode_query = (arg[1:] if arg[0] == '=' else arg) except (ValueError) as e: print(USAGE_STRING) sys.exit(2) elif opt in ('-a', '--post-analysis'): if not arg: print(USAGE_STRING) sys.exit(2) __opt_post_analysis = (arg[1:] if arg[0] == '=' else arg) elif opt in ('-q', '--quick-pol'): if not arg: print(USAGE_STRING) sys.exit(2) __opt_quick_pol = (arg[1:] if arg[0] == '=' else arg) elif opt in ('-k', '--skip'): if not arg: print(USAGE_STRING) sys.exit(2) __opt_skip = (arg[1:] if arg[0] == '=' else arg) __setSkip(__opt_skip.split(",")) registerTimePrint() if __opt_post_analysis: if relatedFilesEnabled(): tprnt("Starting post-analysis of related files...\n") engine = FrequentFileEngine() engine.processFrequentItemLists(__opt_post_analysis) else: tprnt("Starting post-analysis of usability/security scores...\n") from AnalysisEngine import AnalysisEngine if outputFsEnabled(): engine = AnalysisEngine(inputDir=__opt_post_analysis, outputDir=outputFsEnabled()) else: engine = AnalysisEngine(inputDir=__opt_post_analysis) engine.analyse() sys.exit(0) # Make the application, event and file stores store = ApplicationStore.get() evStore = EventStore.get() fileStore = FileStore.get() initMimeTypes() datapath = getDataPath() # Load up user-related variables userConf = UserConfigLoader.get(path=datapath + USERCONFIGNAME) # Load up and check the SQLite database sql = None tprnt("\nLoading the SQLite database: %s..." % (datapath + DATABASENAME)) try: sql = SqlLoader(datapath + DATABASENAME) except ValueError as e: print("Failed to parse SQL: %s" % e.args[0], file=sys.stderr) sys.exit(-1) if checkMissingEnabled(): tprnt("Checking for missing application identities...") sql.listMissingActors() sql.loadDb(store) sqlAppCount = sql.appCount sqlInstCount = sql.instCount sqlEvCount = sql.eventCount sqlValidEvCount = sql.validEventRatio tprnt("Loaded the SQLite database.") # Load up the PreloadLogger file parser tprnt("\nLoading the PreloadLogger logs in folder: %s..." % datapath) pll = PreloadLoggerLoader(datapath) if checkMissingEnabled(): tprnt("Checking for missing application identities...") pll.listMissingActors() pll.loadDb(store) pllAppCount = pll.appCount pllInstCount = pll.instCount pllEvCount = pll.eventCount pllValidEvCount = pll.validEventRatio tprnt("Loaded the PreloadLogger logs.") # Resolve actor ids in all apps' events tprnt("\nUsing PreloadLogger Applications to resolve interpreters in " "Zeitgeist Applications...") (interpretersAdded, instancesEliminated) = store.resolveInterpreters() tprnt("Resolved interpreter ids in %d Applications, and removed %d " "instances by merging them with another as a result." % (interpretersAdded, instancesEliminated)) # Update events' actor ids in the ApplicationStore, then take them and send # them to the EvnetStore. Finally, sort the EventStore by timestamp. tprnt("\nInserting and sorting all events...") store.sendEventsToStore() evStore.sort() evCount = evStore.getEventCount() tprnt("Sorted all %d events in the event store." % evCount) # Simulate the events to build a file model tprnt("\nSimulating all events to build a file model...") evStore.simulateAllEvents() del sql del pll evStore.sort() tprnt("Simulated all events. %d files initialised." % len(fileStore)) appCount = store.getAppCount() userAppCount = store.getUserAppCount() instCount = len(store) userInstCount = store.getUserInstCount() fileCount = len(fileStore) docCount = fileStore.getUserDocumentCount(userConf.getSetting("HomeDir")) if printExtensions(): exts = set() for f in fileStore: exts.add(f.getExtension()) try: exts.remove(None) except (KeyError): pass tprnt("Info: the following file extensions were found:") for e in sorted(exts): print("\t%s: %s" % (e, mimetypes.guess_type("f.%s" % e, strict=False))) if checkExcludedFilesEnabled(): tprnt("\nPrinting files written and read by instances which wrote" "to excluded directories...") dbgPrintExcludedEvents() import time as t t.sleep(10) # Manage --inode queries if __opt_inode_query: inodes = __opt_inode_query.split(",") for inode in sorted(int(i) for i in inodes): f = fileStore.getFile(inode) tprnt("\nInode queried: %d" % inode) tprnt("Corresponding file: %s\n\t(%s)" % (f.getName(), f)) sys.exit(0) # Print the model as proof of concept if debugEnabled(): tprnt("\nPrinting the file model...\n") fileStore.printFiles(showDeleted=True, showCreationTime=True, showDocumentsOnly=True, userHome=userConf.getSetting("HomeDir"), showDesignatedOnly=False) # Make the filesystem corresponding to the model if outputFsEnabled(): tprnt("\nMaking a copy of the file model at '%s'...\n" % outputFsEnabled()) fileStore.makeFiles(outputDir=outputFsEnabled(), showDeleted=True, showDocumentsOnly=False, userHome=userConf.getSetting("HomeDir"), showDesignatedOnly=False) with open(os.path.join(outputFsEnabled(), "statistics.txt"), "w") as f: msg = "SQL: %d apps; %d instances; %d events; %d%% valid\n" % \ (sqlAppCount, sqlInstCount, sqlEvCount, sqlValidEvCount) msg += "PreloadLogger: %d apps; %d instances; %d events; " \ "%d%% valid\n" % \ (pllAppCount, pllInstCount, pllEvCount, pllValidEvCount) msg += "Simulated: %d apps; %d instances; %d user apps; %d user" \ " instances; %d events; %d files; %d user documents\n" % \ (appCount, instCount, userAppCount, userInstCount, evCount, fileCount, docCount) exclLists = userConf.getDefinedSecurityExclusionLists() for l in exclLists: msg += "Exclusion list '%s' defined.\n" % l print(msg, file=f) # Build a general access graph. if graphEnabled(): skipList = skipEnabled() if skipList and 'graphs' in skipList: tprnt("\nGraphs in skip list, skipping global graph generation.") else: engine = GraphEngine.get() engine.runGraph(policy=None) # Policy engine. Create a policy and run a simulation to score it. if scoreEnabled() or attacksEnabled() or graphEnabled(): engine = PolicyEngine() if __opt_quick_pol: policies = [__opt_quick_pol] polArgs = [None] else: policies = [ CompoundLibraryPolicy, CustomLibraryPolicy, DesignationPolicy, DistantFolderPolicy, FilenamePolicy, FileTypePolicy, FolderPolicy, OneDistantFolderPolicy, OneFolderPolicy, OneLibraryPolicy, UnsecurePolicy, Win10Policy, Win8Policy, HSecurePolicy, HBalancedPolicy, 'HSecureSbPolicy', 'HSecureSbFaPolicy', 'HSecureFaPolicy', 'HBalancedSbPolicy', 'HBalancedSbFaPolicy', 'HBalancedFaPolicy', 'OneDistantFolderSbPolicy', 'OneDistantFolderSbFaPolicy', 'OneDistantFolderFaPolicy', 'HUsableSecuredSbPolicy', 'HUsableSecuredSbFaPolicy', 'HUsableSecuredFaPolicy', 'HBalancedSecuredSbPolicy', 'HBalancedSecuredSbFaPolicy', 'HBalancedSecuredFaPolicy', 'DistantFolderSbPolicy', 'DistantFolderSbFaPolicy', 'DistantFolderFaPolicy', 'LibraryFolderSbPolicy', 'LibraryFolderSbFaPolicy', 'LibraryFolderFaPolicy', 'FileTypeSbPolicy', 'FileTypeSbFaPolicy', 'FileTypeFaPolicy', 'OneFolderSbPolicy', 'OneFolderSbFaPolicy', 'OneFolderFaPolicy', 'FolderSbPolicy', 'FolderSbFaPolicy', 'FolderFaPolicy', 'OneLibrarySbPolicy', 'OneLibrarySbFaPolicy', 'OneLibraryFaPolicy', 'CompoundLibrarySbPolicy', 'CompoundLibrarySbFaPolicy', 'CompoundLibraryFaPolicy', 'CustomLibrarySbPolicy', 'CustomLibrarySbFaPolicy', 'CustomLibraryFaPolicy', ] polArgs = [ None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ] # dict(folders=["~/Downloads", "/tmp"]) skipList = skipEnabled() for (polIdx, polName) in enumerate(policies): pol = None arg = polArgs[polIdx] # Names with certain suffixes are dynamically generated policies. if isinstance(polName, str): if polName.endswith('SbPolicy'): pols = [ getattr(sys.modules[__name__], polName[:-8] + 'Policy'), StickyBitPolicy ] args = [arg, dict(folders=["~", "/media", "/mnt"])] pol = CompositionalPolicy(pols, args, polName) elif polName.endswith('SbFaPolicy'): pols = [ getattr(sys.modules[__name__], polName[:-10] + 'Policy'), StickyBitPolicy, FutureAccessListPolicy ] args = [arg, dict(folders=["~", "/media", "/mnt"]), None] pol = CompositionalPolicy(pols, args, polName) elif polName.endswith('FaPolicy'): pols = [ getattr(sys.modules[__name__], polName[:-8] + 'Policy'), FutureAccessListPolicy ] args = [arg, None] pol = CompositionalPolicy(pols, args, polName) # A normal policy, just invoke it directly. else: polName = getattr(sys.modules[__name__], polName) # Existing policies, with arguments / or normal policies passed as # strings, including via the --quick flag. if not pol: pol = polName(**arg) if arg else polName() tprnt("\nRunning %s..." % pol.name) if skipList and pol.name in skipList: tprnt("%s is in skip list, skipping." % pol.name) continue engine.runPolicy(pol, outputDir=outputFsEnabled(), printClusters=printClustersEnabled()) if pol.name == "FileTypePolicy" and checkMissingEnabled(): pol.abortIfUnsupportedExtensions() if attacksEnabled(): tprnt("Simulating attacks on %s..." % pol.name) sim = AttackSimulator(seed=0) sim.runAttacks(pol, outputDir=outputFsEnabled() or "/tmp/") del pol # Calculate frequently co-accessed files: if relatedFilesEnabled(): engine = FrequentFileEngine() tprnt("\nMining for frequently co-accessed file types...") engine.mineFileTypes()