class EventStoreTestCase(unittest.TestCase): '''Test the python event store interface.''' def setUp(self): self.filename = 'example.root' self.assertTrue( os.path.isfile(self.filename) ) self.store = EventStore(self.filename) def test_eventloop(self): '''Test that - the store contains events, - the length of the store is the number of entries, - one can loop on the events. ''' self.assertTrue( self.store.getEntries() >= 0 ) self.assertEqual( self.store.getEntries(), len(self.store) ) for iev, event in enumerate(self.store): self.assertTrue( True ) def test_read_particles(self): '''Test that - the GenParticle objects can be used - the processing is faster than 100 Hz. ''' start = datetime.now() for iev, event in enumerate(self.store): genptcs = event.get('GenParticle') self.assertTrue(len(genptcs)>0.) for ptc in genptcs: self.assertTrue(ptc.Core().P4.Pz!=0.) stop = datetime.now() time_per_event = float( (stop - start).seconds )/ len(self.store) self.assertLess(time_per_event, 1e-2) # 3e-4 on Colin's mac def test_direct_navigation(self): '''Test that - one can move to a given event ''' event0 = self.store[0] # event is of the store class self.assertEqual( event0.__class__, self.store.__class__) def test_collections(self): '''test that an existing collection can be read, and that a non existing collection gives None. ''' evinfo = self.store.get("EventInfo") self.assertTrue( len(evinfo)>0 ) particles = self.store.get("CollectionNotThere") self.assertFalse(particles is None)
def sendEventsToStore(self): """Send this app's Events to the EventStore, and clear them.""" # Get the EventStore from EventStore import EventStore eventStore = EventStore.get() # Send each Event, after updating it to point to the updated self. for event in self.events: event.actor = self eventStore.append(event) # Clear events, to save RAM. self.clearEvents()
class EventStoreTestCase(unittest.TestCase): def setUp(self): self.filename = 'example.root' self.assertTrue( os.path.isfile(self.filename) ) self.store = EventStore(self.filename) def test_eventloop(self): self.assertTrue( self.store.getEntries() >= 0 ) self.assertEqual( self.store.getEntries(), len(self.store) ) for iev, event in enumerate(self.store): self.assertTrue( True ) if iev>5: break event0 = self.store[0] self.assertEqual( event0.__class__, self.store.__class__) def test_collections(self): evinfo = self.store.get("clusters") self.assertTrue( len(evinfo)>0 ) particles = self.store.get("CollectionNotThere") self.assertFalse(particles)
class EventStoreTestCase(unittest.TestCase): def setUp(self): self.filename = "example.root" self.assertTrue(os.path.isfile(self.filename)) self.store = EventStore([self.filename]) def test_eventloop(self): self.assertTrue(self.store.getEntries() >= 0) self.assertEqual(self.store.getEntries(), len(self.store)) for iev, event in enumerate(self.store): self.assertTrue(True) if iev > 5: break def test_navigation(self): event0 = self.store[0] self.assertEqual(event0.__class__, self.store.__class__) def test_collections(self): evinfo = self.store.get("info") self.assertTrue(len(evinfo) > 0) particles = self.store.get("CollectionNotThere") self.assertFalse(particles) def test_chain(self): self.store = EventStore([self.filename, self.filename]) rootfile = TFile(self.filename) events = rootfile.Get("events") numbers = [] for iev, event in enumerate(self.store): evinfo = self.store.get("info") numbers.append(evinfo[0].Number()) self.assertEqual(iev + 1, 2 * events.GetEntries()) # testing that numbers is [0, .. 1999, 0, .. 1999] self.assertEqual(numbers, range(events.GetEntries()) * 2) # trying to go to an event in the second file self.assertRaises(ValueError, self.store.__getitem__, events.GetEntries() + 1)
def setUp(self): self.eventStore = EventStore.get() self.appStore = ApplicationStore.get() self.fileFactory = FileFactory.get() self.fileStore = FileStore.get() self.userConf = UserConfigLoader.get("user.ini") self.ar1 = Application("ristretto.desktop", pid=21, tstart=1, tend=2000) self.ar2 = Application("ristretto.desktop", pid=22, tstart=2600, tend=2900) self.ag1 = Application("gimp.desktop", pid=23, tstart=1, tend=4000) self.ag2 = Application("gimp.desktop", pid=24, tstart=4500, tend=4590) self.appStore.insert(self.ar1) self.appStore.insert(self.ar2) self.appStore.insert(self.ag1) self.appStore.insert(self.ag2) # Insert a file that will bridge r1 and g1. s2 = "open64|/home/user/Images/Picture.jpg|fd 4: with flag 524288, e0|" e2 = Event(actor=self.ag1, time=10, syscallStr=s2) self.eventStore.append(e2) e2b = Event(actor=self.ar1, time=12, syscallStr=s2) self.eventStore.append(e2b) # Insert a file that will bridge r1 and r2. s3 = "open64|/home/user/Images/Photo.jpg|fd 10: with flag 524288, e0|" e3 = Event(actor=self.ar1, time=10, syscallStr=s3) self.eventStore.append(e3) e3b = Event(actor=self.ar2, time=2710, syscallStr=s3) self.eventStore.append(e3b) # Insert a file that will bridge g1 and g2. s4 = "open64|/home/user/Images/Art.xcf|fd 10: with flag 524288, e0|" e4 = Event(actor=self.ag1, time=10, syscallStr=s4) self.eventStore.append(e4) e4b = Event(actor=self.ag2, time=4540, syscallStr=s4) self.eventStore.append(e4b) # Simulate. self.eventStore.simulateAllEvents()
def setUp(self): self.eventStore = EventStore.get() self.appStore = ApplicationStore.get() self.fileFactory = FileFactory.get() self.userConf = UserConfigLoader.get("user.ini") self.engine = PolicyEngine() self.ar1 = Application("ristretto.desktop", pid=21, tstart=1, tend=2000) self.ar2 = Application("ristretto.desktop", pid=22, tstart=2600, tend=2900) self.ag1 = Application("gimp.desktop", pid=23, tstart=1, tend=4000) self.ag2 = Application("gimp.desktop", pid=24, tstart=4500, tend=4590) self.appStore.insert(self.ar1) self.appStore.insert(self.ar2) self.appStore.insert(self.ag1) self.appStore.insert(self.ag2)
import unittest from EventStore import EventStore if __name__ == '__main__': filename = 'example.root' store = EventStore([filename]) for i, event in enumerate(store): if i%1000 == 0: print 'reading event', i evinfo = store.get("info")[0] clusters = store.get("clusters") for cluster in clusters: for ihit in range(cluster.Hits_size()): hit = cluster.Hits(ihit) print ' Referenced hit has an energy of', hit.energy()
def setUp(self): self.store = EventStore.get()
def setUp(self): self.userConf = UserConfigLoader.get("user.ini") self.appStore = ApplicationStore.get() self.eventStore = EventStore.get() self.fileStore = FileStore.get() self.fileFactory = FileFactory.get() self.sim = AttackSimulator() self.a1 = Application("ristretto.desktop", pid=1, tstart=1, tend=2000) self.a2 = Application("firefox.desktop", pid=2, tstart=1, tend=2000) self.a3 = Application("ristretto.desktop", pid=3, tstart=3000, tend=6000) self.ac = Application("catfish.desktop", pid=100, tstart=1, tend=2900) self.appStore.insert(self.a1) self.appStore.insert(self.a2) self.appStore.insert(self.a3) self.appStore.insert(self.ac) self.p001 = "/home/user/.cache/firefox/file" s001 = "open64|%s|fd 10: with flag 524288, e0|" % self.p001 e001 = Event(actor=self.a1, time=10, syscallStr=s001) e001.evflags &= ~EventFileFlags.designation # not by designation self.eventStore.append(e001) e001b = Event(actor=self.a2, time=11, syscallStr=s001) e001b.evflags &= ~EventFileFlags.designation # not by designation self.eventStore.append(e001b) self.p002 = "/home/user/Images/picture.jpg" s002 = "open64|%s|fd 10: with flag 524288, e0|" % self.p002 e002 = Event(actor=self.a1, time=12, syscallStr=s002) e002.evflags &= ~EventFileFlags.designation # not by designation self.eventStore.append(e002) e002b = Event(actor=self.ac, time=30, syscallStr=s002) e002b.evflags &= ~EventFileFlags.designation # not by designation self.eventStore.append(e002b) self.p003 = "/home/user/Downloads/logo.jpg" s003 = "open64|%s|fd 10: with flag 524288, e0|" % self.p003 e003 = Event(actor=self.a1, time=13, syscallStr=s003) e003.evflags |= EventFileFlags.designation # this event by designation self.eventStore.append(e003) e003b = Event(actor=self.a3, time=3003, syscallStr=s003) e003b.evflags &= ~EventFileFlags.designation # not by designation self.eventStore.append(e003b) self.p004 = "/home/user/Downloads/logo.png" s004 = "open64|%s|fd 10: with flag 64, e0|" % self.p004 e004 = Event(actor=self.a1, time=14, syscallStr=s004) e004.evflags &= ~EventFileFlags.designation # not by designation self.eventStore.append(e004) self.p005 = "/home/user/Dropbox/Photos/holidays.jpg" s005 = "open64|%s|fd 10: with flag 524288, e0|" % self.p005 e005 = Event(actor=self.a1, time=15, syscallStr=s005) e005.evflags &= ~EventFileFlags.designation # not by designation self.eventStore.append(e005) e005b = Event(actor=self.a1, time=1005, syscallStr=s005) e005b.evflags &= ~EventFileFlags.designation # not by designation self.eventStore.append(e005b) self.p006 = "/home/user/Images/random.txt" s006 = "open64|%s|fd 10: with flag 524288, e0|" % self.p006 e006 = Event(actor=self.a1, time=16, syscallStr=s006) e006.evflags &= ~EventFileFlags.designation # not by designation self.eventStore.append(e006) self.p007 = "/home/user/Images/file.jpg" s007 = "open64|%s|fd 10: with flag 524288, e0|" % self.p007 e007 = Event(actor=self.a1, time=17, syscallStr=s007) e007.evflags |= EventFileFlags.designation # this event by designation self.eventStore.append(e007) self.p008 = "/home/user/Images/other.foo" s008 = "open64|%s|fd 10: with flag 64, e0|" % self.p008 e008 = Event(actor=self.a1, time=18, syscallStr=s008) e008.evflags &= ~EventFileFlags.designation # not by designation self.eventStore.append(e008) self.p009 = "/home/user/Downloads/unknown.data" s009 = "open64|%s|fd 10: with flag 64, e0|" % self.p009 # e009 = Event(actor=self.a1, time=18, syscallStr=s009) # e009.evflags |= EventFileFlags.designation # this event by designation # self.eventStore.append(e009) e009b = Event(actor=self.a3, time=3009, syscallStr=s009) e009b.evflags &= ~EventFileFlags.designation # not by designation self.eventStore.append(e009b) self.p010 = "/home/user/Dropbox/Photos/holidays.metadata" s010 = "open64|%s|fd 10: with flag 524288, e0|" % self.p010 e010 = Event(actor=self.a1, time=20, syscallStr=s010) e010.evflags &= ~EventFileFlags.designation # not by designation self.eventStore.append(e010) self.p011 = "/home/user/Dropbox/Photos/holidays.evenmoremetadata" s011 = "open64|%s|fd 11: with flag 524288, e0|" % self.p011 e011 = Event(actor=self.a3, time=3020, syscallStr=s011) e011.evflags &= ~EventFileFlags.designation # not by designation self.eventStore.append(e011) self.eventStore.simulateAllEvents() self.acCache = AccessListCache.get() self.lookUps = dict() self.allowedCache = dict()
class EventStoreTestCase(unittest.TestCase): def setUp(self): self.filename = str('example.root') self.assertTrue(os.path.isfile(self.filename)) self.store = EventStore([self.filename]) def test_eventloop(self): self.assertTrue(len(self.store) >= 0) self.assertEqual(self.store.current_store.getEntries(), len(self.store)) for iev, event in enumerate(self.store): self.assertTrue(True) if iev > 5: break def test_navigation(self): event0 = self.store[0] self.assertEqual(event0.__class__, self.store.__class__) def test_collections(self): evinfo = self.store.get("info") self.assertTrue(len(evinfo) > 0) particles = self.store.get("CollectionNotThere") self.assertFalse(particles) def test_read_only(self): hits = self.store.get("hits") # testing that one can't modify attributes in # read-only pods self.assertEqual(hits[0].energy(), 23.) hits[0].energy(10) self.assertEqual(hits[0].energy(), 10) # oops def test_one_to_many(self): clusters = self.store.get("clusters") ref_hits = [] # testing that cluster hits can be accessed and make sense for cluster in clusters: sume = 0 for ihit in range(cluster.Hits_size()): hit = cluster.Hits(ihit) ref_hits.append(hit) sume += hit.energy() self.assertEqual(cluster.energy(), sume) hits = self.store.get("hits") # testing that the hits stored as a one to many relation # in the cluster can be found in the hit collection for hit in ref_hits: self.assertTrue(hit in hits) def test_relation_range(self): """Test that the RelationRange functionality is also accessible in python""" clusters = self.store.get("clusters") hits = self.store.get("hits") for cluster in clusters: sume = 0 for hit in cluster.Hits(): self.assertTrue(hit in hits) sume += hit.energy() self.assertEqual(cluster.energy(), sume) def test_hash(self): clusters = self.store.get("clusters") ref_hits = [] # testing that cluster hits can be accessed and make sense for cluster in clusters: sume = 0 for ihit in range(cluster.Hits_size()): hit = cluster.Hits(ihit) ref_hits.append(hit) sume += hit.energy() self.assertEqual(cluster.energy(), sume) hits = self.store.get("hits") if hits[0] == ref_hits[0]: self.assertEqual(hits[0].getObjectID().index, ref_hits[0].getObjectID().index) self.assertEqual(hits[0].getObjectID().collectionID, ref_hits[0].getObjectID().collectionID) self.assertEqual(hits[0].getObjectID(), ref_hits[0].getObjectID()) # testing that the hits stored as a one to many relation # import pdb; pdb.set_trace() def test_chain(self): self.store = EventStore([self.filename, self.filename]) rootfile = TFile(self.filename) events = rootfile.Get(str('events')) numbers = [] for iev, event in enumerate(self.store): evinfo = self.store.get("info") numbers.append(evinfo[0].Number()) self.assertEqual(iev + 1, 2 * events.GetEntries()) # testing that numbers is [0, .. 1999, 0, .. 1999] self.assertEqual(numbers, list(range(events.GetEntries())) * 2) # trying to go to an event beyond the last one self.assertRaises(ValueError, self.store.__getitem__, 4001) # this is in the first event in the second file, # so its event number should be 0. self.assertEqual(self.store[2000].get("info")[0].Number(), 0) def test_context_managers(self): with EventStore([self.filename]) as store: self.assertTrue(len(store) >= 0) self.assertTrue(store.isValid()) def test_no_file(self): '''Test that non-accessible files are gracefully handled.''' with self.assertRaises(ValueError): self.store = EventStore('foo.root')
class EventStoreTestCase(unittest.TestCase): def setUp(self): self.filename = 'example1.root' self.assertTrue(os.path.isfile(self.filename)) self.store = EventStore([self.filename]) def test_eventloop(self): self.assertTrue(len(self.store) >= 0) self.assertEqual(self.store.current_store.getEntries(), len(self.store)) for iev, event in enumerate(self.store): self.assertTrue(True) if iev > 5: break def test_navigation(self): event0 = self.store[0] self.assertEqual(event0.__class__, self.store.__class__) def test_collections(self): evinfo = self.store.get("info") self.assertTrue(len(evinfo) > 0) particles = self.store.get("CollectionNotThere") self.assertFalse(particles) def test_read_only(self): hits = self.store.get("hits") # testing that one can't modify attributes in # read-only pods self.assertEqual(hits[0].energy(), 23.) hits[0].energy(10) self.assertEqual(hits[0].energy(), 10) # oops # self.assertEqual(type(hits[0]), ConstExampleHit) # should be True def test_one_to_many(self): clusters = self.store.get("clusters") ref_hits = [] # testing that cluster hits can be accessed and make sense for cluster in clusters: sume = 0 for ihit in range(cluster.Hits_size()): hit = cluster.Hits(ihit) ref_hits.append(hit) sume += hit.energy() self.assertEqual(cluster.energy(), sume) hits = self.store.get("hits") # testing that the hits stored as a one to many relation # in the cluster can be found in the hit collection for hit in ref_hits: self.assertTrue(hit in hits) def test_hash(self): clusters = self.store.get("clusters") ref_hits = [] # testing that cluster hits can be accessed and make sense for cluster in clusters: sume = 0 for ihit in range(cluster.Hits_size()): hit = cluster.Hits(ihit) ref_hits.append(hit) sume += hit.energy() self.assertEqual(cluster.energy(), sume) hits = self.store.get("hits") if hits[0] == ref_hits[0]: self.assertEqual(hits[0].getObjectID().index, ref_hits[0].getObjectID().index) self.assertEqual(hits[0].getObjectID().collectionID, ref_hits[0].getObjectID().collectionID) self.assertEqual(hits[0].getObjectID(), ref_hits[0].getObjectID()) # testing that the hits stored as a one to many relation # import pdb; pdb.set_trace() def test_chain(self): self.store = EventStore([self.filename, self.filename]) rootfile = TFile(self.filename) events = rootfile.Get('events') numbers = [] for iev, event in enumerate(self.store): evinfo = self.store.get("info") numbers.append(evinfo[0].Number()) self.assertEqual(iev+1, 2*events.GetEntries()) # testing that numbers is [0, .. 1999, 0, .. 1999] self.assertEqual(numbers, range(events.GetEntries())*2) # trying to go to an event beyond the last one self.assertRaises(ValueError, self.store.__getitem__, 4001) # this is in the first event in the second file, # so its event number should be 0. self.assertEqual(self.store[2000].get("info")[0].Number(), 0) def test_context_managers(self): with EventStore([self.filename]) as store: self.assertTrue(len(store) >= 0) self.assertTrue(store.isValid()) def test_no_file(self): '''Test that non-accessible files are gracefully handled.''' with self.assertRaises(ValueError): self.store = EventStore('foo.root')
import unittest from EventStore import EventStore if __name__ == '__main__': filename = 'example.root' store = EventStore([filename]) for i, event in enumerate(store): if i % 1000 == 0: print 'reading event', i evinfo = store.get("info")[0] clusters = store.get("clusters") for cluster in clusters: for ihit in range(cluster.Hits_size()): hit = cluster.Hits(ihit) print ' Referenced hit has an energy of', hit.energy()
def setUp(self): self.eventStore = EventStore.get() self.appStore = ApplicationStore.get() self.userConf = UserConfigLoader.get("user.ini") self.fileFactory = FileFactory.get()
def main(argv): __opt_inode_query = None __opt_post_analysis = None __opt_quick_pol = None # Parse command-line parameters try: (opts, args) = getopt.getopt(argv, "hta:cedf:o:q:sk:rpgGi:u:x", [ "help", "attacks", "post-analysis=", "check-missing", "check-excluded-files", "debug", "frequency", "inode", "extensions", "related-files", "output=", "output-fs=", "score", "quick-pol=", "skip=", "user", "clusters", "print-clusters", "graph", "graph-clusters", "disable-plotting" ]) except (getopt.GetoptError): print(USAGE_STRING) sys.exit(2) else: for opt, arg in opts: if opt in ('-h', '--help'): print(USAGE_STRING + "\n\n\n\n") print("--attacks:\n\tSimulates attacks and reports " "on proportions of infected files and apps.\n") print("--check-excluded-files:\n\tPrints the lists of files " "accessed by apps that also wrote to excluded\n\tfiles," " then aborts execution of the program.\n") print("--check-missing:\n\tChecks whether some Desktop IDs " "for apps in the user's directory are\n\tmissing. If so," " aborts execution of the program.\n") print("--clusters:\n\tPrints clusters of files with " "information flows to one another. Requires\n\tthe " "--score option.\n") print("--debug:\n\tPrints additional debug information in " "various code paths to help debug\n\tthe program.\n") print("--disable-plotting:\n\tDo not plot cluster graphs. See " "the --graph option.\n") print("--extensions:\n\tPrints file extensions and MIME type " "associations for this user.\n") print("--frequency:\n\tSets the frequency used by the " "frequent-itemsets algorithm in the\n\t--related-files " "post-analysis. Requires the --related-files option.\n") print("--graph:\n\tFind communities in file/app " "accesses using graph theory methods.\n") print("--help:\n\tPrints this help information and exits.\n") print("--output=<DIR>:\n\tSaves a copy of the simulated " "files, and some information on events\n\trelated to " "them, in a folder created at the <DIR> path.\n") print("--post-analysis=<DIR,DIR,DIR>:\n\t" "Uses the value pointed to" " by --output in order to produce graphs and\n\t" "statistics.\n") print("--quick-pol=Policy:\n\tReplace the default policies " "with this one single Policy.\n") print("--related-files:\n\tMines for files that are frequently" " accessed together by apps. Produces\n\toutput files in" " scoring mode, and an analysis output in post-analysis" "\n\tmode. See also --frequency.\n") print("--score:\n\tCalculates the usability and security " "scores of a number of file access\n\tcontrol policies" ", replayed over the simulated accesses. Prints results" "\n\tand saves them to the output directory.\n") print( "--skip=<Policy,Policy,'graphs'>:\n\tSkip the scoring of " "policies in the lists. If the list contains the word" "\n\t'graphs', skips the general graph computation.\n") sys.exit() elif opt in ('-c', '--check-missing'): __setCheckMissing(True) elif opt in ('-e', '--check-excluded-files'): __setCheckExcludedFiles(True) elif opt in ('-x', '--extensions'): __setPrintExtensions(True) elif opt in ('-d', '--debug'): __setDebug(True) elif opt in ('-r', '--related-files'): __setRelatedFiles(True) elif opt in ('-s', '--score'): __setScore(True) elif opt in ('-p', '--print-clusters', '--clusters'): __setPrintClusters(True) elif opt in ('-g', '--graph-clusters', '--graph'): __setGraph(True) elif opt in ('-t', '--attacks'): __setAttacks(True) elif opt in ('-G', '--disable-plotting'): __setPlottingDisabled(True) elif opt in ('-f', '--frequency'): if not arg: print(USAGE_STRING) sys.exit(2) __setFrequency(arg[1:] if arg[0] == '=' else arg) elif opt in ('-o', '--output-fs', '--output'): if not arg: print(USAGE_STRING) sys.exit(2) __setOutputFs(arg[1:] if arg[0] == '=' else arg) elif opt in ('-u', '--user'): if not arg: print(USAGE_STRING) sys.exit(2) __setUser(arg[1:] if arg[0] == '=' else arg) elif opt in ('-i', '--inode'): if not arg: print(USAGE_STRING) sys.exit(2) try: __opt_inode_query = (arg[1:] if arg[0] == '=' else arg) except (ValueError) as e: print(USAGE_STRING) sys.exit(2) elif opt in ('-a', '--post-analysis'): if not arg: print(USAGE_STRING) sys.exit(2) __opt_post_analysis = (arg[1:] if arg[0] == '=' else arg) elif opt in ('-q', '--quick-pol'): if not arg: print(USAGE_STRING) sys.exit(2) __opt_quick_pol = (arg[1:] if arg[0] == '=' else arg) elif opt in ('-k', '--skip'): if not arg: print(USAGE_STRING) sys.exit(2) __opt_skip = (arg[1:] if arg[0] == '=' else arg) __setSkip(__opt_skip.split(",")) registerTimePrint() if __opt_post_analysis: if relatedFilesEnabled(): tprnt("Starting post-analysis of related files...\n") engine = FrequentFileEngine() engine.processFrequentItemLists(__opt_post_analysis) else: tprnt("Starting post-analysis of usability/security scores...\n") from AnalysisEngine import AnalysisEngine if outputFsEnabled(): engine = AnalysisEngine(inputDir=__opt_post_analysis, outputDir=outputFsEnabled()) else: engine = AnalysisEngine(inputDir=__opt_post_analysis) engine.analyse() sys.exit(0) # Make the application, event and file stores store = ApplicationStore.get() evStore = EventStore.get() fileStore = FileStore.get() initMimeTypes() datapath = getDataPath() # Load up user-related variables userConf = UserConfigLoader.get(path=datapath + USERCONFIGNAME) # Load up and check the SQLite database sql = None tprnt("\nLoading the SQLite database: %s..." % (datapath + DATABASENAME)) try: sql = SqlLoader(datapath + DATABASENAME) except ValueError as e: print("Failed to parse SQL: %s" % e.args[0], file=sys.stderr) sys.exit(-1) if checkMissingEnabled(): tprnt("Checking for missing application identities...") sql.listMissingActors() sql.loadDb(store) sqlAppCount = sql.appCount sqlInstCount = sql.instCount sqlEvCount = sql.eventCount sqlValidEvCount = sql.validEventRatio tprnt("Loaded the SQLite database.") # Load up the PreloadLogger file parser tprnt("\nLoading the PreloadLogger logs in folder: %s..." % datapath) pll = PreloadLoggerLoader(datapath) if checkMissingEnabled(): tprnt("Checking for missing application identities...") pll.listMissingActors() pll.loadDb(store) pllAppCount = pll.appCount pllInstCount = pll.instCount pllEvCount = pll.eventCount pllValidEvCount = pll.validEventRatio tprnt("Loaded the PreloadLogger logs.") # Resolve actor ids in all apps' events tprnt("\nUsing PreloadLogger Applications to resolve interpreters in " "Zeitgeist Applications...") (interpretersAdded, instancesEliminated) = store.resolveInterpreters() tprnt("Resolved interpreter ids in %d Applications, and removed %d " "instances by merging them with another as a result." % (interpretersAdded, instancesEliminated)) # Update events' actor ids in the ApplicationStore, then take them and send # them to the EvnetStore. Finally, sort the EventStore by timestamp. tprnt("\nInserting and sorting all events...") store.sendEventsToStore() evStore.sort() evCount = evStore.getEventCount() tprnt("Sorted all %d events in the event store." % evCount) # Simulate the events to build a file model tprnt("\nSimulating all events to build a file model...") evStore.simulateAllEvents() del sql del pll evStore.sort() tprnt("Simulated all events. %d files initialised." % len(fileStore)) appCount = store.getAppCount() userAppCount = store.getUserAppCount() instCount = len(store) userInstCount = store.getUserInstCount() fileCount = len(fileStore) docCount = fileStore.getUserDocumentCount(userConf.getSetting("HomeDir")) if printExtensions(): exts = set() for f in fileStore: exts.add(f.getExtension()) try: exts.remove(None) except (KeyError): pass tprnt("Info: the following file extensions were found:") for e in sorted(exts): print("\t%s: %s" % (e, mimetypes.guess_type("f.%s" % e, strict=False))) if checkExcludedFilesEnabled(): tprnt("\nPrinting files written and read by instances which wrote" "to excluded directories...") dbgPrintExcludedEvents() import time as t t.sleep(10) # Manage --inode queries if __opt_inode_query: inodes = __opt_inode_query.split(",") for inode in sorted(int(i) for i in inodes): f = fileStore.getFile(inode) tprnt("\nInode queried: %d" % inode) tprnt("Corresponding file: %s\n\t(%s)" % (f.getName(), f)) sys.exit(0) # Print the model as proof of concept if debugEnabled(): tprnt("\nPrinting the file model...\n") fileStore.printFiles(showDeleted=True, showCreationTime=True, showDocumentsOnly=True, userHome=userConf.getSetting("HomeDir"), showDesignatedOnly=False) # Make the filesystem corresponding to the model if outputFsEnabled(): tprnt("\nMaking a copy of the file model at '%s'...\n" % outputFsEnabled()) fileStore.makeFiles(outputDir=outputFsEnabled(), showDeleted=True, showDocumentsOnly=False, userHome=userConf.getSetting("HomeDir"), showDesignatedOnly=False) with open(os.path.join(outputFsEnabled(), "statistics.txt"), "w") as f: msg = "SQL: %d apps; %d instances; %d events; %d%% valid\n" % \ (sqlAppCount, sqlInstCount, sqlEvCount, sqlValidEvCount) msg += "PreloadLogger: %d apps; %d instances; %d events; " \ "%d%% valid\n" % \ (pllAppCount, pllInstCount, pllEvCount, pllValidEvCount) msg += "Simulated: %d apps; %d instances; %d user apps; %d user" \ " instances; %d events; %d files; %d user documents\n" % \ (appCount, instCount, userAppCount, userInstCount, evCount, fileCount, docCount) exclLists = userConf.getDefinedSecurityExclusionLists() for l in exclLists: msg += "Exclusion list '%s' defined.\n" % l print(msg, file=f) # Build a general access graph. if graphEnabled(): skipList = skipEnabled() if skipList and 'graphs' in skipList: tprnt("\nGraphs in skip list, skipping global graph generation.") else: engine = GraphEngine.get() engine.runGraph(policy=None) # Policy engine. Create a policy and run a simulation to score it. if scoreEnabled() or attacksEnabled() or graphEnabled(): engine = PolicyEngine() if __opt_quick_pol: policies = [__opt_quick_pol] polArgs = [None] else: policies = [ CompoundLibraryPolicy, CustomLibraryPolicy, DesignationPolicy, DistantFolderPolicy, FilenamePolicy, FileTypePolicy, FolderPolicy, OneDistantFolderPolicy, OneFolderPolicy, OneLibraryPolicy, UnsecurePolicy, Win10Policy, Win8Policy, HSecurePolicy, HBalancedPolicy, 'HSecureSbPolicy', 'HSecureSbFaPolicy', 'HSecureFaPolicy', 'HBalancedSbPolicy', 'HBalancedSbFaPolicy', 'HBalancedFaPolicy', 'OneDistantFolderSbPolicy', 'OneDistantFolderSbFaPolicy', 'OneDistantFolderFaPolicy', 'HUsableSecuredSbPolicy', 'HUsableSecuredSbFaPolicy', 'HUsableSecuredFaPolicy', 'HBalancedSecuredSbPolicy', 'HBalancedSecuredSbFaPolicy', 'HBalancedSecuredFaPolicy', 'DistantFolderSbPolicy', 'DistantFolderSbFaPolicy', 'DistantFolderFaPolicy', 'LibraryFolderSbPolicy', 'LibraryFolderSbFaPolicy', 'LibraryFolderFaPolicy', 'FileTypeSbPolicy', 'FileTypeSbFaPolicy', 'FileTypeFaPolicy', 'OneFolderSbPolicy', 'OneFolderSbFaPolicy', 'OneFolderFaPolicy', 'FolderSbPolicy', 'FolderSbFaPolicy', 'FolderFaPolicy', 'OneLibrarySbPolicy', 'OneLibrarySbFaPolicy', 'OneLibraryFaPolicy', 'CompoundLibrarySbPolicy', 'CompoundLibrarySbFaPolicy', 'CompoundLibraryFaPolicy', 'CustomLibrarySbPolicy', 'CustomLibrarySbFaPolicy', 'CustomLibraryFaPolicy', ] polArgs = [ None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, ] # dict(folders=["~/Downloads", "/tmp"]) skipList = skipEnabled() for (polIdx, polName) in enumerate(policies): pol = None arg = polArgs[polIdx] # Names with certain suffixes are dynamically generated policies. if isinstance(polName, str): if polName.endswith('SbPolicy'): pols = [ getattr(sys.modules[__name__], polName[:-8] + 'Policy'), StickyBitPolicy ] args = [arg, dict(folders=["~", "/media", "/mnt"])] pol = CompositionalPolicy(pols, args, polName) elif polName.endswith('SbFaPolicy'): pols = [ getattr(sys.modules[__name__], polName[:-10] + 'Policy'), StickyBitPolicy, FutureAccessListPolicy ] args = [arg, dict(folders=["~", "/media", "/mnt"]), None] pol = CompositionalPolicy(pols, args, polName) elif polName.endswith('FaPolicy'): pols = [ getattr(sys.modules[__name__], polName[:-8] + 'Policy'), FutureAccessListPolicy ] args = [arg, None] pol = CompositionalPolicy(pols, args, polName) # A normal policy, just invoke it directly. else: polName = getattr(sys.modules[__name__], polName) # Existing policies, with arguments / or normal policies passed as # strings, including via the --quick flag. if not pol: pol = polName(**arg) if arg else polName() tprnt("\nRunning %s..." % pol.name) if skipList and pol.name in skipList: tprnt("%s is in skip list, skipping." % pol.name) continue engine.runPolicy(pol, outputDir=outputFsEnabled(), printClusters=printClustersEnabled()) if pol.name == "FileTypePolicy" and checkMissingEnabled(): pol.abortIfUnsupportedExtensions() if attacksEnabled(): tprnt("Simulating attacks on %s..." % pol.name) sim = AttackSimulator(seed=0) sim.runAttacks(pol, outputDir=outputFsEnabled() or "/tmp/") del pol # Calculate frequently co-accessed files: if relatedFilesEnabled(): engine = FrequentFileEngine() tprnt("\nMining for frequently co-accessed file types...") engine.mineFileTypes()