class EventStoreTestCase(unittest.TestCase):
'''Test the python event store interface.'''
def setUp(self):
self.filename = 'example.root'
self.assertTrue( os.path.isfile(self.filename) )
self.store = EventStore(self.filename)
def test_eventloop(self):
'''Test that
- the store contains events,
- the length of the store is the number of entries,
- one can loop on the events.
'''
self.assertTrue( self.store.getEntries() >= 0 )
self.assertEqual( self.store.getEntries(), len(self.store) )
for iev, event in enumerate(self.store):
self.assertTrue( True )
def test_read_particles(self):
'''Test that
- the GenParticle objects can be used
- the processing is faster than 100 Hz.
'''
start = datetime.now()
for iev, event in enumerate(self.store):
genptcs = event.get('GenParticle')
self.assertTrue(len(genptcs)>0.)
for ptc in genptcs:
self.assertTrue(ptc.Core().P4.Pz!=0.)
stop = datetime.now()
time_per_event = float( (stop - start).seconds )/ len(self.store)
self.assertLess(time_per_event, 1e-2) # 3e-4 on Colin's mac
def test_direct_navigation(self):
'''Test that
- one can move to a given event
'''
event0 = self.store[0]
# event is of the store class
self.assertEqual( event0.__class__, self.store.__class__)
def test_collections(self):
'''test that an existing collection can be read,
and that a non existing collection gives None.
'''
evinfo = self.store.get("EventInfo")
self.assertTrue( len(evinfo)>0 )
particles = self.store.get("CollectionNotThere")
self.assertFalse(particles is None)
def sendEventsToStore(self):
"""Send this app's Events to the EventStore, and clear them."""
# Get the EventStore
from EventStore import EventStore
eventStore = EventStore.get()
# Send each Event, after updating it to point to the updated self.
for event in self.events:
event.actor = self
eventStore.append(event)
# Clear events, to save RAM.
self.clearEvents()
class EventStoreTestCase(unittest.TestCase):
def setUp(self):
self.filename = 'example.root'
self.assertTrue( os.path.isfile(self.filename) )
self.store = EventStore(self.filename)
def test_eventloop(self):
self.assertTrue( self.store.getEntries() >= 0 )
self.assertEqual( self.store.getEntries(), len(self.store) )
for iev, event in enumerate(self.store):
self.assertTrue( True )
if iev>5:
break
event0 = self.store[0]
self.assertEqual( event0.__class__, self.store.__class__)
def test_collections(self):
evinfo = self.store.get("clusters")
self.assertTrue( len(evinfo)>0 )
particles = self.store.get("CollectionNotThere")
self.assertFalse(particles)
class EventStoreTestCase(unittest.TestCase):
def setUp(self):
self.filename = "example.root"
self.assertTrue(os.path.isfile(self.filename))
self.store = EventStore([self.filename])
def test_eventloop(self):
self.assertTrue(self.store.getEntries() >= 0)
self.assertEqual(self.store.getEntries(), len(self.store))
for iev, event in enumerate(self.store):
self.assertTrue(True)
if iev > 5:
break
def test_navigation(self):
event0 = self.store[0]
self.assertEqual(event0.__class__, self.store.__class__)
def test_collections(self):
evinfo = self.store.get("info")
self.assertTrue(len(evinfo) > 0)
particles = self.store.get("CollectionNotThere")
self.assertFalse(particles)
def test_chain(self):
self.store = EventStore([self.filename, self.filename])
rootfile = TFile(self.filename)
events = rootfile.Get("events")
numbers = []
for iev, event in enumerate(self.store):
evinfo = self.store.get("info")
numbers.append(evinfo[0].Number())
self.assertEqual(iev + 1, 2 * events.GetEntries())
# testing that numbers is [0, .. 1999, 0, .. 1999]
self.assertEqual(numbers, range(events.GetEntries()) * 2)
# trying to go to an event in the second file
self.assertRaises(ValueError, self.store.__getitem__, events.GetEntries() + 1)
def setUp(self):
self.eventStore = EventStore.get()
self.appStore = ApplicationStore.get()
self.fileFactory = FileFactory.get()
self.fileStore = FileStore.get()
self.userConf = UserConfigLoader.get("user.ini")
self.ar1 = Application("ristretto.desktop",
pid=21,
tstart=1,
tend=2000)
self.ar2 = Application("ristretto.desktop",
pid=22,
tstart=2600,
tend=2900)
self.ag1 = Application("gimp.desktop", pid=23, tstart=1, tend=4000)
self.ag2 = Application("gimp.desktop", pid=24, tstart=4500, tend=4590)
self.appStore.insert(self.ar1)
self.appStore.insert(self.ar2)
self.appStore.insert(self.ag1)
self.appStore.insert(self.ag2)
# Insert a file that will bridge r1 and g1.
s2 = "open64|/home/user/Images/Picture.jpg|fd 4: with flag 524288, e0|"
e2 = Event(actor=self.ag1, time=10, syscallStr=s2)
self.eventStore.append(e2)
e2b = Event(actor=self.ar1, time=12, syscallStr=s2)
self.eventStore.append(e2b)
# Insert a file that will bridge r1 and r2.
s3 = "open64|/home/user/Images/Photo.jpg|fd 10: with flag 524288, e0|"
e3 = Event(actor=self.ar1, time=10, syscallStr=s3)
self.eventStore.append(e3)
e3b = Event(actor=self.ar2, time=2710, syscallStr=s3)
self.eventStore.append(e3b)
# Insert a file that will bridge g1 and g2.
s4 = "open64|/home/user/Images/Art.xcf|fd 10: with flag 524288, e0|"
e4 = Event(actor=self.ag1, time=10, syscallStr=s4)
self.eventStore.append(e4)
e4b = Event(actor=self.ag2, time=4540, syscallStr=s4)
self.eventStore.append(e4b)
# Simulate.
self.eventStore.simulateAllEvents()
def setUp(self):
self.eventStore = EventStore.get()
self.appStore = ApplicationStore.get()
self.fileFactory = FileFactory.get()
self.userConf = UserConfigLoader.get("user.ini")
self.engine = PolicyEngine()
self.ar1 = Application("ristretto.desktop",
pid=21,
tstart=1,
tend=2000)
self.ar2 = Application("ristretto.desktop",
pid=22,
tstart=2600,
tend=2900)
self.ag1 = Application("gimp.desktop", pid=23, tstart=1, tend=4000)
self.ag2 = Application("gimp.desktop", pid=24, tstart=4500, tend=4590)
self.appStore.insert(self.ar1)
self.appStore.insert(self.ar2)
self.appStore.insert(self.ag1)
self.appStore.insert(self.ag2)
import unittest
from EventStore import EventStore
if __name__ == '__main__':
filename = 'example.root'
store = EventStore([filename])
for i, event in enumerate(store):
if i%1000 == 0:
print 'reading event', i
evinfo = store.get("info")[0]
clusters = store.get("clusters")
for cluster in clusters:
for ihit in range(cluster.Hits_size()):
hit = cluster.Hits(ihit)
print ' Referenced hit has an energy of', hit.energy()
def setUp(self):
self.store = EventStore.get()
def setUp(self):
self.userConf = UserConfigLoader.get("user.ini")
self.appStore = ApplicationStore.get()
self.eventStore = EventStore.get()
self.fileStore = FileStore.get()
self.fileFactory = FileFactory.get()
self.sim = AttackSimulator()
self.a1 = Application("ristretto.desktop", pid=1, tstart=1, tend=2000)
self.a2 = Application("firefox.desktop", pid=2, tstart=1, tend=2000)
self.a3 = Application("ristretto.desktop", pid=3, tstart=3000,
tend=6000)
self.ac = Application("catfish.desktop", pid=100, tstart=1, tend=2900)
self.appStore.insert(self.a1)
self.appStore.insert(self.a2)
self.appStore.insert(self.a3)
self.appStore.insert(self.ac)
self.p001 = "/home/user/.cache/firefox/file"
s001 = "open64|%s|fd 10: with flag 524288, e0|" % self.p001
e001 = Event(actor=self.a1, time=10, syscallStr=s001)
e001.evflags &= ~EventFileFlags.designation # not by designation
self.eventStore.append(e001)
e001b = Event(actor=self.a2, time=11, syscallStr=s001)
e001b.evflags &= ~EventFileFlags.designation # not by designation
self.eventStore.append(e001b)
self.p002 = "/home/user/Images/picture.jpg"
s002 = "open64|%s|fd 10: with flag 524288, e0|" % self.p002
e002 = Event(actor=self.a1, time=12, syscallStr=s002)
e002.evflags &= ~EventFileFlags.designation # not by designation
self.eventStore.append(e002)
e002b = Event(actor=self.ac, time=30, syscallStr=s002)
e002b.evflags &= ~EventFileFlags.designation # not by designation
self.eventStore.append(e002b)
self.p003 = "/home/user/Downloads/logo.jpg"
s003 = "open64|%s|fd 10: with flag 524288, e0|" % self.p003
e003 = Event(actor=self.a1, time=13, syscallStr=s003)
e003.evflags |= EventFileFlags.designation # this event by designation
self.eventStore.append(e003)
e003b = Event(actor=self.a3, time=3003, syscallStr=s003)
e003b.evflags &= ~EventFileFlags.designation # not by designation
self.eventStore.append(e003b)
self.p004 = "/home/user/Downloads/logo.png"
s004 = "open64|%s|fd 10: with flag 64, e0|" % self.p004
e004 = Event(actor=self.a1, time=14, syscallStr=s004)
e004.evflags &= ~EventFileFlags.designation # not by designation
self.eventStore.append(e004)
self.p005 = "/home/user/Dropbox/Photos/holidays.jpg"
s005 = "open64|%s|fd 10: with flag 524288, e0|" % self.p005
e005 = Event(actor=self.a1, time=15, syscallStr=s005)
e005.evflags &= ~EventFileFlags.designation # not by designation
self.eventStore.append(e005)
e005b = Event(actor=self.a1, time=1005, syscallStr=s005)
e005b.evflags &= ~EventFileFlags.designation # not by designation
self.eventStore.append(e005b)
self.p006 = "/home/user/Images/random.txt"
s006 = "open64|%s|fd 10: with flag 524288, e0|" % self.p006
e006 = Event(actor=self.a1, time=16, syscallStr=s006)
e006.evflags &= ~EventFileFlags.designation # not by designation
self.eventStore.append(e006)
self.p007 = "/home/user/Images/file.jpg"
s007 = "open64|%s|fd 10: with flag 524288, e0|" % self.p007
e007 = Event(actor=self.a1, time=17, syscallStr=s007)
e007.evflags |= EventFileFlags.designation # this event by designation
self.eventStore.append(e007)
self.p008 = "/home/user/Images/other.foo"
s008 = "open64|%s|fd 10: with flag 64, e0|" % self.p008
e008 = Event(actor=self.a1, time=18, syscallStr=s008)
e008.evflags &= ~EventFileFlags.designation # not by designation
self.eventStore.append(e008)
self.p009 = "/home/user/Downloads/unknown.data"
s009 = "open64|%s|fd 10: with flag 64, e0|" % self.p009
# e009 = Event(actor=self.a1, time=18, syscallStr=s009)
# e009.evflags |= EventFileFlags.designation # this event by designation
# self.eventStore.append(e009)
e009b = Event(actor=self.a3, time=3009, syscallStr=s009)
e009b.evflags &= ~EventFileFlags.designation # not by designation
self.eventStore.append(e009b)
self.p010 = "/home/user/Dropbox/Photos/holidays.metadata"
s010 = "open64|%s|fd 10: with flag 524288, e0|" % self.p010
e010 = Event(actor=self.a1, time=20, syscallStr=s010)
e010.evflags &= ~EventFileFlags.designation # not by designation
self.eventStore.append(e010)
self.p011 = "/home/user/Dropbox/Photos/holidays.evenmoremetadata"
s011 = "open64|%s|fd 11: with flag 524288, e0|" % self.p011
e011 = Event(actor=self.a3, time=3020, syscallStr=s011)
e011.evflags &= ~EventFileFlags.designation # not by designation
self.eventStore.append(e011)
self.eventStore.simulateAllEvents()
self.acCache = AccessListCache.get()
self.lookUps = dict()
self.allowedCache = dict()
class EventStoreTestCase(unittest.TestCase):
def setUp(self):
self.filename = str('example.root')
self.assertTrue(os.path.isfile(self.filename))
self.store = EventStore([self.filename])
def test_eventloop(self):
self.assertTrue(len(self.store) >= 0)
self.assertEqual(self.store.current_store.getEntries(),
len(self.store))
for iev, event in enumerate(self.store):
self.assertTrue(True)
if iev > 5:
break
def test_navigation(self):
event0 = self.store[0]
self.assertEqual(event0.__class__, self.store.__class__)
def test_collections(self):
evinfo = self.store.get("info")
self.assertTrue(len(evinfo) > 0)
particles = self.store.get("CollectionNotThere")
self.assertFalse(particles)
def test_read_only(self):
hits = self.store.get("hits")
# testing that one can't modify attributes in
# read-only pods
self.assertEqual(hits[0].energy(), 23.)
hits[0].energy(10)
self.assertEqual(hits[0].energy(), 10) # oops
def test_one_to_many(self):
clusters = self.store.get("clusters")
ref_hits = []
# testing that cluster hits can be accessed and make sense
for cluster in clusters:
sume = 0
for ihit in range(cluster.Hits_size()):
hit = cluster.Hits(ihit)
ref_hits.append(hit)
sume += hit.energy()
self.assertEqual(cluster.energy(), sume)
hits = self.store.get("hits")
# testing that the hits stored as a one to many relation
# in the cluster can be found in the hit collection
for hit in ref_hits:
self.assertTrue(hit in hits)
def test_relation_range(self):
"""Test that the RelationRange functionality is also accessible in python"""
clusters = self.store.get("clusters")
hits = self.store.get("hits")
for cluster in clusters:
sume = 0
for hit in cluster.Hits():
self.assertTrue(hit in hits)
sume += hit.energy()
self.assertEqual(cluster.energy(), sume)
def test_hash(self):
clusters = self.store.get("clusters")
ref_hits = []
# testing that cluster hits can be accessed and make sense
for cluster in clusters:
sume = 0
for ihit in range(cluster.Hits_size()):
hit = cluster.Hits(ihit)
ref_hits.append(hit)
sume += hit.energy()
self.assertEqual(cluster.energy(), sume)
hits = self.store.get("hits")
if hits[0] == ref_hits[0]:
self.assertEqual(hits[0].getObjectID().index,
ref_hits[0].getObjectID().index)
self.assertEqual(hits[0].getObjectID().collectionID,
ref_hits[0].getObjectID().collectionID)
self.assertEqual(hits[0].getObjectID(), ref_hits[0].getObjectID())
# testing that the hits stored as a one to many relation
# import pdb; pdb.set_trace()
def test_chain(self):
self.store = EventStore([self.filename,
self.filename])
rootfile = TFile(self.filename)
events = rootfile.Get(str('events'))
numbers = []
for iev, event in enumerate(self.store):
evinfo = self.store.get("info")
numbers.append(evinfo[0].Number())
self.assertEqual(iev + 1, 2 * events.GetEntries())
# testing that numbers is [0, .. 1999, 0, .. 1999]
self.assertEqual(numbers, list(range(events.GetEntries())) * 2)
# trying to go to an event beyond the last one
self.assertRaises(ValueError, self.store.__getitem__, 4001)
# this is in the first event in the second file,
# so its event number should be 0.
self.assertEqual(self.store[2000].get("info")[0].Number(), 0)
def test_context_managers(self):
with EventStore([self.filename]) as store:
self.assertTrue(len(store) >= 0)
self.assertTrue(store.isValid())
def test_no_file(self):
'''Test that non-accessible files are gracefully handled.'''
with self.assertRaises(ValueError):
self.store = EventStore('foo.root')
class EventStoreTestCase(unittest.TestCase):
def setUp(self):
self.filename = 'example1.root'
self.assertTrue(os.path.isfile(self.filename))
self.store = EventStore([self.filename])
def test_eventloop(self):
self.assertTrue(len(self.store) >= 0)
self.assertEqual(self.store.current_store.getEntries(),
len(self.store))
for iev, event in enumerate(self.store):
self.assertTrue(True)
if iev > 5:
break
def test_navigation(self):
event0 = self.store[0]
self.assertEqual(event0.__class__, self.store.__class__)
def test_collections(self):
evinfo = self.store.get("info")
self.assertTrue(len(evinfo) > 0)
particles = self.store.get("CollectionNotThere")
self.assertFalse(particles)
def test_read_only(self):
hits = self.store.get("hits")
# testing that one can't modify attributes in
# read-only pods
self.assertEqual(hits[0].energy(), 23.)
hits[0].energy(10)
self.assertEqual(hits[0].energy(), 10) # oops
# self.assertEqual(type(hits[0]), ConstExampleHit) # should be True
def test_one_to_many(self):
clusters = self.store.get("clusters")
ref_hits = []
# testing that cluster hits can be accessed and make sense
for cluster in clusters:
sume = 0
for ihit in range(cluster.Hits_size()):
hit = cluster.Hits(ihit)
ref_hits.append(hit)
sume += hit.energy()
self.assertEqual(cluster.energy(), sume)
hits = self.store.get("hits")
# testing that the hits stored as a one to many relation
# in the cluster can be found in the hit collection
for hit in ref_hits:
self.assertTrue(hit in hits)
def test_hash(self):
clusters = self.store.get("clusters")
ref_hits = []
# testing that cluster hits can be accessed and make sense
for cluster in clusters:
sume = 0
for ihit in range(cluster.Hits_size()):
hit = cluster.Hits(ihit)
ref_hits.append(hit)
sume += hit.energy()
self.assertEqual(cluster.energy(), sume)
hits = self.store.get("hits")
if hits[0] == ref_hits[0]:
self.assertEqual(hits[0].getObjectID().index,
ref_hits[0].getObjectID().index)
self.assertEqual(hits[0].getObjectID().collectionID,
ref_hits[0].getObjectID().collectionID)
self.assertEqual(hits[0].getObjectID(), ref_hits[0].getObjectID())
# testing that the hits stored as a one to many relation
# import pdb; pdb.set_trace()
def test_chain(self):
self.store = EventStore([self.filename,
self.filename])
rootfile = TFile(self.filename)
events = rootfile.Get('events')
numbers = []
for iev, event in enumerate(self.store):
evinfo = self.store.get("info")
numbers.append(evinfo[0].Number())
self.assertEqual(iev+1, 2*events.GetEntries())
# testing that numbers is [0, .. 1999, 0, .. 1999]
self.assertEqual(numbers, range(events.GetEntries())*2)
# trying to go to an event beyond the last one
self.assertRaises(ValueError, self.store.__getitem__,
4001)
# this is in the first event in the second file,
# so its event number should be 0.
self.assertEqual(self.store[2000].get("info")[0].Number(), 0)
def test_context_managers(self):
with EventStore([self.filename]) as store:
self.assertTrue(len(store) >= 0)
self.assertTrue(store.isValid())
def test_no_file(self):
'''Test that non-accessible files are gracefully handled.'''
with self.assertRaises(ValueError):
self.store = EventStore('foo.root')
import unittest
from EventStore import EventStore
if __name__ == '__main__':
filename = 'example.root'
store = EventStore([filename])
for i, event in enumerate(store):
if i % 1000 == 0:
print 'reading event', i
evinfo = store.get("info")[0]
clusters = store.get("clusters")
for cluster in clusters:
for ihit in range(cluster.Hits_size()):
hit = cluster.Hits(ihit)
print ' Referenced hit has an energy of', hit.energy()
def setUp(self):
self.eventStore = EventStore.get()
self.appStore = ApplicationStore.get()
self.userConf = UserConfigLoader.get("user.ini")
self.fileFactory = FileFactory.get()
def main(argv):
__opt_inode_query = None
__opt_post_analysis = None
__opt_quick_pol = None
# Parse command-line parameters
try:
(opts, args) = getopt.getopt(argv, "hta:cedf:o:q:sk:rpgGi:u:x", [
"help", "attacks", "post-analysis=", "check-missing",
"check-excluded-files", "debug", "frequency", "inode",
"extensions", "related-files", "output=", "output-fs=", "score",
"quick-pol=", "skip=", "user", "clusters", "print-clusters",
"graph", "graph-clusters", "disable-plotting"
])
except (getopt.GetoptError):
print(USAGE_STRING)
sys.exit(2)
else:
for opt, arg in opts:
if opt in ('-h', '--help'):
print(USAGE_STRING + "\n\n\n\n")
print("--attacks:\n\tSimulates attacks and reports "
"on proportions of infected files and apps.\n")
print("--check-excluded-files:\n\tPrints the lists of files "
"accessed by apps that also wrote to excluded\n\tfiles,"
" then aborts execution of the program.\n")
print("--check-missing:\n\tChecks whether some Desktop IDs "
"for apps in the user's directory are\n\tmissing. If so,"
" aborts execution of the program.\n")
print("--clusters:\n\tPrints clusters of files with "
"information flows to one another. Requires\n\tthe "
"--score option.\n")
print("--debug:\n\tPrints additional debug information in "
"various code paths to help debug\n\tthe program.\n")
print("--disable-plotting:\n\tDo not plot cluster graphs. See "
"the --graph option.\n")
print("--extensions:\n\tPrints file extensions and MIME type "
"associations for this user.\n")
print("--frequency:\n\tSets the frequency used by the "
"frequent-itemsets algorithm in the\n\t--related-files "
"post-analysis. Requires the --related-files option.\n")
print("--graph:\n\tFind communities in file/app "
"accesses using graph theory methods.\n")
print("--help:\n\tPrints this help information and exits.\n")
print("--output=<DIR>:\n\tSaves a copy of the simulated "
"files, and some information on events\n\trelated to "
"them, in a folder created at the <DIR> path.\n")
print("--post-analysis=<DIR,DIR,DIR>:\n\t"
"Uses the value pointed to"
" by --output in order to produce graphs and\n\t"
"statistics.\n")
print("--quick-pol=Policy:\n\tReplace the default policies "
"with this one single Policy.\n")
print("--related-files:\n\tMines for files that are frequently"
" accessed together by apps. Produces\n\toutput files in"
" scoring mode, and an analysis output in post-analysis"
"\n\tmode. See also --frequency.\n")
print("--score:\n\tCalculates the usability and security "
"scores of a number of file access\n\tcontrol policies"
", replayed over the simulated accesses. Prints results"
"\n\tand saves them to the output directory.\n")
print(
"--skip=<Policy,Policy,'graphs'>:\n\tSkip the scoring of "
"policies in the lists. If the list contains the word"
"\n\t'graphs', skips the general graph computation.\n")
sys.exit()
elif opt in ('-c', '--check-missing'):
__setCheckMissing(True)
elif opt in ('-e', '--check-excluded-files'):
__setCheckExcludedFiles(True)
elif opt in ('-x', '--extensions'):
__setPrintExtensions(True)
elif opt in ('-d', '--debug'):
__setDebug(True)
elif opt in ('-r', '--related-files'):
__setRelatedFiles(True)
elif opt in ('-s', '--score'):
__setScore(True)
elif opt in ('-p', '--print-clusters', '--clusters'):
__setPrintClusters(True)
elif opt in ('-g', '--graph-clusters', '--graph'):
__setGraph(True)
elif opt in ('-t', '--attacks'):
__setAttacks(True)
elif opt in ('-G', '--disable-plotting'):
__setPlottingDisabled(True)
elif opt in ('-f', '--frequency'):
if not arg:
print(USAGE_STRING)
sys.exit(2)
__setFrequency(arg[1:] if arg[0] == '=' else arg)
elif opt in ('-o', '--output-fs', '--output'):
if not arg:
print(USAGE_STRING)
sys.exit(2)
__setOutputFs(arg[1:] if arg[0] == '=' else arg)
elif opt in ('-u', '--user'):
if not arg:
print(USAGE_STRING)
sys.exit(2)
__setUser(arg[1:] if arg[0] == '=' else arg)
elif opt in ('-i', '--inode'):
if not arg:
print(USAGE_STRING)
sys.exit(2)
try:
__opt_inode_query = (arg[1:] if arg[0] == '=' else arg)
except (ValueError) as e:
print(USAGE_STRING)
sys.exit(2)
elif opt in ('-a', '--post-analysis'):
if not arg:
print(USAGE_STRING)
sys.exit(2)
__opt_post_analysis = (arg[1:] if arg[0] == '=' else arg)
elif opt in ('-q', '--quick-pol'):
if not arg:
print(USAGE_STRING)
sys.exit(2)
__opt_quick_pol = (arg[1:] if arg[0] == '=' else arg)
elif opt in ('-k', '--skip'):
if not arg:
print(USAGE_STRING)
sys.exit(2)
__opt_skip = (arg[1:] if arg[0] == '=' else arg)
__setSkip(__opt_skip.split(","))
registerTimePrint()
if __opt_post_analysis:
if relatedFilesEnabled():
tprnt("Starting post-analysis of related files...\n")
engine = FrequentFileEngine()
engine.processFrequentItemLists(__opt_post_analysis)
else:
tprnt("Starting post-analysis of usability/security scores...\n")
from AnalysisEngine import AnalysisEngine
if outputFsEnabled():
engine = AnalysisEngine(inputDir=__opt_post_analysis,
outputDir=outputFsEnabled())
else:
engine = AnalysisEngine(inputDir=__opt_post_analysis)
engine.analyse()
sys.exit(0)
# Make the application, event and file stores
store = ApplicationStore.get()
evStore = EventStore.get()
fileStore = FileStore.get()
initMimeTypes()
datapath = getDataPath()
# Load up user-related variables
userConf = UserConfigLoader.get(path=datapath + USERCONFIGNAME)
# Load up and check the SQLite database
sql = None
tprnt("\nLoading the SQLite database: %s..." % (datapath + DATABASENAME))
try:
sql = SqlLoader(datapath + DATABASENAME)
except ValueError as e:
print("Failed to parse SQL: %s" % e.args[0], file=sys.stderr)
sys.exit(-1)
if checkMissingEnabled():
tprnt("Checking for missing application identities...")
sql.listMissingActors()
sql.loadDb(store)
sqlAppCount = sql.appCount
sqlInstCount = sql.instCount
sqlEvCount = sql.eventCount
sqlValidEvCount = sql.validEventRatio
tprnt("Loaded the SQLite database.")
# Load up the PreloadLogger file parser
tprnt("\nLoading the PreloadLogger logs in folder: %s..." % datapath)
pll = PreloadLoggerLoader(datapath)
if checkMissingEnabled():
tprnt("Checking for missing application identities...")
pll.listMissingActors()
pll.loadDb(store)
pllAppCount = pll.appCount
pllInstCount = pll.instCount
pllEvCount = pll.eventCount
pllValidEvCount = pll.validEventRatio
tprnt("Loaded the PreloadLogger logs.")
# Resolve actor ids in all apps' events
tprnt("\nUsing PreloadLogger Applications to resolve interpreters in "
"Zeitgeist Applications...")
(interpretersAdded, instancesEliminated) = store.resolveInterpreters()
tprnt("Resolved interpreter ids in %d Applications, and removed %d "
"instances by merging them with another as a result." %
(interpretersAdded, instancesEliminated))
# Update events' actor ids in the ApplicationStore, then take them and send
# them to the EvnetStore. Finally, sort the EventStore by timestamp.
tprnt("\nInserting and sorting all events...")
store.sendEventsToStore()
evStore.sort()
evCount = evStore.getEventCount()
tprnt("Sorted all %d events in the event store." % evCount)
# Simulate the events to build a file model
tprnt("\nSimulating all events to build a file model...")
evStore.simulateAllEvents()
del sql
del pll
evStore.sort()
tprnt("Simulated all events. %d files initialised." % len(fileStore))
appCount = store.getAppCount()
userAppCount = store.getUserAppCount()
instCount = len(store)
userInstCount = store.getUserInstCount()
fileCount = len(fileStore)
docCount = fileStore.getUserDocumentCount(userConf.getSetting("HomeDir"))
if printExtensions():
exts = set()
for f in fileStore:
exts.add(f.getExtension())
try:
exts.remove(None)
except (KeyError):
pass
tprnt("Info: the following file extensions were found:")
for e in sorted(exts):
print("\t%s: %s" %
(e, mimetypes.guess_type("f.%s" % e, strict=False)))
if checkExcludedFilesEnabled():
tprnt("\nPrinting files written and read by instances which wrote"
"to excluded directories...")
dbgPrintExcludedEvents()
import time as t
t.sleep(10)
# Manage --inode queries
if __opt_inode_query:
inodes = __opt_inode_query.split(",")
for inode in sorted(int(i) for i in inodes):
f = fileStore.getFile(inode)
tprnt("\nInode queried: %d" % inode)
tprnt("Corresponding file: %s\n\t(%s)" % (f.getName(), f))
sys.exit(0)
# Print the model as proof of concept
if debugEnabled():
tprnt("\nPrinting the file model...\n")
fileStore.printFiles(showDeleted=True,
showCreationTime=True,
showDocumentsOnly=True,
userHome=userConf.getSetting("HomeDir"),
showDesignatedOnly=False)
# Make the filesystem corresponding to the model
if outputFsEnabled():
tprnt("\nMaking a copy of the file model at '%s'...\n" %
outputFsEnabled())
fileStore.makeFiles(outputDir=outputFsEnabled(),
showDeleted=True,
showDocumentsOnly=False,
userHome=userConf.getSetting("HomeDir"),
showDesignatedOnly=False)
with open(os.path.join(outputFsEnabled(), "statistics.txt"), "w") as f:
msg = "SQL: %d apps; %d instances; %d events; %d%% valid\n" % \
(sqlAppCount, sqlInstCount, sqlEvCount, sqlValidEvCount)
msg += "PreloadLogger: %d apps; %d instances; %d events; " \
"%d%% valid\n" % \
(pllAppCount, pllInstCount, pllEvCount, pllValidEvCount)
msg += "Simulated: %d apps; %d instances; %d user apps; %d user" \
" instances; %d events; %d files; %d user documents\n" % \
(appCount, instCount, userAppCount, userInstCount,
evCount, fileCount, docCount)
exclLists = userConf.getDefinedSecurityExclusionLists()
for l in exclLists:
msg += "Exclusion list '%s' defined.\n" % l
print(msg, file=f)
# Build a general access graph.
if graphEnabled():
skipList = skipEnabled()
if skipList and 'graphs' in skipList:
tprnt("\nGraphs in skip list, skipping global graph generation.")
else:
engine = GraphEngine.get()
engine.runGraph(policy=None)
# Policy engine. Create a policy and run a simulation to score it.
if scoreEnabled() or attacksEnabled() or graphEnabled():
engine = PolicyEngine()
if __opt_quick_pol:
policies = [__opt_quick_pol]
polArgs = [None]
else:
policies = [
CompoundLibraryPolicy,
CustomLibraryPolicy,
DesignationPolicy,
DistantFolderPolicy,
FilenamePolicy,
FileTypePolicy,
FolderPolicy,
OneDistantFolderPolicy,
OneFolderPolicy,
OneLibraryPolicy,
UnsecurePolicy,
Win10Policy,
Win8Policy,
HSecurePolicy,
HBalancedPolicy,
'HSecureSbPolicy',
'HSecureSbFaPolicy',
'HSecureFaPolicy',
'HBalancedSbPolicy',
'HBalancedSbFaPolicy',
'HBalancedFaPolicy',
'OneDistantFolderSbPolicy',
'OneDistantFolderSbFaPolicy',
'OneDistantFolderFaPolicy',
'HUsableSecuredSbPolicy',
'HUsableSecuredSbFaPolicy',
'HUsableSecuredFaPolicy',
'HBalancedSecuredSbPolicy',
'HBalancedSecuredSbFaPolicy',
'HBalancedSecuredFaPolicy',
'DistantFolderSbPolicy',
'DistantFolderSbFaPolicy',
'DistantFolderFaPolicy',
'LibraryFolderSbPolicy',
'LibraryFolderSbFaPolicy',
'LibraryFolderFaPolicy',
'FileTypeSbPolicy',
'FileTypeSbFaPolicy',
'FileTypeFaPolicy',
'OneFolderSbPolicy',
'OneFolderSbFaPolicy',
'OneFolderFaPolicy',
'FolderSbPolicy',
'FolderSbFaPolicy',
'FolderFaPolicy',
'OneLibrarySbPolicy',
'OneLibrarySbFaPolicy',
'OneLibraryFaPolicy',
'CompoundLibrarySbPolicy',
'CompoundLibrarySbFaPolicy',
'CompoundLibraryFaPolicy',
'CustomLibrarySbPolicy',
'CustomLibrarySbFaPolicy',
'CustomLibraryFaPolicy',
]
polArgs = [
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
]
# dict(folders=["~/Downloads", "/tmp"])
skipList = skipEnabled()
for (polIdx, polName) in enumerate(policies):
pol = None
arg = polArgs[polIdx]
# Names with certain suffixes are dynamically generated policies.
if isinstance(polName, str):
if polName.endswith('SbPolicy'):
pols = [
getattr(sys.modules[__name__],
polName[:-8] + 'Policy'), StickyBitPolicy
]
args = [arg, dict(folders=["~", "/media", "/mnt"])]
pol = CompositionalPolicy(pols, args, polName)
elif polName.endswith('SbFaPolicy'):
pols = [
getattr(sys.modules[__name__],
polName[:-10] + 'Policy'), StickyBitPolicy,
FutureAccessListPolicy
]
args = [arg, dict(folders=["~", "/media", "/mnt"]), None]
pol = CompositionalPolicy(pols, args, polName)
elif polName.endswith('FaPolicy'):
pols = [
getattr(sys.modules[__name__],
polName[:-8] + 'Policy'),
FutureAccessListPolicy
]
args = [arg, None]
pol = CompositionalPolicy(pols, args, polName)
# A normal policy, just invoke it directly.
else:
polName = getattr(sys.modules[__name__], polName)
# Existing policies, with arguments / or normal policies passed as
# strings, including via the --quick flag.
if not pol:
pol = polName(**arg) if arg else polName()
tprnt("\nRunning %s..." % pol.name)
if skipList and pol.name in skipList:
tprnt("%s is in skip list, skipping." % pol.name)
continue
engine.runPolicy(pol,
outputDir=outputFsEnabled(),
printClusters=printClustersEnabled())
if pol.name == "FileTypePolicy" and checkMissingEnabled():
pol.abortIfUnsupportedExtensions()
if attacksEnabled():
tprnt("Simulating attacks on %s..." % pol.name)
sim = AttackSimulator(seed=0)
sim.runAttacks(pol, outputDir=outputFsEnabled() or "/tmp/")
del pol
# Calculate frequently co-accessed files:
if relatedFilesEnabled():
engine = FrequentFileEngine()
tprnt("\nMining for frequently co-accessed file types...")
engine.mineFileTypes()
