def storeEmails(Evidences):
"""
A function to store the metadata of extracted emails files.
"""
for evi in Evidences:
# evidence creation in the database
EviDb = fritModel.Evidence.query.filter_by(
name=fritutils.unicodify(evi.fileName)).first()
if not EviDb:
EviDb = fritModel.Evidence(name=fritutils.unicodify(evi.fileName),
configName=fritutils.unicodify(
evi.configName))
for fs in evi.fileSystems:
# filesystem creation in the database
FsDb = fritModel.Filesystem.query.filter_by(
evidence=EviDb,
configName=fritutils.unicodify(fs.configName)).first()
if not FsDb:
FsDb = fritModel.Filesystem(evidence=EviDb,
configName=fritutils.unicodify(
fs.configName))
nbFiles = 0
fritutils.termout.printNormal(
'Start inserting emails files metadata in database for "%s"\n'
% fs.configName)
for f in fs.listEmails():
insertFile(f, '', u'Contained', EviDb, FsDb)
nbFiles += 1
print "\t%s : %d\r" % (fs.configName, nbFiles),
print "\n"
def update(Evidences):
for evi in Evidences:
# evidence creation in the database
EviDb = fritModel.Evidence.query.filter_by(name=fritutils.unicodify(evi.fileName)).first()
if not EviDb:
EviDb = fritModel.Evidence(name=fritutils.unicodify(evi.fileName),configName=fritutils.unicodify(evi.configName))
if evi.isLocked("store"):
fritutils.termout.printWarning('%s is already locked by a "store" instance.' % evi.configName)
else:
# we start to insert filesystems normal files metadata in the database
evi.mount("store","Mounted to create initial database")
for fs in evi.fileSystems:
# first, we count the files that are already in the DB
fcount = fs.dbCountFiles()[u'Normal']
if fcount['Files'] > 0:
fritutils.termout.printMessage('%d files are already in the database, not inserting.' % fcount['Files'])
else:
# filesystem creation in the database
FsDb = fritModel.Filesystem.query.filter_by(evidence=EviDb,configName=fritutils.unicodify(fs.configName)).first()
if not FsDb:
FsDb = fritModel.Filesystem(evidence=EviDb,configName=fritutils.unicodify(fs.configName))
if fs.isLocked("store"):
fritutils.termout.printWarning('Filesystem "%s" is already locked by a "store" instance.' % fs.configName)
else:
fs.mount("store","Mounted to create initial database")
spos = len(fs.fsMountPoint)
nbFiles = 0
fritutils.termout.printNormal('Start inserting files metadata in database for "%s"\n' % fs.configName)
for f in fs.listFiles():
insertFile(f,fs.fsMountPoint,u'Normal',EviDb,FsDb)
nbFiles += 1
print "\t%s : %d\r" % (fs.configName,nbFiles),
print "\n"
fs.umount("store")
evi.umount("store")
def update(Evidences):
for evi in Evidences:
# evidence creation in the database
EviDb = fritModel.Evidence.query.filter_by(
name=fritutils.unicodify(evi.fileName)).first()
if not EviDb:
EviDb = fritModel.Evidence(name=fritutils.unicodify(evi.fileName),
configName=fritutils.unicodify(
evi.configName))
if evi.isLocked("store"):
fritutils.termout.printWarning(
'%s is already locked by a "store" instance.' % evi.configName)
else:
# we start to insert filesystems normal files metadata in the database
evi.mount("store", "Mounted to create initial database")
for fs in evi.fileSystems:
# first, we count the files that are already in the DB
fcount = fs.dbCountFiles()[u'Normal']
if fcount['Files'] > 0:
fritutils.termout.printMessage(
'%d files are already in the database, not inserting.'
% fcount['Files'])
else:
# filesystem creation in the database
FsDb = fritModel.Filesystem.query.filter_by(
evidence=EviDb,
configName=fritutils.unicodify(fs.configName)).first()
if not FsDb:
FsDb = fritModel.Filesystem(
evidence=EviDb,
configName=fritutils.unicodify(fs.configName))
if fs.isLocked("store"):
fritutils.termout.printWarning(
'Filesystem "%s" is already locked by a "store" instance.'
% fs.configName)
else:
fs.mount("store", "Mounted to create initial database")
spos = len(fs.fsMountPoint)
nbFiles = 0
fritutils.termout.printNormal(
'Start inserting files metadata in database for "%s"\n'
% fs.configName)
for f in fs.listFiles():
insertFile(f, fs.fsMountPoint, u'Normal', EviDb,
FsDb)
nbFiles += 1
print "\t%s : %d\r" % (fs.configName, nbFiles),
print "\n"
fs.umount("store")
evi.umount("store")
def insertFile(File, prefix, state, eviDb, fsDb):
"""
A function that insert a file into the database.
prefix: the prefix to remove from the path before inserting (.frit/filesystems/Evidencex/filesystemx...)
state: the database file state (normal, undeleted, carved, contained ...)
"""
fsize = 0
try:
fsize = os.path.getsize(File)
except:
print >> sys.stderr, "ERROR GETTING SIZE OF: %s\n" % File
fname, ext = os.path.splitext(File)
ext = fritutils.unicodify(ext.lower())
if ext == '':
ext = u'No Extension'
dname = fritutils.unicodify(os.path.dirname(File))
if prefix != '':
dname = dname.replace(prefix, '')
if dname == '':
dname = u'/'
bname = fritutils.unicodify(os.path.basename(File))
Ext = fritModel.Extension.query.filter_by(extension=ext).first()
if not Ext:
Ext = fritModel.Extension(extension=ext)
Fpath = fritModel.FullPath.query.filter_by(fullpath=dname).first()
if not Fpath:
Fpath = fritModel.FullPath(fullpath=dname)
nFile = fritModel.File.query.filter_by(evidence=eviDb,
filesystem=fsDb,
filename=bname,
fullpath=Fpath).first()
if not nFile:
nFile = fritModel.File(evidence=eviDb, filesystem=fsDb)
nFile.state = fritModel.FileState.query.filter_by(state=state).first()
nFile.filename = bname
nFile.filesize = fsize
nFile.fullpath = Fpath
nFile.extension = Ext
fritModel.elixir.session.commit()
def insertFile(File,prefix,state,eviDb,fsDb):
"""
A function that insert a file into the database.
prefix: the prefix to remove from the path before inserting (.frit/filesystems/Evidencex/filesystemx...)
state: the database file state (normal, undeleted, carved, contained ...)
"""
fsize = 0
try:
fsize = os.path.getsize(File)
except:
print >> sys.stderr, "ERROR GETTING SIZE OF: %s\n" % File
fname,ext = os.path.splitext(File)
ext = fritutils.unicodify(ext.lower())
if ext == '':
ext = u'No Extension'
dname = fritutils.unicodify(os.path.dirname(File))
if prefix != '':
dname = dname.replace(prefix,'')
if dname == '':
dname = u'/'
bname = fritutils.unicodify(os.path.basename(File))
Ext = fritModel.Extension.query.filter_by(extension=ext).first()
if not Ext:
Ext = fritModel.Extension(extension=ext)
Fpath = fritModel.FullPath.query.filter_by(fullpath=dname).first()
if not Fpath:
Fpath = fritModel.FullPath(fullpath=dname)
nFile = fritModel.File.query.filter_by(evidence=eviDb,filesystem=fsDb,filename=bname,fullpath=Fpath).first()
if not nFile:
nFile = fritModel.File(evidence=eviDb,filesystem=fsDb)
nFile.state = fritModel.FileState.query.filter_by(state=state).first()
nFile.filename = bname
nFile.filesize = fsize
nFile.fullpath = Fpath
nFile.extension = Ext
fritModel.elixir.session.commit()
def storeEmails(Evidences):
"""
A function to store the metadata of extracted emails files.
"""
for evi in Evidences:
# evidence creation in the database
EviDb = fritModel.Evidence.query.filter_by(name=fritutils.unicodify(evi.fileName)).first()
if not EviDb:
EviDb = fritModel.Evidence(name=fritutils.unicodify(evi.fileName),configName=fritutils.unicodify(evi.configName))
for fs in evi.fileSystems:
# filesystem creation in the database
FsDb = fritModel.Filesystem.query.filter_by(evidence=EviDb,configName=fritutils.unicodify(fs.configName)).first()
if not FsDb:
FsDb = fritModel.Filesystem(evidence=EviDb,configName=fritutils.unicodify(fs.configName))
nbFiles = 0
fritutils.termout.printNormal('Start inserting emails files metadata in database for "%s"\n' % fs.configName)
for f in fs.listEmails():
insertFile(f,'',u'Contained',EviDb,FsDb)
nbFiles += 1
print "\t%s : %d\r" % (fs.configName,nbFiles),
print "\n"
def factory(Evidences, args, options, fritConf):
validArgs = ('count', 'extract', 'list')
stateOptions = {
'--normal': u'Normal',
'--contained': u'Contained',
'--undeleted': u'Undeleted',
'--carved': u'Carved'
}
definedExtensions = getExtLists(fritConf)
if not fritModel.dbExists():
fritutils.termout.printWarning(
'The database does not exists yet. You should create it first by issuing "frit store create".'
)
logger.warning('Database was not found')
sys.exit(1)
states = []
extList = []
if not args or len(args) == 0:
fritutils.termout.printWarning(
'extensions command need at least an argument to define an action (%s).'
% ', '.join(validArgs))
sys.exit(1)
elif args[0] not in validArgs:
fritutils.termout.printWarning(
'extensions command need a valid argument (%s)' %
', '.join(validArgs))
sys.exit(1)
else:
subcommand = args[0]
args.remove(subcommand)
logger.info('subcommand issued: %s' % subcommand)
if options:
logger.info('options: %s' % ','.join(options))
for o in options:
if o in stateOptions.keys():
states.append(stateOptions[o])
if len(states) == 0:
states = list(fritModel.FILESTATES)
logger.info('states: %s' % ','.join(states))
# Finding extensions to work with
# Searching if one or more predefined extensions list is in the args
for a in list(args):
if a in definedExtensions.keys():
logger.info('Extension list "%s" asked in command line.' %
args)
args.remove(a)
extList.extend(definedExtensions[a])
# the remaining args should be the extensions that we want to list
# if there is no more args, we list all extensions
if (not args or len(args) == 0) and len(extList) == 0:
for ex in fritModel.elixir.session.query(
fritModel.Extension.extension).all():
extList.append(ex[0])
else:
for ex in args:
extList.append(fritutils.unicodify(ex))
logger.info('Extensions: "%s"' % " ".join(extList))
if subcommand == 'count':
logger.info('Starting subcommand count')
fritModel.listExtensions(Evidences, extList, states)
elif subcommand == 'list':
logger.info('Starting list subcommand.')
for evi in Evidences:
for fs in evi.fileSystems:
for ext in sorted(extList):
for state in states:
for fp in fs.ExtensionsFritFiles(ext, state):
fritutils.termout.printNormal(fp)
elif subcommand == 'extract':
logger.info('Starting extract subcommand')
# The '--merge' option is used to merge extractions in a single
# directory base instead of having a directory by extension.
merge = False
if options and '--merge' in options:
merge = True
# we start by extracting 'normal files' because we need to mount the containers and filesystems
if u'Normal' in states:
logger.info('Starting Normal files extraction.')
states.remove(u'Normal')
for evi in Evidences:
# We count files to extract to see if it's needed to go further
enbe = evi.dbCountExtension(extList, u'Normal')
if enbe['count'] > 0:
logger.info(
'Found %d files to exctract, mounting Evidence container "%s".'
% (enbe['count'], evi.configName))
evi.mount('extensions',
'Extracting files based on extensions')
for fs in evi.fileSystems:
fritutils.termout.printMessage(
"\t%s" % fs.evidence.configName + '/' +
fs.configName)
fs.mount('extensions',
'Extracting files based on extensions')
for ext in sorted(extList):
nbe = fs.dbCountExtension(ext, u'Normal')
fritutils.termout.printMessage(
"Extracting %d files (%s)" %
(nbe['count'],
fritutils.humanize(nbe['size'])))
for filepath in fs.ExtensionsOriginalFiles(
ext, u'Normal'):
if ext == "No Extension":
extPath = "no_extension"
else:
extPath = ext[1:]
basePath = os.path.dirname(filepath)
if merge:
Destination = unicode(
os.path.join(
'.frit/extractions/by_extensions/',
evi.configName, fs.configName,
basePath))
else:
Destination = unicode(
os.path.join(
'.frit/extractions/by_extensions/',
evi.configName, fs.configName,
extPath, basePath))
mountedPath = os.path.join(
fs.fsMountPoint, filepath)
extractFile(mountedPath, Destination)
fs.umount('extensions')
evi.umount('extensions')
else:
logger.info(
'No Normal files to extract on Evidence "%s", skipping'
% evi.configName)
for state in states:
logger.info('Starting to extract %s files' % state)
for evi in Evidences:
for fs in evi.fileSystems:
for ext in sorted(extList):
nbe = fs.dbCountExtension(ext, state)
if nbe['count'] > 0:
fritutils.termout.printMessage(
"Extracting %s %d files (%s)" %
(state, nbe['count'],
fritutils.humanize(nbe['size'])))
for filepath in fs.ExtensionsOriginalFiles(
ext, state):
# as we do not store the first character of the path, we have to re-add the '.'
filepath = '.' + filepath
if ext == "No Extension":
extPath = "no_extension"
else:
extPath = ext[1:]
# we dont want to have '.frit/extractions' in the middle of the destination path:
basePath = os.path.dirname(
filepath.replace(
'.frit/extractions/', ''))
if merge:
Destination = unicode(
os.path.join(
'.frit/extractions/by_extensions/',
evi.configName, fs.configName,
basePath))
else:
Destination = unicode(
os.path.join(
'.frit/extractions/by_extensions/',
evi.configName, fs.configName,
extPath, basePath))
extractFile(filepath, Destination)
else:
logger.info(
'Nothing found to extract on "%s".' %
(evi.configName + '/' + fs.configName))
def factory(Evidences, args, options, fritConf):
validArgs = ('count', 'extract','list')
stateOptions = {'--normal':u'Normal','--contained':u'Contained','--undeleted':u'Undeleted','--carved':u'Carved'}
definedExtensions = getExtLists(fritConf)
if not fritModel.dbExists():
fritutils.termout.printWarning('The database does not exists yet. You should create it first by issuing "frit store create".')
logger.warning('Database was not found')
sys.exit(1)
states = []
extList = []
if not args or len(args) == 0:
fritutils.termout.printWarning('extensions command need at least an argument to define an action (%s).' % ', '.join(validArgs))
sys.exit(1)
elif args[0] not in validArgs:
fritutils.termout.printWarning('extensions command need a valid argument (%s)' % ', '.join(validArgs))
sys.exit(1)
else:
subcommand = args[0]
args.remove(subcommand)
logger.info('subcommand issued: %s' % subcommand)
if options:
logger.info('options: %s' % ','.join(options))
for o in options:
if o in stateOptions.keys():
states.append(stateOptions[o])
if len(states) == 0:
states = list(fritModel.FILESTATES)
logger.info('states: %s' % ','.join(states))
# Finding extensions to work with
# Searching if one or more predefined extensions list is in the args
for a in list(args):
if a in definedExtensions.keys():
logger.info('Extension list "%s" asked in command line.' % args)
args.remove(a)
extList.extend(definedExtensions[a])
# the remaining args should be the extensions that we want to list
# if there is no more args, we list all extensions
if (not args or len(args) == 0) and len(extList) == 0:
for ex in fritModel.elixir.session.query(fritModel.Extension.extension).all():
extList.append(ex[0])
else:
for ex in args:
extList.append(fritutils.unicodify(ex))
logger.info('Extensions: "%s"' % " ".join(extList))
if subcommand == 'count':
logger.info('Starting subcommand count')
fritModel.listExtensions(Evidences,extList,states)
elif subcommand == 'list':
logger.info('Starting list subcommand.')
for evi in Evidences:
for fs in evi.fileSystems:
for ext in sorted(extList):
for state in states:
for fp in fs.ExtensionsFritFiles(ext,state):
fritutils.termout.printNormal(fp)
elif subcommand == 'extract':
logger.info('Starting extract subcommand')
# The '--merge' option is used to merge extractions in a single
# directory base instead of having a directory by extension.
merge = False
if options and '--merge' in options:
merge = True
# we start by extracting 'normal files' because we need to mount the containers and filesystems
if u'Normal' in states:
logger.info('Starting Normal files extraction.')
states.remove(u'Normal')
for evi in Evidences:
# We count files to extract to see if it's needed to go further
enbe = evi.dbCountExtension(extList, u'Normal')
if enbe['count'] > 0:
logger.info('Found %d files to exctract, mounting Evidence container "%s".' % (enbe['count'],evi.configName))
evi.mount('extensions', 'Extracting files based on extensions')
for fs in evi.fileSystems:
fritutils.termout.printMessage("\t%s" % fs.evidence.configName + '/' + fs.configName)
fs.mount('extensions', 'Extracting files based on extensions')
for ext in sorted(extList):
nbe = fs.dbCountExtension(ext,u'Normal')
fritutils.termout.printMessage("Extracting %d files (%s)" % (nbe['count'],fritutils.humanize(nbe['size'])))
for filepath in fs.ExtensionsOriginalFiles(ext,u'Normal'):
if ext == "No Extension":
extPath = "no_extension"
else:
extPath = ext[1:]
basePath = os.path.dirname(filepath)
if merge:
Destination = unicode(os.path.join('.frit/extractions/by_extensions/',evi.configName,fs.configName,basePath))
else:
Destination = unicode(os.path.join('.frit/extractions/by_extensions/',evi.configName,fs.configName,extPath,basePath))
mountedPath = os.path.join(fs.fsMountPoint,filepath)
extractFile(mountedPath,Destination)
fs.umount('extensions')
evi.umount('extensions')
else:
logger.info('No Normal files to extract on Evidence "%s", skipping' % evi.configName)
for state in states:
logger.info('Starting to extract %s files' % state)
for evi in Evidences:
for fs in evi.fileSystems:
for ext in sorted(extList):
nbe = fs.dbCountExtension(ext,state)
if nbe['count'] >0 :
fritutils.termout.printMessage("Extracting %s %d files (%s)" % (state,nbe['count'],fritutils.humanize(nbe['size'])))
for filepath in fs.ExtensionsOriginalFiles(ext,state):
# as we do not store the first character of the path, we have to re-add the '.'
filepath = '.' + filepath
if ext == "No Extension":
extPath = "no_extension"
else:
extPath = ext[1:]
# we dont want to have '.frit/extractions' in the middle of the destination path:
basePath = os.path.dirname(filepath.replace('.frit/extractions/',''))
if merge:
Destination = unicode(os.path.join('.frit/extractions/by_extensions/',evi.configName,fs.configName,basePath))
else:
Destination = unicode(os.path.join('.frit/extractions/by_extensions/',evi.configName,fs.configName,extPath,basePath))
extractFile(filepath,Destination)
else:
logger.info('Nothing found to extract on "%s".' % (evi.configName + '/' + fs.configName))
