def listExtensions(Evidences, extlist, states):
"""
A function to print a count and a size sum of the specified extension
for all filsystems of specified Evidences list.
"""
totalExt = {}
for ext in extlist:
totalExt[ext] = [0, 0]
grandTotalNb = 0
grandTotalSize = 0
for evi in Evidences:
fritutils.termout.printNormal('%s (%s)' %
(evi.configName, evi.fileName))
for filesystem in evi.fileSystems:
fritutils.termout.printNormal('\t%s' % filesystem.configName)
totalSize = 0
totalNumber = 0
fso = filesystem.getFsDb()
for ext in extlist:
for state in states:
fq = File.query.filter(File.filesystem == fso)
fq = fq.filter(File.state.has(state=state))
fq = fq.filter(File.extension.has(extension=ext))
nb = fq.count()
if nb > 0:
size = fq.value(func.sum(File.filesize))
fritutils.termout.printNormal(
'\t\t%s %d (%s) (state: %s)' %
(ext, nb, fritutils.humanize(size), state))
totalSize += size
totalNumber += nb
grandTotalNb += nb
grandTotalSize += size
if ext in totalExt.keys():
totalExt[ext][0] += nb
totalExt[ext][1] += size
else:
totalExt[ext] = [nb, size]
fritutils.termout.printNormal('\t\tFilesystem Total Files : %d' %
totalNumber)
fritutils.termout.printNormal('\t\tFilesystem Total Size : %s' %
fritutils.humanize(totalSize))
fritutils.termout.printSuccess('Summary:')
for ext in extlist:
fritutils.termout.printNormal(
'\t%s %d %s' %
(ext, totalExt[ext][0], fritutils.humanize(totalExt[ext][1])))
fritutils.termout.printNormal(
'Total files (size): %d (%s)' %
(grandTotalNb, fritutils.humanize(grandTotalSize)))
def vshadowList(Evidences, ags, options):
for evi in Evidences:
for fs in evi.fileSystems:
fs.mount('vshadow', 'Used by vshadow command')
if pyvshadow.check_volume_signature(fs.loopDevice):
fritutils.termout.printSuccess(
"Volume shadow copy found on '{}/{}'".format(
fs.evidenceConfigName, fs.configName))
vshadowVol = pyvshadow.volume()
vshadowVol.open(fs.loopDevice)
fritutils.termout.printNormal(
" Number of stores on volume: {}".format(
vshadowVol.number_of_stores))
for st in vshadowVol.get_stores():
fritutils.termout.printNormal(
" Store identifier: {}".format(st.identifier))
fritutils.termout.printNormal(
" Store creation time: {}".format(
st.get_creation_time()))
fritutils.termout.printNormal(
" Store size: {}".format(
fritutils.humanize(st.size)))
fritutils.termout.printNormal(
" Shadow-copy set ID: {}".format(
st.copy_set_identifier))
fritutils.termout.printNormal(
" Shadow-copy ID: {}".format(
st.copy_identifier))
vshadowVol.close()
fs.umount('vshadow')
if evi.isMounted():
evi.umount('vshadow')
def fullFileSpec(self,hashtype='md5'):
fp = os.path.join(self.fullpath.fullpath,self.filename)
fsize = fritutils.humanize(self.filesize)
hashes = { 'md5' : self.md5.md5, 'sha1': self.sha1.sha1, 'sha256': self.sha256.sha256, 'ssdeep': self.ssdeep.ssdeep }
fhash = 'NO_HASH_COMPUTED'
if hashes[hashtype]:
fhash = hashes[hashtype]
specs = '%s,%s,%s: "%s" ,%s/%s,"%s"' % (self.state.state, fsize,hashtype, fhash , self.evidence.configName, self.filesystem.configName,fp)
return specs
def listExtensions(Evidences,extlist,states):
"""
A function to print a count and a size sum of the specified extension
for all filsystems of specified Evidences list.
"""
totalExt = {}
for ext in extlist:
totalExt[ext] = [0,0]
grandTotalNb = 0
grandTotalSize = 0
for evi in Evidences:
fritutils.termout.printNormal('%s (%s)' % (evi.configName,evi.fileName))
for filesystem in evi.fileSystems:
fritutils.termout.printNormal('\t%s' % filesystem.configName)
totalSize = 0
totalNumber = 0
fso = filesystem.getFsDb()
for ext in extlist:
for state in states:
fq = File.query.filter(File.filesystem==fso)
fq = fq.filter(File.state.has(state=state))
fq = fq.filter(File.extension.has(extension=ext))
nb = fq.count()
if nb >0 :
size=fq.value(func.sum(File.filesize))
fritutils.termout.printNormal('\t\t%s %d (%s) (state: %s)' % (ext,nb,fritutils.humanize(size),state))
totalSize += size
totalNumber += nb
grandTotalNb += nb
grandTotalSize += size
if ext in totalExt.keys():
totalExt[ext][0] += nb
totalExt[ext][1] += size
else:
totalExt[ext] = [nb, size]
fritutils.termout.printNormal('\t\tFilesystem Total Files : %d' % totalNumber)
fritutils.termout.printNormal('\t\tFilesystem Total Size : %s' % fritutils.humanize(totalSize))
fritutils.termout.printSuccess('Summary:')
for ext in extlist:
fritutils.termout.printNormal('\t%s %d %s' % (ext, totalExt[ext][0], fritutils.humanize(totalExt[ext][1])))
fritutils.termout.printNormal('Total files (size): %d (%s)' % (grandTotalNb, fritutils.humanize(grandTotalSize)))
def fullFileSpec(self, hashtype='md5'):
fp = os.path.join(self.fullpath.fullpath, self.filename)
fsize = fritutils.humanize(self.filesize)
hashes = {
'md5': self.md5.md5,
'sha1': self.sha1.sha1,
'sha256': self.sha256.sha256,
'ssdeep': self.ssdeep.ssdeep
}
fhash = 'NO_HASH_COMPUTED'
if hashes[hashtype]:
fhash = hashes[hashtype]
specs = '%s,%s,%s: "%s" ,%s/%s,"%s"' % (
self.state.state, fsize, hashtype, fhash, self.evidence.configName,
self.filesystem.configName, fp)
return specs
def vshadowList(Evidences, ags, options):
for evi in Evidences:
for fs in evi.fileSystems:
fs.mount('vshadow','Used by vshadow command')
if pyvshadow.check_volume_signature(fs.loopDevice):
fritutils.termout.printSuccess("Volume shadow copy found on '{}/{}'".format(fs.evidenceConfigName,fs.configName))
vshadowVol = pyvshadow.volume()
vshadowVol.open(fs.loopDevice)
fritutils.termout.printNormal(" Number of stores on volume: {}".format(vshadowVol.number_of_stores))
for st in vshadowVol.get_stores():
fritutils.termout.printNormal(" Store identifier: {}".format(st.identifier))
fritutils.termout.printNormal(" Store creation time: {}".format(st.get_creation_time()))
fritutils.termout.printNormal(" Store size: {}".format(fritutils.humanize(st.size)))
fritutils.termout.printNormal(" Shadow-copy set ID: {}".format(st.copy_set_identifier))
fritutils.termout.printNormal(" Shadow-copy ID: {}".format(st.copy_identifier))
vshadowVol.close()
fs.umount('vshadow')
if evi.isMounted():
evi.umount('vshadow')
def factory(Evidences, args, options, fritConf):
validArgs = ('count', 'extract', 'list')
stateOptions = {
'--normal': u'Normal',
'--contained': u'Contained',
'--undeleted': u'Undeleted',
'--carved': u'Carved'
}
definedExtensions = getExtLists(fritConf)
if not fritModel.dbExists():
fritutils.termout.printWarning(
'The database does not exists yet. You should create it first by issuing "frit store create".'
)
logger.warning('Database was not found')
sys.exit(1)
states = []
extList = []
if not args or len(args) == 0:
fritutils.termout.printWarning(
'extensions command need at least an argument to define an action (%s).'
% ', '.join(validArgs))
sys.exit(1)
elif args[0] not in validArgs:
fritutils.termout.printWarning(
'extensions command need a valid argument (%s)' %
', '.join(validArgs))
sys.exit(1)
else:
subcommand = args[0]
args.remove(subcommand)
logger.info('subcommand issued: %s' % subcommand)
if options:
logger.info('options: %s' % ','.join(options))
for o in options:
if o in stateOptions.keys():
states.append(stateOptions[o])
if len(states) == 0:
states = list(fritModel.FILESTATES)
logger.info('states: %s' % ','.join(states))
# Finding extensions to work with
# Searching if one or more predefined extensions list is in the args
for a in list(args):
if a in definedExtensions.keys():
logger.info('Extension list "%s" asked in command line.' %
args)
args.remove(a)
extList.extend(definedExtensions[a])
# the remaining args should be the extensions that we want to list
# if there is no more args, we list all extensions
if (not args or len(args) == 0) and len(extList) == 0:
for ex in fritModel.elixir.session.query(
fritModel.Extension.extension).all():
extList.append(ex[0])
else:
for ex in args:
extList.append(fritutils.unicodify(ex))
logger.info('Extensions: "%s"' % " ".join(extList))
if subcommand == 'count':
logger.info('Starting subcommand count')
fritModel.listExtensions(Evidences, extList, states)
elif subcommand == 'list':
logger.info('Starting list subcommand.')
for evi in Evidences:
for fs in evi.fileSystems:
for ext in sorted(extList):
for state in states:
for fp in fs.ExtensionsFritFiles(ext, state):
fritutils.termout.printNormal(fp)
elif subcommand == 'extract':
logger.info('Starting extract subcommand')
# The '--merge' option is used to merge extractions in a single
# directory base instead of having a directory by extension.
merge = False
if options and '--merge' in options:
merge = True
# we start by extracting 'normal files' because we need to mount the containers and filesystems
if u'Normal' in states:
logger.info('Starting Normal files extraction.')
states.remove(u'Normal')
for evi in Evidences:
# We count files to extract to see if it's needed to go further
enbe = evi.dbCountExtension(extList, u'Normal')
if enbe['count'] > 0:
logger.info(
'Found %d files to exctract, mounting Evidence container "%s".'
% (enbe['count'], evi.configName))
evi.mount('extensions',
'Extracting files based on extensions')
for fs in evi.fileSystems:
fritutils.termout.printMessage(
"\t%s" % fs.evidence.configName + '/' +
fs.configName)
fs.mount('extensions',
'Extracting files based on extensions')
for ext in sorted(extList):
nbe = fs.dbCountExtension(ext, u'Normal')
fritutils.termout.printMessage(
"Extracting %d files (%s)" %
(nbe['count'],
fritutils.humanize(nbe['size'])))
for filepath in fs.ExtensionsOriginalFiles(
ext, u'Normal'):
if ext == "No Extension":
extPath = "no_extension"
else:
extPath = ext[1:]
basePath = os.path.dirname(filepath)
if merge:
Destination = unicode(
os.path.join(
'.frit/extractions/by_extensions/',
evi.configName, fs.configName,
basePath))
else:
Destination = unicode(
os.path.join(
'.frit/extractions/by_extensions/',
evi.configName, fs.configName,
extPath, basePath))
mountedPath = os.path.join(
fs.fsMountPoint, filepath)
extractFile(mountedPath, Destination)
fs.umount('extensions')
evi.umount('extensions')
else:
logger.info(
'No Normal files to extract on Evidence "%s", skipping'
% evi.configName)
for state in states:
logger.info('Starting to extract %s files' % state)
for evi in Evidences:
for fs in evi.fileSystems:
for ext in sorted(extList):
nbe = fs.dbCountExtension(ext, state)
if nbe['count'] > 0:
fritutils.termout.printMessage(
"Extracting %s %d files (%s)" %
(state, nbe['count'],
fritutils.humanize(nbe['size'])))
for filepath in fs.ExtensionsOriginalFiles(
ext, state):
# as we do not store the first character of the path, we have to re-add the '.'
filepath = '.' + filepath
if ext == "No Extension":
extPath = "no_extension"
else:
extPath = ext[1:]
# we dont want to have '.frit/extractions' in the middle of the destination path:
basePath = os.path.dirname(
filepath.replace(
'.frit/extractions/', ''))
if merge:
Destination = unicode(
os.path.join(
'.frit/extractions/by_extensions/',
evi.configName, fs.configName,
basePath))
else:
Destination = unicode(
os.path.join(
'.frit/extractions/by_extensions/',
evi.configName, fs.configName,
extPath, basePath))
extractFile(filepath, Destination)
else:
logger.info(
'Nothing found to extract on "%s".' %
(evi.configName + '/' + fs.configName))
def factory(Evidences, args, options, fritConf):
validArgs = ('count', 'extract','list')
stateOptions = {'--normal':u'Normal','--contained':u'Contained','--undeleted':u'Undeleted','--carved':u'Carved'}
definedExtensions = getExtLists(fritConf)
if not fritModel.dbExists():
fritutils.termout.printWarning('The database does not exists yet. You should create it first by issuing "frit store create".')
logger.warning('Database was not found')
sys.exit(1)
states = []
extList = []
if not args or len(args) == 0:
fritutils.termout.printWarning('extensions command need at least an argument to define an action (%s).' % ', '.join(validArgs))
sys.exit(1)
elif args[0] not in validArgs:
fritutils.termout.printWarning('extensions command need a valid argument (%s)' % ', '.join(validArgs))
sys.exit(1)
else:
subcommand = args[0]
args.remove(subcommand)
logger.info('subcommand issued: %s' % subcommand)
if options:
logger.info('options: %s' % ','.join(options))
for o in options:
if o in stateOptions.keys():
states.append(stateOptions[o])
if len(states) == 0:
states = list(fritModel.FILESTATES)
logger.info('states: %s' % ','.join(states))
# Finding extensions to work with
# Searching if one or more predefined extensions list is in the args
for a in list(args):
if a in definedExtensions.keys():
logger.info('Extension list "%s" asked in command line.' % args)
args.remove(a)
extList.extend(definedExtensions[a])
# the remaining args should be the extensions that we want to list
# if there is no more args, we list all extensions
if (not args or len(args) == 0) and len(extList) == 0:
for ex in fritModel.elixir.session.query(fritModel.Extension.extension).all():
extList.append(ex[0])
else:
for ex in args:
extList.append(fritutils.unicodify(ex))
logger.info('Extensions: "%s"' % " ".join(extList))
if subcommand == 'count':
logger.info('Starting subcommand count')
fritModel.listExtensions(Evidences,extList,states)
elif subcommand == 'list':
logger.info('Starting list subcommand.')
for evi in Evidences:
for fs in evi.fileSystems:
for ext in sorted(extList):
for state in states:
for fp in fs.ExtensionsFritFiles(ext,state):
fritutils.termout.printNormal(fp)
elif subcommand == 'extract':
logger.info('Starting extract subcommand')
# The '--merge' option is used to merge extractions in a single
# directory base instead of having a directory by extension.
merge = False
if options and '--merge' in options:
merge = True
# we start by extracting 'normal files' because we need to mount the containers and filesystems
if u'Normal' in states:
logger.info('Starting Normal files extraction.')
states.remove(u'Normal')
for evi in Evidences:
# We count files to extract to see if it's needed to go further
enbe = evi.dbCountExtension(extList, u'Normal')
if enbe['count'] > 0:
logger.info('Found %d files to exctract, mounting Evidence container "%s".' % (enbe['count'],evi.configName))
evi.mount('extensions', 'Extracting files based on extensions')
for fs in evi.fileSystems:
fritutils.termout.printMessage("\t%s" % fs.evidence.configName + '/' + fs.configName)
fs.mount('extensions', 'Extracting files based on extensions')
for ext in sorted(extList):
nbe = fs.dbCountExtension(ext,u'Normal')
fritutils.termout.printMessage("Extracting %d files (%s)" % (nbe['count'],fritutils.humanize(nbe['size'])))
for filepath in fs.ExtensionsOriginalFiles(ext,u'Normal'):
if ext == "No Extension":
extPath = "no_extension"
else:
extPath = ext[1:]
basePath = os.path.dirname(filepath)
if merge:
Destination = unicode(os.path.join('.frit/extractions/by_extensions/',evi.configName,fs.configName,basePath))
else:
Destination = unicode(os.path.join('.frit/extractions/by_extensions/',evi.configName,fs.configName,extPath,basePath))
mountedPath = os.path.join(fs.fsMountPoint,filepath)
extractFile(mountedPath,Destination)
fs.umount('extensions')
evi.umount('extensions')
else:
logger.info('No Normal files to extract on Evidence "%s", skipping' % evi.configName)
for state in states:
logger.info('Starting to extract %s files' % state)
for evi in Evidences:
for fs in evi.fileSystems:
for ext in sorted(extList):
nbe = fs.dbCountExtension(ext,state)
if nbe['count'] >0 :
fritutils.termout.printMessage("Extracting %s %d files (%s)" % (state,nbe['count'],fritutils.humanize(nbe['size'])))
for filepath in fs.ExtensionsOriginalFiles(ext,state):
# as we do not store the first character of the path, we have to re-add the '.'
filepath = '.' + filepath
if ext == "No Extension":
extPath = "no_extension"
else:
extPath = ext[1:]
# we dont want to have '.frit/extractions' in the middle of the destination path:
basePath = os.path.dirname(filepath.replace('.frit/extractions/',''))
if merge:
Destination = unicode(os.path.join('.frit/extractions/by_extensions/',evi.configName,fs.configName,basePath))
else:
Destination = unicode(os.path.join('.frit/extractions/by_extensions/',evi.configName,fs.configName,extPath,basePath))
extractFile(filepath,Destination)
else:
logger.info('Nothing found to extract on "%s".' % (evi.configName + '/' + fs.configName))
