def test_interruptible(self):
def wait_until_interrupted(cv):
name = threading.currentThread().getName()
with cv:
while not JThread.currentThread().isInterrupted():
try:
cv.wait()
except InterruptedException as e:
break
num_threads = 5
unfair_condition = Condition()
threads = [
Thread(name="thread #%d" % i,
target=wait_until_interrupted,
args=(unfair_condition, )) for i in range(num_threads)
]
for thread in threads:
thread.start()
time.sleep(0.1)
for thread in threads:
JThread.interrupt(thread)
joined_threads = 0
for thread in threads:
thread.join(1.) # timeout just in case so we don't stall regrtest
joined_threads += 1
self.assertEqual(joined_threads, num_threads)
def test_interruptible(self):
def wait_until_interrupted(cv):
name = threading.currentThread().getName()
with cv:
while not JThread.currentThread().isInterrupted():
try:
cv.wait()
except InterruptedException as e:
break
num_threads = 5
unfair_condition = Condition()
threads = [
Thread(
name="thread #%d" % i,
target=wait_until_interrupted,
args=(unfair_condition,))
for i in range(num_threads)]
for thread in threads:
thread.start()
time.sleep(0.1)
for thread in threads:
JThread.interrupt(thread)
joined_threads = 0
for thread in threads:
thread.join(1.) # timeout just in case so we don't stall regrtest
joined_threads += 1
self.assertEqual(joined_threads, num_threads)
class JavaIntegrationTestCase(unittest.TestCase):
"""Verifies that Thread.__tojava__ correctly gets the underlying Java thread"""
def test_interruptible(self):
def wait_until_interrupted(cv):
name = threading.currentThread().getName()
with cv:
while not JThread.currentThread().isInterrupted():
try:
cv.wait()
except InterruptedException, e:
break
num_threads = 5
unfair_condition = Condition()
threads = [
Thread(
name="thread #%d" % i,
target=wait_until_interrupted,
args=(unfair_condition,))
for i in xrange(num_threads)]
for thread in threads:
thread.start()
time.sleep(0.1)
for thread in threads:
JThread.interrupt(thread)
joined_threads = 0
for thread in threads:
thread.join(1.) # timeout just in case so we don't stall regrtest
joined_threads += 1
self.assertEqual(joined_threads, num_threads)
def be_unfair():
unfair_condition = Condition()
threads = [
Thread(
name="thread #%d" % i,
target=wait_until_interrupted,
args=(unfair_condition,))
for i in xrange(5)]
for thread in threads:
thread.start()
time.sleep(5)
# threads should not be doing anything now, can verify by looking at some shared state
# instead of notifying, we will interrupt the threads
for thread in threads:
JThread.interrupt(thread)
# or you can use this equivalent op
# thread.__tojava__(JThread).interrupt()
for thread in threads:
thread.join()
def test_interruption(self):
# based on this code http://www.jython.org/jythonbook/en/1.0/Concurrency.html#interruption,
# which demonstrates __tojava__ works properly
class ExtendedThread(threading.Thread, ExtendedInterface):
def returnSomething(self):
return "yo yo yo"
def wait_until_interrupted(cv):
with cv:
while not Thread.currentThread().isInterrupted():
try:
# this condition variable is never notified, so will only
# succeed if interrupted
cv.wait()
except InterruptedException as e:
break
unfair_condition = threading.Condition()
threads = [
ExtendedThread(
name="thread #%d" % i,
target=wait_until_interrupted,
args=(unfair_condition,))
for i in range(5)]
for thread in threads:
thread.start()
for thread in threads:
Thread.interrupt(thread)
for thread in threads:
thread.join(5)
# this assertion only succeeds if threads terminated because
# they were interrupted
for thread in threads:
self.assertFalse(thread.isAlive())
class FDRIModule(DataSourceIngestModule):
_logger = Logger.getLogger(FDRIModuleFactory.moduleName)
def log(self, level, msg):
self._logger.logp(level, self.__class__.__name__,
inspect.stack()[1][3], msg)
def __init__(self, settings):
self.context = None
self.localSettings = settings
self.extensions = []
self.deleteAfter = False
self.doRecognition = True
self.userPaths = {
"0": "",
"1": "",
"2": ""
}
# True to create the DFXML file
self.createDFXML = C_CREATE_DFXML
# Time acumulators to determine the
# cumulative time needed to compute
# MD5, SHA1 and SHA256
self.needed_time_MD5 = 0.0
self.needed_time_SHA1 = 0.0
self.needed_time_SHA256 = 0.0
#CONFIGURATION_PATH = Case.getCurrentCase().getModuleDirectory() + "\\FDRI.json"
# Error list in acordance with .exe code
# for unknow errors please run executable via command line
self.errorList = {
1: ' FDRI.exe Parameters error',
2: ' Error loading parameter file ',
3: ' Error parsing parameter file ',
4: ' Error finding image directory ',
5: ' Error initializing recognition network ',
6: ' Error initializing shape predictor ',
7: ' Error initializing detection network ',
8: ' Didn\'t find any positive faces ',
9: ' Didn\'t find any target faces ',
10: ' CUDA out of memory ',
11: ' Didn\'t find any usable CUDA devices '
}
# Where any setup and configuration is done
# 'context' is an instance of org.sleuthkit.autopsy.ingest.IngestJobContext.
# See: http://sleuthkit.org/autopsy/docs/api-docs/4.4/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_ingest_job_context.html
def startUp(self, context):
# Supported file format from dlib
acceptedFiles = ['.jpg', '.jpeg', '.png']
i = 0
for ext in acceptedFiles:
if self.localSettings.getFlag(i):
self.extensions.append(ext)
i += 1
if not self.extensions:
raise IngestModuleException(
"Need to select at least one type of file!")
# self.generate_hash controls whether MD5,SHA1 and SHA256 hashes
# are added to the DFXML generated file.
if self.localSettings.getFlag(3):
self.generate_hash = True
else:
self.generate_hash = False
#
# Checking for default detectors and auxiliary files
#
self.pathToExe = os.path.join(os.path.dirname(
os.path.abspath(__file__)), "FDRI.exe")
self.defaultPaths = {
'0': os.path.join(os.path.dirname(os.path.abspath(__file__)),
"mmod_human_face_detector.dat"),
'1': os.path.join(os.path.dirname(os.path.abspath(__file__)),
"dlib_face_recognition_resnet_model_v1.dat"),
'2': os.path.join(os.path.dirname(os.path.abspath(__file__)),
"shape_predictor_5_face_landmarks.dat")
}
save_file = False
if os.path.exists(GLOBAL_CONFIGURATION_PATH):
with open(GLOBAL_CONFIGURATION_PATH, "r") as out:
content = json.load(out)
save_file = content['save_files']
self.userPaths = content['paths']
for code in self.userPaths:
if not self.userPaths[code]:
# user didn't set models path, we assume module location
self.userPaths[code] = self.defaultPaths[code]
# Update global config file
#self.log(Level.INFO, GLOBAL_CONFIGURATION_PATH)
with open(GLOBAL_CONFIGURATION_PATH, "w") as out:
json.dump({"save_files": save_file,
"paths": self.userPaths}, out)
folder_positive_photos = self.localSettings.getPath("1")
# No folder for positive photos was given: recognition OFF
if len(folder_positive_photos) == 0:
self.doRecognition = False
Msg_S = "Face recognition OFF (no folder with positive photo(s) given)"
self.log(Level.INFO,Msg_S)
elif not os.path.exists(folder_positive_photos):
# The folder with positive photo doesn' exist: recognition will
# be OFF
self.doRecognition = False
Msg_S = "Folder with positive photos NOT found: '%s'" %\
(folder_positive_photos)
self.log(Level.WARNING,Msg_S)
else:
# Ok, recognition is ON
self.doRecognition = True
Msg_S = "Face recognition ON (folder positive photo(s):'%s')"%\
(folder_positive_photos)
self.log(Level.INFO,Msg_S)
with open(Case.getCurrentCase().getModuleDirectory() + "\\config.json", 'w') as safe_file:
json.dump({"flags": self.localSettings.getAllFlags(), "wanted_folder": folder_positive_photos}, safe_file)
# Activate for DEBUG
#with open(CONFIGURATION_PATH, "w") as out:
# json.dump({"flags": self.localSettings.getAllFlags(),
# "paths": self.localSettings.getAllPaths()}, out)
self.context = context
#--------------------------------------------------------------------
# Added by Patricio
# 2018-07-21
#--------------------------------------------------------------------
def shutDown(self):
"""shutdown code"""
# Elaspsed time
FDRIModuleFactory.g_elapsed_time_secs = time.time() -\
FDRIModuleFactory.g_start_time
Log_S = "Total elapsed time: %f secs" %\
(FDRIModuleFactory.g_elapsed_time_secs)
self.log(Level.INFO, Log_S)
# Where the analysis is done.
# The 'dataSource' object being passed in is of type org.sleuthkit.datamodel.Content.
# See: http://www.sleuthkit.org/sleuthkit/docs/jni-docs/4.4.1/interfaceorg_1_1sleuthkit_1_1datamodel_1_1_content.html
# 'progressBar' is of type org.sleuthkit.autopsy.ingest.DataSourceIngestModuleProgress
# See: http://sleuthkit.org/autopsy/docs/api-docs/4.4/classorg_1_1sleuthkit_1_1autopsy_1_1ingest_1_1_data_source_ingest_module_progress.html
def process(self, dataSource, progressBar):
# we don't know how much work there is yet
progressBar.switchToIndeterminate()
# Start timer for file copy operation
start_copy_time = time.time()
# case insensitive SQL LIKE clause is used to query the case database
# FileManager API: http://sleuthkit.org/autopsy/docs/api-docs/4.4.1/classorg_1_1sleuthkit_1_1autopsy_1_1casemodule_1_1services_1_1_file_manager.html
fileManager = Case.getCurrentCase().getServices().getFileManager()
files = []
for extension in self.extensions:
try:
files.extend(fileManager.findFiles(
dataSource, "%" + extension))
except TskCoreException:
self.log(Level.INFO, "Error getting files from: '" +
extension + "'")
numFiles = len(files)
if not numFiles:
self.log(Level.WARNING, "Didn't find any usable files!")
return IngestModule.ProcessResult.OK
# Check if the user pressed cancel while we were busy
if self.context.isJobCancelled():
return IngestModule.ProcessResult.OK
output_dir = Case.getCurrentCase().getModuleDirectory()
module_dir = os.path.join(output_dir,dataSource.getName(),C_FDRI_DIR)
# Create top-level DIR to save FDIR's created files
full_dirname_dataSource = os.path.join(output_dir,dataSource.getName())
if not os.path.exists(full_dirname_dataSource):
os.mkdir(full_dirname_dataSource)
# TEMP is needed by Autopsy
temp_dir = os.path.join(Case.getCurrentCase().getTempDirectory(),
dataSource.getName())
if not os.path.exists(temp_dir):
os.mkdir(temp_dir)
temp_dir = os.path.join(temp_dir, C_FDRI_DIR)
if not os.path.exists(temp_dir):
os.mkdir(temp_dir)
# We always copy the files (except if a copy already exists)
# as we will want to change them.
# We detect the existence of a previous copy if the creation of the dir
# 'module_dir' triggers an exception
try:
os.mkdir(module_dir)
except:
self.log(Level.INFO, "Directory already exists for this module")
#----------------------------------------
# Init file which holds filenames + size
#----------------------------------------
file_path = os.path.join(module_dir,C_FILE_WITH_FNAMES_AND_SIZES)
fnames_and_sizes_F = open(file_path,"w")
fnames_and_sizes_F.write(C_SEP_S)
fnames_and_sizes_F.write("# Filename:size (bytes)\n")
timestamp_S = datetime.now().strftime('%Y-%m-%d_%Hh%Mm%Ss')
fnames_and_sizes_F.write("# START: %s\n" % (timestamp_S))
fnames_and_sizes_F.write(C_SEP_S)
# Dict to detect identical files
files_hash_D = {}
# Flag to record whether files were copied or not
were_files_copied = True
# Minimum size (in bytes) for an image file to be processed
total_files = 0
total_small_files = 0
# A initial version mispelled 'Annotated"...
avoid_prefix_1 = "Anotated_"
avoid_prefix_2 = "Annotated_"
try:
dir_img = os.path.join(module_dir,"img")
os.mkdir(dir_img)
dir_small_files = os.path.join(module_dir,"small_files") + "\\"
os.mkdir(dir_small_files)
for file in files:
total_files = total_files + 1
filename_S = file.getName()
Log_S = ""
if filename_S.find(avoid_prefix_1) is 0:
Log_S = "%s file found '%s': skipping" %\
(avoid_prefix_1,filename_S)
elif filename_S.find(avoid_prefix_2) is 0:
Log_S = "%s file found '%s': skipping" %\
(avoid_prefix_2,filename_S)
if len(Log_S):
# Annotated_ found
# Log and skip this file
self.log(Level.INFO, Log_S)
continue
file_size = file.getSize()
filename, file_extension = os.path.splitext(file.getName())
# Record filename and file size in C_FILE_WITH_FNAMES_AND_SIZES
fnames_and_sizes_F.write("%s:%d\n" %(file.getName(),file_size))
# If file size is more than C_FILE_MIN_SIZE
# TODO:: User Choice as option
if file_size >= C_FILE_MIN_SIZE:
new_fname = "%s__id__%s%s" %\
(filename,str(file.getId()),file_extension)
fullpath_dest = os.path.join(dir_img,new_fname)
ContentUtils.writeToFile(file, File(fullpath_dest))
# We copy small files to a different DIR, so that we
# can look at them, if needed
if file_size < C_FILE_MIN_SIZE:
total_small_files = total_small_files + 1
dest_filename = "%s%s__id__%d%s" %\
(dir_small_files,filename,file.getId(),file_extension)
ContentUtils.writeToFile(file, File(dest_filename))
Log_S = "Skipping file: %s (%d bytes)" %\
(file.getName(),file.getSize())
# LOG
self.log(Level.INFO, Log_S)
#--------------------------------
# Code to detect repeated files
# We simply use a dictionary
# keyed by the MD5 of the file
# Patricio
#--------------------------------
if file_size > 0:
md5_hash = self.create_hash(file, "md5")
if md5_hash in files_hash_D:
# hash already exists: repetition
files_hash_D[md5_hash].append(file.getName())
else:
# hash doesn't yet exist in dictionary: 1st time
files_hash_D[md5_hash] = [file_size,file.getName()]
##except:
except Exception, e:
were_files_copied = False
self.log(Level.INFO,"Image folder already exists, skiping file copy")
self.log(Level.INFO,"Exception: " + str(e))
#----------------------------------------
# Close filename+size file
# Patricio
#----------------------------------------
timestamp_S = datetime.now().strftime('%Y-%m-%d_%Hh%Mm%Ss')
fnames_and_sizes_F.write("# DONE: %s\n" % (timestamp_S))
if were_files_copied is False:
Msg_S = "# Exception occurred\n"
fnames_and_sizes_F.write(Msg_S)
fnames_and_sizes_F.close()
#----------------------------------------
# Dump hash with repeated files
# (only if files were copied)
#----------------------------------------
if were_files_copied is True:
file_path = os.path.join(module_dir,C_REPEATED_FILES_LOG)
repeated_files_log_F = open(file_path,"w")
repeated_files_log_F.write(C_SEP_S)
repeated_files_log_F.write("# Repeated files\n")
timestamp_S = datetime.now().strftime('%Y-%m-%d_%Hh%Mm%Ss')
repeated_files_log_F.write("# %s\n" % (timestamp_S))
repeated_files_log_F.write(C_SEP_S)
for key, info_L in files_hash_D.iteritems():
if len(info_L) > 2:
# only list with more than 2 entries
# (one entry is the file size)
S = ""
for datum in info_L:
S = "%s%s:" % (S,datum)
repeated_files_log_F.write("%s\n" %(S))
repeated_files_log_F.write(C_SEP_S)
repeated_files_log_F.write("# DONE: %s\n" % (timestamp_S))
repeated_files_log_F.write(C_SEP_S)
repeated_files_log_F.close()
#----------------------------------------
# Log stats
#----------------------------------------
# shutdown copy file timer
elapsed_copy_time_secs = time.time() - start_copy_time
Log_S = "%d image files (%d of these were left out -- size <= "\
"%d bytes)" % (total_files, total_small_files,C_FILE_MIN_SIZE)
self.log(Level.INFO, Log_S)
total_copied_files = total_files - total_small_files
Log_S = "Files copy operation (%d files) took %f secs" %\
(total_copied_files,elapsed_copy_time_secs)
self.log(Level.INFO, Log_S)
#----------------------------------------
# Start processing timer
#----------------------------------------
start_FDRIexe_time = time.time()
# Location where the output of executable will appear
timestamp = datetime.now().strftime('%Y-%m-%d_%Hh%Mm%Ss')
workspace = os.path.join(module_dir,timestamp)
configFilePath = os.path.join(workspace,C_PARAMS_JSON_FNAME)
os.mkdir(workspace)
with open(configFilePath, "w") as out:
json.dump({
"paths": self.userPaths,#self.localSettings.getAllPaths(),
"wanted_faces" : self.localSettings.getPath("1"),
"imagesPath": os.path.join(module_dir,"img"),
"doRecognition": self.doRecognition,
"workspace": workspace,
}, out)
#
# Different calls can also be provided to specify the image size
#
# Note that 2GB of GPU memory handle around 2000*2000 images
# Note that 4GB of GPU memory handle around 3500*3500 images
# Note that 8GB of GPU memory handle around 6000*6000 images
#
# Example:
# Required Minimum size Maximum size
# target=lambda: self.thread_work(self.pathToExe, configFilePath, 1200*1200, 2000*2000))
# target=lambda: self.thread_work(self.pathToExe, configFilePath, 1200*1200))
executable_thread = Thread(
target=lambda: self.thread_work(self.pathToExe, configFilePath))
executable_thread.start()
while(executable_thread.isAlive()):
if self.context.isJobCancelled():
self.log(Level.INFO, "User cancelled job! Terminating thread")
JThread.interrupt(executable_thread)
self.log(Level.INFO, "Thread terminated")
self.deleteFiles(module_dir)
return IngestModule.ProcessResult.OK
time.sleep(1)
# Checking if cancel was pressed before starting another job
if self.context.isJobCancelled():
return IngestModule.ProcessResult.OK
#----------------------------------------
# Compute time takne by FDRI.exe
#----------------------------------------
elapsed_FDRIexe_time_secs = time.time() - start_FDRIexe_time
Log_S = "Process of image files by FDRI.exe took %f secs" %\
(elapsed_FDRIexe_time_secs)
self.log(Level.INFO, Log_S)
#----------------------------------------
# Start timer of last stage
#----------------------------------------
self.log(Level.INFO, "START of last stage")
start_last_stage_time = time.time()
# Use blackboard class to index blackboard artifacts for keyword search
blackboard = Case.getCurrentCase().getServices().getBlackboard()
# Tag files with faces
artifact_type = BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT
# Copy files from workspace to temp_dir
tree_source = os.path.join(workspace, C_ANNOTATED_DIR)
tree_destination = os.path.join(temp_dir, C_ANNOTATED_DIR)
copy_tree(tree_source,tree_destination)
# Add images with the wanted faces to blackboard
outPositiveFile = os.path.join(workspace,C_FDRI_WANTED_FNAME)
if os.path.exists(outPositiveFile):
with open(outPositiveFile, "r") as out:
for line in out:
file_id = line.split('__id__')[1].split('.')
interestingFile = self.findByID(files, file_id[0])
if interestingFile == None:
continue
# Creating new artifacts with faces found
artifactList = interestingFile.getArtifacts(artifact_type)
if artifactList:
self.log(
Level.INFO, "Artifact already exists! ignoring")
else:
art = interestingFile.newArtifact(artifact_type)
att = BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID(),
FDRIModuleFactory.moduleName, dataSource.getName() + "/Wanted faces")
art.addAttribute(att)
try:
# index the artifact for keyword search
blackboard.indexArtifact(art)
except Blackboard.BlackboardException as e:
self.log(
Level.SEVERE, "Error indexing artifact " + art.getDisplayName())
# Adding derivated files to case
# These are files with borders on the found faces
# Code to deal with filenames with multiple "."
# Patricio, 2018.08.09
interestingFName = interestingFile.getName()
try:
name, extension = self.split_fname(interestingFName)
except Exception, e:
Err_S = "Error in splitting name/extension of '%s' (skipping file)" % (interestingFName)
self.log(Level.SEVERE,Err_S)
self.log(Level.SEVERE,"Exception: " + str(e))
continue
# Still here? Good.
f_path = "%s__id__%s.%s" %\
(name,str(interestingFile.getId()),extension)
# We need path relative to temp folder for Autopsy API
f_temp_path = os.path.join("Temp",dataSource.getName(),
C_FDRI_DIR, C_ANNOTATED_DIR, f_path)
f_abs_path = os.path.join(workspace,
C_ANNOTATED_DIR, f_path)
# Temporary fix
if os.path.exists(f_abs_path):
f_size = os.path.getsize(f_abs_path)
case = Case.getCurrentCase().getSleuthkitCase()
try:
abstract_f = case.getAbstractFileById(
interestingFile.getId())
# https://sleuthkit.org/autopsy/docs/api-docs/4.4/classorg_1_1sleuthkit_1_1autopsy_1_1casemodule_1_1services_1_1_file_manager.html
label_S = C_ANNOTATED_LABEL + interestingFName
case.addDerivedFile(label_S, f_temp_path,
f_size, 0, 0, 0, 0, True, abstract_f,
"", FDRIModuleFactory.moduleName,
FDRIModuleFactory.moduleVersion,
"Image with faces",
TskData.EncodingType.NONE)
except:
self.log(Level.SEVERE,"Error getting abs file")
if self.generate_hash:
dfxml_path = os.path.join(workspace,C_DFXML_FNAME)
self.complete_dfxml( dfxml_path, interestingFile)
class ChooseCorrectToJavaTest(unittest.TestCase):
# Verifies fix for http://bugs.jython.org/issue1795
#
# Note that we use threading.Thread because we have imported
# java.lang.Thread as Thread
def test_extended_thread(self):
class ExtendedThread(threading.Thread, ExtendedInterface):
def returnSomething(self):
return "yo yo yo"
result = [None]
def f(r):
r[0] = 47
t = ExtendedThread(target=f, args=(result, ))
self.assertEqual(UseExtendedInterface().countWords(t), 3)
# Also verify that t still works as a regular thread
t.start()
t.join()
self.assertFalse(t.isAlive())
self.assertEqual(result[0], 47)
def test_interruption(self):
# based on this code http://www.jython.org/jythonbook/en/1.0/Concurrency.html#interruption,
# which demonstrates __tojava__ works properly
class ExtendedThread(threading.Thread, ExtendedInterface):
def returnSomething(self):
return "yo yo yo"
def wait_until_interrupted(cv):
with cv:
while not Thread.currentThread().isInterrupted():
try:
# this condition variable is never notified, so will only
# succeed if interrupted
cv.wait()
except InterruptedException, e:
break
unfair_condition = threading.Condition()
threads = [
ExtendedThread(name="thread #%d" % i,
target=wait_until_interrupted,
args=(unfair_condition, )) for i in xrange(5)
]
for thread in threads:
thread.start()
for thread in threads:
Thread.interrupt(thread)
for thread in threads:
thread.join(5)
# this assertion only succeeds if threads terminated because
# they were interrupted
for thread in threads:
self.assertFalse(thread.isAlive())