def tika_extract(fullpath, context, metadata, config, rcontext):
"""
Use the Tika input stream and extract all embedded files (if possible). Invokes Uforia
recursively over the extracted files.
fullpath - Path of the file to extract
context - The Tika parse context
metadata - Tika metadata object
oonfig - The Uforia configuration file
rcontext - The Uforia recursion context variables
"""
# To skip recursive call if there are no files to extract
extractor = tika.ParsingEmbeddedDocumentExtractor(context)
needs_extraction = extractor.shouldParseEmbedded(metadata)
if needs_extraction:
# Call Uforia recursively on embedded files
tempdir = None
try:
# Perform extraction
tempdir = tempfile.mkdtemp(dir=config.EXTRACTDIR)
_do_tika_extract(fullpath, tempdir)
# Call Uforia again
recursive.call_uforia_recursive(config, rcontext, tempdir, fullpath)
except:
traceback.print_exc(file=sys.stderr)
finally:
try:
if tempdir:
shutil.rmtree(tempdir) # delete directory
except OSError as exc:
traceback.print_exc(file=sys.stderr)
def tika_extract(fullpath, context, metadata, config, rcontext):
"""
Use the Tika input stream and extract all embedded files (if possible). Invokes Uforia
recursively over the extracted files.
fullpath - Path of the file to extract
context - The Tika parse context
metadata - Tika metadata object
oonfig - The Uforia configuration file
rcontext - The Uforia recursion context variables
"""
# To skip recursive call if there are no files to extract
extractor = tika.ParsingEmbeddedDocumentExtractor(context)
needs_extraction = extractor.shouldParseEmbedded(metadata)
if needs_extraction:
# Call Uforia recursively on embedded files
tempdir = None
try:
# Perform extraction
tempdir = tempfile.mkdtemp(dir=config.EXTRACTDIR)
_do_tika_extract(fullpath, tempdir)
# Call Uforia again
recursive.call_uforia_recursive(config, rcontext, tempdir,
fullpath)
except:
traceback.print_exc(file=sys.stderr)
finally:
try:
if tempdir:
shutil.rmtree(tempdir) # delete directory
except OSError as exc:
traceback.print_exc(file=sys.stderr)
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
if file.btype.startswith("Microsoft Outlook email folder"):
readpst_path = libutil.get_executable("libpst", "readpst")
tempdir = None
try:
tempdir = tempfile.mkdtemp(dir=config.EXTRACTDIR)
p = subprocess.Popen(
[readpst_path, '-e', '-q', '-o', tempdir, fullpath])
err = p.communicate()[1]
if err is not None:
raise Exception("readpst failed to extract " + fullpath)
recursive.call_uforia_recursive(config, rcontext, tempdir,
fullpath)
return [fullpath]
except:
traceback.print_exc(file=sys.stderr)
return None
finally:
try:
if tempdir:
pass
#shutil.rmtree(tempdir) # delete directory
except OSError as exc:
traceback.print_exc(file=sys.stderr)
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
# Try to parse 7z data
try:
# Get instance of 7z module
zip_module = imp.load_source('7zfilerecursor',
'modules/application/' +
'x-7z-compressed/7zfilerecursor.py')
file = open(fullpath, 'rb')
assorted = _get_volume_descriptors(file)
file.close()
# Try to extract the content of the 7zip file.
try:
# Create a temporary directory
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
# Extract the 7zip file
zip_module._extractall(fullpath, tmpdir)
recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath)
except:
traceback.print_exc(file=sys.stderr)
# Delete the temporary directory, proceed even if it causes
# an error
try:
shutil.rmtree(tmpdir)
except:
traceback.print_exc(file=sys.stderr)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
return assorted
except:
traceback.print_exc(file=sys.stderr)
# Store values in database so not the whole application crashes
return None
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
# Try to parse 7z data
try:
# Get instance of 7z module
zip_module = imp.load_source(
'7zfilerecursor',
'modules/application/' + 'x-7z-compressed/7zfilerecursor.py')
file = open(fullpath, 'rb')
assorted = _get_volume_descriptors(file)
file.close()
# Try to extract the content of the 7zip file.
try:
# Create a temporary directory
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
# Extract the 7zip file
zip_module._extractall(fullpath, tmpdir)
recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath)
except:
traceback.print_exc(file=sys.stderr)
# Delete the temporary directory, proceed even if it causes
# an error
try:
shutil.rmtree(tmpdir)
except:
traceback.print_exc(file=sys.stderr)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
return assorted
except:
traceback.print_exc(file=sys.stderr)
# Store values in database so not the whole application crashes
return None
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
try:
# Create a temporary directory
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
# Open gzip file for reading
file = gzip.open(fullpath, 'rb')
# Store gzip metadata values
assorted = [file.extrabuf,
file.extrasize,
file.extrastart]
# Read the uncompressed data
file_content = file.read()
# Write it to the temp folder
uncompressed_file = open(tmpdir + os.path.sep
+ _uncompressed_filename(fullpath),
"wb")
uncompressed_file.write(file_content)
# Close both files
uncompressed_file.close()
file.close()
# Call Uforia recursively
recursive.call_uforia_recursive(config, rcontext, tmpdir,
os.path.dirname(fullpath))
# Delete the temporary directory, proceed even if it causes
# an error
try:
shutil.rmtree(tmpdir)
except:
traceback.print_exc(file=sys.stderr)
return assorted
except:
traceback.print_exc(file=sys.stderr)
return None
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
try:
# Create a temporary directory
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
# Open gzip file for reading
file = gzip.open(fullpath, 'rb')
# Store gzip metadata values
assorted = [file.extrabuf, file.extrasize, file.extrastart]
# Read the uncompressed data
file_content = file.read()
# Write it to the temp folder
uncompressed_file = open(
tmpdir + os.path.sep + _uncompressed_filename(fullpath), "wb")
uncompressed_file.write(file_content)
# Close both files
uncompressed_file.close()
file.close()
# Call Uforia recursively
recursive.call_uforia_recursive(config, rcontext, tmpdir,
os.path.dirname(fullpath))
# Delete the temporary directory, proceed even if it causes
# an error
try:
shutil.rmtree(tmpdir)
except:
traceback.print_exc(file=sys.stderr)
return assorted
except:
traceback.print_exc(file=sys.stderr)
return None
def xpdf_extract(fullpath, config, rcontext):
"""
Extract the images of the specified PDF file with xpdf_extract
fullpath - Path of the pdf file to extract images from
config - The Uforia configuration file
rcontext - The Uforia recursion context variables
"""
tempdir = None
try:
# Perform extraction
tempdir = tempfile.mkdtemp(dir=config.EXTRACTDIR)
_do_xpdf_extract(fullpath, tempdir)
# Call Uforia again
recursive.call_uforia_recursive(config, rcontext, tempdir, fullpath)
except:
traceback.print_exc(file=sys.stderr)
finally:
try:
if tempdir:
shutil.rmtree(tempdir) # delete directory
except OSError as exc:
traceback.print_exc(file=sys.stderr)
def xpdf_extract(fullpath, config, rcontext):
"""
Extract the images of the specified PDF file with xpdf_extract
fullpath - Path of the pdf file to extract images from
config - The Uforia configuration file
rcontext - The Uforia recursion context variables
"""
tempdir = None
try:
# Perform extraction
tempdir = tempfile.mkdtemp(dir=config.EXTRACTDIR)
_do_xpdf_extract(fullpath, tempdir)
# Call Uforia again
recursive.call_uforia_recursive(config, rcontext, tempdir, fullpath)
except:
traceback.print_exc(file=sys.stderr)
finally:
try:
if tempdir:
shutil.rmtree(tempdir) # delete directory
except OSError as exc:
traceback.print_exc(file=sys.stderr)
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
if file.btype.startswith("Microsoft Outlook email folder"):
readpst_path = libutil.get_executable("libpst", "readpst")
tempdir = None
try:
tempdir = tempfile.mkdtemp(dir=config.EXTRACTDIR)
p = subprocess.Popen([
readpst_path,
'-e',
'-q',
'-o',
tempdir,
fullpath
])
err = p.communicate()[1]
if err is not None:
raise Exception("readpst failed to extract " + fullpath)
recursive.call_uforia_recursive(config, rcontext, tempdir, fullpath)
return [fullpath]
except:
traceback.print_exc(file=sys.stderr)
return None
finally:
try:
if tempdir:
pass
#shutil.rmtree(tempdir) # delete directory
except OSError as exc:
traceback.print_exc(file=sys.stderr)
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
# Try to parse 7z data
try:
seven_zip = py7zlib.Archive7z(open(fullpath, 'rb'))
assorted = [seven_zip.getnames(), seven_zip.numfiles,
seven_zip.solid, seven_zip.version]
# Get .7zip's content metadata and store it in an dictionary.
# In the dictionary the key is the file name and
# the value is an other dict with its info.
content_info = {}
for member in seven_zip.getmembers():
content = {}
content["is_emptystream"] = member.emptystream
content["has_crc"] = member.checkcrc()
content["digest"] = member.digest
content["attributes"] = member.attributes
content["compressed_size"] = member.compressed
content["uncompressed_size"] = member.uncompressed
content_info[member.filename] = content
# Store content info in DB.
assorted.append(content_info)
del content_info
# Try to extract the content of the 7zip file.
try:
# Create a temporary directory
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
# Extract the 7zip file
_extractall(fullpath, tmpdir)
recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath)
except:
traceback.print_exc(file=sys.stderr)
# Delete the temporary directory, proceed even if it causes
# an error
try:
pass
shutil.rmtree(tmpdir)
except:
traceback.print_exc(file=sys.stderr)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\n7z file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i] + ':', assorted[i])
return assorted
except:
traceback.print_exc(file=sys.stderr)
# Store values in database so not the whole application crashes
return None
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
f = open(fullpath, 'r')
if "Date;Time;Called;Calling;Direction;Duration;ServiceCode;IMEI;CellID;SiteName;Suburb" not in f.read(
):
return None
else:
f.seek(0)
numlines = sum(1 for _ in f)
if numlines < 1:
# Empty file
return None
if numlines == 2:
# Header and single line, should go into the database
try:
f.seek(0)
firstline = f.readline().strip()
itemlist = f.readline().split(';')
Date = itemlist[0]
RawTime = str(itemlist[1])
Time = RawTime.zfill(6)
DateTime = datetime.datetime.fromtimestamp(
time.mktime(
time.strptime(
Date + ' ' + Time,
'%d/%m/%Y %H%M%S'))).strftime('%Y-%m-%d %H:%M:%S')
From = itemlist[2].strip() if itemlist[2] else None
To = itemlist[3].strip() if itemlist[3] else None
Direction = itemlist[4].strip() if itemlist[4] else None
Duration = itemlist[5].strip() if itemlist[5] else '0'
ServiceCode = itemlist[6].strip() if itemlist[6] else 'No Code'
IMEI = itemlist[7].strip() if itemlist[7] else 'No IMEI'
CellID = itemlist[8].strip() if itemlist[8] else 'Unknown'
SiteName = itemlist[9].strip() if itemlist[9] else 'Unknown'
Suburb = itemlist[10].strip() if itemlist[10] else 'Unknown'
Row = [
DateTime, From, To, Direction, Duration, ServiceCode, IMEI,
CellID, SiteName, Suburb
]
if config.DEBUG:
print "\ncell phone data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i] + ':', assorted[i])
return Row
except TypeError:
print('TypeError')
pass
except:
traceback.print_exc(file=sys.stderr)
return None
if numlines > 2:
# Header and multiple lines, split up into files
f.seek(0)
firstline = f.readline().strip()
secondline = f.readline()
lineno = 1
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
targetdir = tmpdir + os.path.sep + os.path.dirname(fullpath)
if not os.path.exists(targetdir):
try:
os.makedirs(targetdir)
except OSError as exc:
if exc.errno != errno.EXIST:
raise
for line in f:
targetfile = fullpath + "_line_" + str(lineno).zfill(
len(str(numlines)))
lineno += 1
with open(tmpdir + targetfile, 'wb') as g:
g.write(firstline + '\n')
g.write(line)
recursive.call_uforia_recursive(config, rcontext, tmpdir,
os.path.dirname(fullpath))
try:
shutil.rmtree(tmpdir)
except:
traceback.print_exc(file=sys.stderr)
return None
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
try:
# Open the tar file
tar = tarfile.open(fullpath)
# Get tar metadata
assorted = [tar.getnames(), len(tar.getnames())]
# Create an array with the the contents of the TarInfo structure
member_info = []
for member in tar.getmembers():
member_dict = {}
wanted_attributes = ['name',
'size',
'mtime',
'mode',
'type',
'linkname',
'uid',
'gid',
'uname',
'gname']
for attribute in wanted_attributes:
member_dict[attribute] = getattr(member, attribute)
member_info.append(member_dict)
assorted.append(member_info)
# Try to extract the content of the tar file.
tmpdir = None
try:
# Create a temporary directory
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
# Extract the tar file
tar.extractall(tmpdir)
# Close the tar file
tar.close()
recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath)
except:
traceback.print_exc(file=sys.stderr)
if tmpdir != None:
# Delete the temporary directory, proceed even if it causes
# an error.
# Do not use shutils because it may cause permission denied
# errors as tar preserves permissions.
try:
for root, dirs, files in os.walk(tmpdir, topdown=False):
for name in files:
filename = os.path.join(root, name)
os.chmod(filename, stat.S_IWUSR)
os.remove(filename)
for name in dirs:
os.rmdir(os.path.join(root, name))
except:
traceback.print_exc(file=sys.stderr)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nTar file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i], assorted[i])
print
return assorted
except:
traceback.print_exc(file=sys.stderr)
return None
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
# Try to parse RAR data
try:
# Set to full path of unrar.exe if it is not in PATH
rarfile.UNRAR_TOOL = config.UNRAR_TOOL
# Set up to 1 if you don't want to deal with decoding comments
# from unknown encoding. rarfile will try couple of common
# encodings in sequence.
rarfile.UNICODE_COMMENTS = 1
rar = rarfile.RarFile(fullpath)
assorted = [
rar.namelist(),
len(rar.namelist()),
rar.needs_password(), rar.comment
]
# Get .rar's content metadata and store it in an dictionary.
# In the dictionary the key is the file name and
# the value is an other dict with its info.
content_info = {}
for info in rar.infolist():
content = {}
content["date_time"] = info.date_time
content["compress_size"] = info.compress_size
content["CRC"] = info.CRC
content["comment"] = info.comment
content["volume"] = info.volume
content["compress_type"] = info.compress_type
content["extract_version"] = info.extract_version
content["host_os"] = info.host_os
content["mode"] = info.mode
content["archival_time"] = info.arctime
content["is_directory"] = info.isdir()
content["needs_password"] = info.needs_password()
content_info[info.filename] = content
# Store content info in DB.
assorted.append(content_info)
del content_info
# Try to extract the content of the rar file.
try:
# Create a temporary directory
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
# Extract the rar file
rar.extractall(tmpdir)
recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath)
# Close the rar file
rar.close()
except:
traceback.print_exc(file=sys.stderr)
# Delete the temporary directory, proceed even if it causes
# an error
try:
pass
shutil.rmtree(tmpdir)
except:
traceback.print_exc(file=sys.stderr)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nRAR file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i] + ':', assorted[i])
return assorted
except:
traceback.print_exc(file=sys.stderr)
# Store values in database so not the whole application crashes
return None
def process(fullpath, config, rcontext, columns=None):
try:
# Get instance of 7z module
zip_module = imp.load_source('7zfilerecursor',
'modules/application/' +
'x-7z-compressed/7zfilerecursor.py')
# Open cab file for reading
file = open(fullpath, 'rb')
# Add signature
assorted = [file.read(4)]
cabhdr = unpack('iiiiibbhhhhh', file.read(32))
# Add offset
assorted.append(cabhdr[3])
# Add version
version = "%d.%d" % (cabhdr[6], cabhdr[5])
assorted.append(version)
# Add amount of folders
assorted.append(cabhdr[7])
# Add amount of files
assorted.append(cabhdr[8])
if cabhdr[9] > 3:
print "CAB9 > 3"
resv = unpack('hbb', file.read(4))
cabflr = unpack('ihh', file.read(8))
#Add OffsetFirstFile and Compression
assorted.append(cabflr[0])
assorted.append(cabflr[2])
# Add None values to the database if cabflr is not correct
if cabflr[2] >= 0:
assorted.append(None)
assorted.append(None)
assorted.append(None)
assorted.append(None)
assorted.append(None)
assorted.append(None)
assorted.append(None)
assorted.append(None)
assorted.append(None)
else:
file.seek(cabflr[0])
cfdata = unpack('ibh', file.read(8))
# Add Checksum, SizeCompBytes, SizeUnCompBytes and PositionFirst
assorted.append(cfdata[0])
assorted.append(cfdata[1])
assorted.append(cfdata[2])
assorted.append(file.tell())
# Add WinCEHeader
assorted.append(file.read(4))
cehdr = unpack('iiiiiiiiiii', file.read(44))
# Add TargetArch
assorted.append(cehdr[4])
minimum_ce_version = "%d.%d" % (cehdr[5], cehdr[6])
maximum_ce_version = "%d.%d" % (cehdr[7], cehdr[8])
minimum_build_number = "%d.%d" % (cehdr[9], cehdr[10])
assorted.append(minimum_ce_version)
assorted.append(maximum_ce_version)
assorted.append(minimum_build_number)
# Try to extract the content of the 7zip file.
try:
# Get instance of 7z module
zip_module = imp.load_source('7zfilerecursor',
'modules/application/' +
'x-7z-compressed/7zfilerecursor.py')
# Create a temporary directory
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
# Extract the 7zip file
zip_module._extractall(fullpath, tmpdir)
recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath)
except:
traceback.print_exc(file=sys.stderr)
# Delete the temporary directory, proceed even if it causes
# an error
try:
pass
shutil.rmtree(tmpdir)
except:
traceback.print_exc(file=sys.stderr)
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nCab file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i] + ':', assorted[i])
return assorted
except:
traceback.print_exc(file=sys.stderr)
# Store values in database so not the whole application crashes
return None
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
if "Message-ID: " not in open(fullpath,'r').read():
return None
# Try to parse rfc822 data
try:
# Get the e-mail headers from a file
email_file = open(fullpath, 'r')
msg = pyzmail.PyzMessage.factory(email_file)
# find all attachments and save them to a temp folder
tempdir = None
attachments = []
try:
tempdir = tempfile.mkdtemp(dir=config.EXTRACTDIR)
for mailpart in msg.mailparts:
if not mailpart.is_body:
attachments.append(mailpart.filename)
f = open(os.path.join(tempdir, mailpart.filename), 'wb')
if mailpart.type.startswith('text/') and mailpart.charset is not None:
f.write(mailpart.get_payload().decode(mailpart.charset))
else:
f.write(mailpart.get_payload())
f.close()
if len(attachments) > 0:
recursive.call_uforia_recursive(config, rcontext, tempdir, fullpath)
except:
traceback.print_exc(file=sys.stderr)
finally:
try:
if tempdir:
shutil.rmtree(tempdir) # delete directory
except OSError as exc:
traceback.print_exc(file=sys.stderr)
# Merge the receivers
To = msg.get_decoded_header('To', None)
XTo = msg.get_decoded_header('X-To', None)
Cc = msg.get_decoded_header('Cc', None)
XCc = msg.get_decoded_header('X-Cc', None)
Bcc = msg.get_decoded_header('Bcc', None)
XBcc = msg.get_decoded_header('X-Bcc', None)
Date = datetime.datetime.fromtimestamp(int(email.utils.mktime_tz(email.utils.parsedate_tz(msg.get_decoded_header("Date", None))))).strftime('%Y-%m-%d %H:%M:%S')
Subject = msg.get_decoded_header("Subject", None)
From = msg.get_decoded_header("From", None)
Received = msg.get_decoded_header("Received", None)
MessageID = msg.get_decoded_header("Message-ID", None)
Receivers = u''
for i in [To,XTo,Cc,XCc,Bcc,XBcc]:
if i:
Receivers += unicode(i)+', '
# Get most common headers
assorted = [msg.get_decoded_header("Delivered-To", None),
msg.get_decoded_header("Original-Recipient", None),
Received,
msg.get_decoded_header("Return-Path", None),
msg.get_decoded_header("Received-SPF", None),
msg.get_decoded_header("Authentication-Results", None),
msg.get_decoded_header("DKIM-Signature", None),
msg.get_decoded_header("DomainKey-Signature", None),
msg.get_decoded_header("Organization", None),
msg.get_decoded_header("MIME-Version", None),
msg.get_decoded_header("List-Unsubscribe", None),
msg.get_decoded_header("X-Received", None),
msg.get_decoded_header("X-Priority", None),
msg.get_decoded_header("X-MSMail-Priority", None),
msg.get_decoded_header("X-Mailer", None),
msg.get_decoded_header("X-MimeOLE", None),
msg.get_decoded_header("X-Notifications", None),
msg.get_decoded_header("X-Notification-ID", None),
msg.get_decoded_header("X-Sender-ID", None),
msg.get_decoded_header("X-Notification-Category", None),
msg.get_decoded_header("X-Notification-Type", None),
msg.get_decoded_header("X-UB", None),
msg.get_decoded_header("Precedence", None),
msg.get_decoded_header("Reply-To", None),
msg.get_decoded_header("Auto-Submitted", None),
MessageID,
Date,
Subject,
From,
Receivers,
msg.get_decoded_header("Content-Type", None)]
# Grab the common headers and all E-mail bodies
Body = ''
Headers = {'From':From,'Subject':Subject,'To':To,'XTo:':XTo,'Cc':Cc,'XCc':XCc,'Bcc':Bcc,'XBcc':XBcc,'Date':Date,'MessageID':MessageID,'Received':Received}
for key in Headers:
if Headers[key]:
Body += key+': '+Headers[key]+'\n'
Body += '\n'
for mailpart in msg.mailparts:
if mailpart.is_body:
payload = mailpart.get_payload()
try:
Body += payload.decode('utf-8')
Encoding = 'utf-8'
except UnicodeError:
try:
Body += payload.decode('ISO-8859-1')
Encoding = 'ISO-8859-1'
except UnicodeError:
Body += payload
assorted.append(Body)
assorted.append(','.join(attachments))
# Spam checking code - R. Broerze & A. Hamed
if SPAMD_DOSPAMCHECK:
try:
raw_email = open(fullpath, 'r').read()
try:
full_email = raw_email.decode('utf-8')
Encoding = 'utf-8'
except UnicodeError:
try:
full_email = raw_email.decode('ISO-8859-1')
Encoding = 'ISO-8859-1'
except UnicodeError:
full_email = raw_email
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((SPAMD_HOST, SPAMD_PORT))
data = 'REPORT SPAMC/1.2\r\n'
data += 'Content-length: %d\r\n' % len(full_email.encode(Encoding))
data += 'User: %s\r\n\r\n' % SPAMD_USER
data += full_email
sock.sendall(data.encode(Encoding));
fd = sock.makefile('rb', 0)
spamd_header = fd.readline()
if spamd_header.find('EX_OK') == -1:
if config.DEBUG:
print('SpamCheck error')
traceback.print_exc(file=sys.stderr)
raise Exception
spamd_score = fd.readline()
spamd_score_splitted = spamd_score.split(";")[1].split("/")[0].strip()
saveReport = False
report = ''
for line in fd.readlines():
if saveReport:
report += line
if line.startswith('----'):
saveReport = True
assorted.append(spamd_score_splitted)
assorted.append(report)
if float(spamd_score_splitted) > SPAMD_SPAMSCORELIMIT:
assorted.append('yes')
else:
assorted.append('no')
sock.close()
except Exception:
if config.DEBUG:
print('SpamCheck error')
traceback.print_exc(file=sys.stderr)
assorted.append(None);
assorted.append(None);
assorted.append('unknown');
else:
assorted.append(None);
assorted.append(None);
assorted.append('unknown');
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nrfc822 file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i] + ':', assorted[i])
return assorted
except TypeError:
print('TypeError')
pass
except:
traceback.print_exc(file=sys.stderr)
# Store values in database so not the whole application crashes
return None
def process(fullpath, config, rcontext, columns=None):
try:
# Open the tar file
tar = tarfile.open(fullpath)
# Get tar metadata
assorted = [tar.getnames(), len(tar.getnames())]
# Create an array with the the contents of the TarInfo structure
member_info = []
for member in tar.getmembers():
member_dict = {}
wanted_attributes = ['name',
'size',
'mtime',
'mode',
'type',
'linkname',
'uid',
'gid',
'uname',
'gname']
for attribute in wanted_attributes:
member_dict[attribute] = getattr(member, attribute)
member_info.append(member_dict)
assorted.append(member_info)
# Try to extract the content of the tar file.
tmpdir = None
try:
# Create a temporary directory
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
# Extract the tar file
tar.extractall(tmpdir)
# Close the tar file
tar.close()
recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath)
except:
traceback.print_exc(file=sys.stderr)
if tmpdir != None:
# Delete the temporary directory, proceed even if it causes
# an error.
# Do not use shutils because it may cause permission denied
# errors as tar preserves permissions.
try:
for root, dirs, files in os.walk(tmpdir, topdown=False):
for name in files:
filename = os.path.join(root, name)
os.chmod(filename, stat.S_IWUSR)
os.remove(filename)
for name in dirs:
os.rmdir(os.path.join(root, name))
except:
traceback.print_exc(file=sys.stderr)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nTar file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i], assorted[i])
print
return assorted
except:
traceback.print_exc(file=sys.stderr)
return None
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
if "Message-ID: " not in open(fullpath, 'r').read():
return None
# Try to parse rfc822 data
try:
# Get the e-mail headers from a file
email_file = open(fullpath, 'r')
msg = pyzmail.PyzMessage.factory(email_file)
# find all attachments and save them to a temp folder
tempdir = None
attachments = []
try:
tempdir = tempfile.mkdtemp(dir=config.EXTRACTDIR)
for mailpart in msg.mailparts:
if not mailpart.is_body:
attachments.append(mailpart.filename)
f = open(os.path.join(tempdir, mailpart.filename), 'wb')
if mailpart.type.startswith(
'text/') and mailpart.charset is not None:
f.write(mailpart.get_payload().decode(
mailpart.charset))
else:
f.write(mailpart.get_payload())
f.close()
if len(attachments) > 0:
recursive.call_uforia_recursive(config, rcontext, tempdir,
fullpath)
except:
traceback.print_exc(file=sys.stderr)
finally:
try:
if tempdir:
shutil.rmtree(tempdir) # delete directory
except OSError as exc:
traceback.print_exc(file=sys.stderr)
# Merge the receivers
To = msg.get_decoded_header('To', None)
XTo = msg.get_decoded_header('X-To', None)
Cc = msg.get_decoded_header('Cc', None)
XCc = msg.get_decoded_header('X-Cc', None)
Bcc = msg.get_decoded_header('Bcc', None)
XBcc = msg.get_decoded_header('X-Bcc', None)
Date = datetime.datetime.fromtimestamp(
int(
email.utils.mktime_tz(
email.utils.parsedate_tz(
msg.get_decoded_header(
"Date", None))))).strftime('%Y-%m-%d %H:%M:%S')
Subject = msg.get_decoded_header("Subject", None)
From = msg.get_decoded_header("From", None)
Received = msg.get_decoded_header("Received", None)
MessageID = msg.get_decoded_header("Message-ID", None)
Receivers = u''
for i in [To, XTo, Cc, XCc, Bcc, XBcc]:
if i:
Receivers += unicode(i) + ', '
# Get most common headers
assorted = [
msg.get_decoded_header("Delivered-To", None),
msg.get_decoded_header("Original-Recipient", None), Received,
msg.get_decoded_header("Return-Path", None),
msg.get_decoded_header("Received-SPF", None),
msg.get_decoded_header("Authentication-Results", None),
msg.get_decoded_header("DKIM-Signature", None),
msg.get_decoded_header("DomainKey-Signature", None),
msg.get_decoded_header("Organization", None),
msg.get_decoded_header("MIME-Version", None),
msg.get_decoded_header("List-Unsubscribe", None),
msg.get_decoded_header("X-Received", None),
msg.get_decoded_header("X-Priority", None),
msg.get_decoded_header("X-MSMail-Priority", None),
msg.get_decoded_header("X-Mailer", None),
msg.get_decoded_header("X-MimeOLE", None),
msg.get_decoded_header("X-Notifications", None),
msg.get_decoded_header("X-Notification-ID", None),
msg.get_decoded_header("X-Sender-ID", None),
msg.get_decoded_header("X-Notification-Category", None),
msg.get_decoded_header("X-Notification-Type", None),
msg.get_decoded_header("X-UB", None),
msg.get_decoded_header("Precedence", None),
msg.get_decoded_header("Reply-To", None),
msg.get_decoded_header("Auto-Submitted", None), MessageID, Date,
Subject, From, Receivers,
msg.get_decoded_header("Content-Type", None)
]
# Grab the common headers and all E-mail bodies
Body = ''
Headers = {
'From': From,
'Subject': Subject,
'To': To,
'XTo:': XTo,
'Cc': Cc,
'XCc': XCc,
'Bcc': Bcc,
'XBcc': XBcc,
'Date': Date,
'MessageID': MessageID,
'Received': Received
}
for key in Headers:
if Headers[key]:
Body += key + ': ' + Headers[key] + '\n'
Body += '\n'
for mailpart in msg.mailparts:
if mailpart.is_body:
payload = mailpart.get_payload()
try:
Body += payload.decode('utf-8')
Encoding = 'utf-8'
except UnicodeError:
try:
Body += payload.decode('ISO-8859-1')
Encoding = 'ISO-8859-1'
except UnicodeError:
Body += payload
assorted.append(Body)
assorted.append(','.join(attachments))
# Spam checking code - R. Broerze & A. Hamed
if SPAMD_DOSPAMCHECK:
try:
raw_email = open(fullpath, 'r').read()
try:
full_email = raw_email.decode('utf-8')
Encoding = 'utf-8'
except UnicodeError:
try:
full_email = raw_email.decode('ISO-8859-1')
Encoding = 'ISO-8859-1'
except UnicodeError:
full_email = raw_email
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((SPAMD_HOST, SPAMD_PORT))
data = 'REPORT SPAMC/1.2\r\n'
data += 'Content-length: %d\r\n' % len(
full_email.encode(Encoding))
data += 'User: %s\r\n\r\n' % SPAMD_USER
data += full_email
sock.sendall(data.encode(Encoding))
fd = sock.makefile('rb', 0)
spamd_header = fd.readline()
if spamd_header.find('EX_OK') == -1:
if config.DEBUG:
print('SpamCheck error')
traceback.print_exc(file=sys.stderr)
raise Exception
spamd_score = fd.readline()
spamd_score_splitted = spamd_score.split(";")[1].split(
"/")[0].strip()
saveReport = False
report = ''
for line in fd.readlines():
if saveReport:
report += line
if line.startswith('----'):
saveReport = True
assorted.append(spamd_score_splitted)
assorted.append(report)
if float(spamd_score_splitted) > SPAMD_SPAMSCORELIMIT:
assorted.append('yes')
else:
assorted.append('no')
sock.close()
except Exception:
if config.DEBUG:
print('SpamCheck error')
traceback.print_exc(file=sys.stderr)
assorted.append(None)
assorted.append(None)
assorted.append('unknown')
else:
assorted.append(None)
assorted.append(None)
assorted.append('unknown')
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nrfc822 file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i] + ':', assorted[i])
return assorted
except TypeError:
print('TypeError')
pass
except:
traceback.print_exc(file=sys.stderr)
# Store values in database so not the whole application crashes
return None
def process(fullpath, config, rcontext, columns=None):
try:
# Get instance of 7z module
zip_module = imp.load_source(
'7zfilerecursor',
'modules/application/' + 'x-7z-compressed/7zfilerecursor.py')
# Open cab file for reading
file = open(fullpath, 'rb')
# Add signature
assorted = [file.read(4)]
cabhdr = unpack('iiiiibbhhhhh', file.read(32))
# Add offset
assorted.append(cabhdr[3])
# Add version
version = "%d.%d" % (cabhdr[6], cabhdr[5])
assorted.append(version)
# Add amount of folders
assorted.append(cabhdr[7])
# Add amount of files
assorted.append(cabhdr[8])
if cabhdr[9] > 3:
print "CAB9 > 3"
resv = unpack('hbb', file.read(4))
cabflr = unpack('ihh', file.read(8))
#Add OffsetFirstFile and Compression
assorted.append(cabflr[0])
assorted.append(cabflr[2])
# Add None values to the database if cabflr is not correct
if cabflr[2] >= 0:
assorted.append(None)
assorted.append(None)
assorted.append(None)
assorted.append(None)
assorted.append(None)
assorted.append(None)
assorted.append(None)
assorted.append(None)
assorted.append(None)
else:
file.seek(cabflr[0])
cfdata = unpack('ibh', file.read(8))
# Add Checksum, SizeCompBytes, SizeUnCompBytes and PositionFirst
assorted.append(cfdata[0])
assorted.append(cfdata[1])
assorted.append(cfdata[2])
assorted.append(file.tell())
# Add WinCEHeader
assorted.append(file.read(4))
cehdr = unpack('iiiiiiiiiii', file.read(44))
# Add TargetArch
assorted.append(cehdr[4])
minimum_ce_version = "%d.%d" % (cehdr[5], cehdr[6])
maximum_ce_version = "%d.%d" % (cehdr[7], cehdr[8])
minimum_build_number = "%d.%d" % (cehdr[9], cehdr[10])
assorted.append(minimum_ce_version)
assorted.append(maximum_ce_version)
assorted.append(minimum_build_number)
# Try to extract the content of the 7zip file.
try:
# Get instance of 7z module
zip_module = imp.load_source(
'7zfilerecursor',
'modules/application/' + 'x-7z-compressed/7zfilerecursor.py')
# Create a temporary directory
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
# Extract the 7zip file
zip_module._extractall(fullpath, tmpdir)
recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath)
except:
traceback.print_exc(file=sys.stderr)
# Delete the temporary directory, proceed even if it causes
# an error
try:
pass
shutil.rmtree(tmpdir)
except:
traceback.print_exc(file=sys.stderr)
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nCab file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i] + ':', assorted[i])
return assorted
except:
traceback.print_exc(file=sys.stderr)
# Store values in database so not the whole application crashes
return None
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
f = open(fullpath, 'r')
if "/nontampered_" not in fullpath:
return None
if "Name;Description;Ext.;Type;Status;Type descr.;Category;Evidence object;Path;Sender;Recipients;Size;Created;Modified;Accessed;Record update;Deletion;Int. creation;Attr.;Owner;Links;File count;1st sector;ID;Int. ID;Int. parent;Dimens.;SC%;Hash;Hash Set;Hash Categ.;Report table;Comment;Metadata" not in f.read(
):
return None
else:
f.seek(0)
numlines = sum(1 for _ in f)
if numlines < 1:
# Empty file
return None
if numlines == 2:
# Header and single line, should go into the database
try:
f.seek(0)
firstline = f.readline().strip()
secondline = f.readline().strip()
itemlist = secondline.split(';')
Name = itemlist[0]
Description = itemlist[1] if itemlist[1] else None
Extension = itemlist[2] if itemlist[2] else None
Type = itemlist[3] if itemlist[3] else None
Status = itemlist[4] if itemlist[4] else None
Type_Description = itemlist[5] if itemlist[5] else None
Category = itemlist[6] if itemlist[6] else None
Evidence_Object = itemlist[7] if itemlist[7] else None
Path = itemlist[8] if itemlist[8] else None
Sender = itemlist[9] if itemlist[9] else None
Recipients = itemlist[10] if itemlist[10] else None
Size = itemlist[11] if itemlist[11] else None
RawDate = itemlist[12].split(' ')[0]
try:
Day, Month, Year = RawDate.split('-')
Date = '{:02}'.format(int(Day)) + '-' + '{:02}'.format(
int(Month)) + '-' + '{:04}'.format(int(Year))
Time = itemlist[12].split(' ')[1]
Time += ":00" if len(Time) < 6 else Time
Created = datetime.datetime.fromtimestamp(
time.mktime(
time.strptime(Date + ' ' + Time,
'%d-%m-%Y %H:%M:%S'))).strftime(
'%Y-%m-%d %H:%M:%S')
except ValueError:
Created = "1970-01-01 00:00:00"
RawDate = itemlist[13].split(' ')[0]
try:
Day, Month, Year = RawDate.split('-')
Date = '{:02}'.format(int(Day)) + '-' + '{:02}'.format(
int(Month)) + '-' + '{:04}'.format(int(Year))
Time = itemlist[13].split(' ')[1]
Time += ":00" if len(Time) < 6 else Time
Modified = datetime.datetime.fromtimestamp(
time.mktime(
time.strptime(Date + ' ' + Time,
'%d-%m-%Y %H:%M:%S'))).strftime(
'%Y-%m-%d %H:%M:%S')
except ValueError:
Modified = "1970-01-01 00:00:00"
RawDate = itemlist[14].split(' ')[0]
try:
Day, Month, Year = RawDate.split('-')
Date = '{:02}'.format(int(Day)) + '-' + '{:02}'.format(
int(Month)) + '-' + '{:04}'.format(int(Year))
Time = itemlist[14].split(' ')[1]
Time += ":00" if len(Time) < 6 else Time
Accessed = datetime.datetime.fromtimestamp(
time.mktime(
time.strptime(Date + ' ' + Time,
'%d-%m-%Y %H:%M:%S'))).strftime(
'%Y-%m-%d %H:%M:%S')
except ValueError:
Accessed = "1970-01-01 00:00:00"
RawDate = itemlist[15].split(' ')[0]
try:
Day, Month, Year = RawDate.split('-')
Date = '{:02}'.format(int(Day)) + '-' + '{:02}'.format(
int(Month)) + '-' + '{:04}'.format(int(Year))
Time = itemlist[15].split(' ')[1]
Time += ":00" if len(Time) < 6 else Time
Updated = datetime.datetime.fromtimestamp(
time.mktime(
time.strptime(Date + ' ' + Time,
'%d-%m-%Y %H:%M:%S'))).strftime(
'%Y-%m-%d %H:%M:%S')
except ValueError:
Updated = "1970-01-01 00:00:00"
RawDate = itemlist[16].split(' ')[0]
try:
Day, Month, Year = RawDate.split('-')
Date = '{:02}'.format(int(Day)) + '-' + '{:02}'.format(
int(Month)) + '-' + '{:04}'.format(int(Year))
Time = itemlist[16].split(' ')[1]
Time += ":00" if len(Time) < 6 else Time
Deleted = datetime.datetime.fromtimestamp(
time.mktime(
time.strptime(Date + ' ' + Time,
'%d-%m-%Y %H:%M:%S'))).strftime(
'%Y-%m-%d %H:%M:%S')
except ValueError:
Deleted = "1970-01-01 00:00:00"
RawDate = itemlist[17].split(' ')[0]
try:
Day, Month, Year = RawDate.split('-')
Date = '{:02}'.format(int(Day)) + '-' + '{:02}'.format(
int(Month)) + '-' + '{:04}'.format(int(Year))
Time = itemlist[17].split(' ')[1]
Time += ":00" if len(Time) < 6 else Time
Internally_Created = datetime.datetime.fromtimestamp(
time.mktime(
time.strptime(Date + ' ' + Time,
'%d-%m-%Y %H:%M:%S'))).strftime(
'%Y-%m-%d %H:%M:%S')
except ValueError:
Internally_Created = "1970-01-01 00:00:00"
Attributes = itemlist[18].strip() if itemlist[18] else None
Owner = itemlist[19].strip() if itemlist[19] else None
Links = itemlist[20].strip() if itemlist[20] else None
File_Count = itemlist[21].strip() if itemlist[21] else None
Sector = itemlist[22].strip() if itemlist[22] else None
ID = itemlist[23].strip() if itemlist[23] else None
Internal_ID = itemlist[24].strip() if itemlist[24] else None
Internal_Parent = itemlist[25].strip(
) if itemlist[25] else None
Dimension = itemlist[26].strip() if itemlist[26] else None
SCPercent = itemlist[27].strip() if itemlist[27] else None
Hash = itemlist[28].strip() if itemlist[28] else None
Hash_Set = itemlist[29].strip() if itemlist[29] else None
Hash_Category = itemlist[30].strip() if itemlist[30] else None
Report_Table = itemlist[31].strip() if itemlist[31] else None
Comment = itemlist[32].strip() if itemlist[32] else None
Metadata = itemlist[33].strip() if itemlist[33] else None
C, M, A, U, D, I = Created.split(' ')[0], Modified.split(
' ')[0], Accessed.split(' ')[0], Updated.split(
' ')[0], Deleted.split(
' ')[0], Internally_Created.split(' ')[0]
detail = "<table><tr>"
detail += "<th>File</th><td>" + Path + "\\" + Name + "</td>"
detail += "</tr><tr>"
detail += "<th>Size</th><td>" + str(Size) + "</td>"
detail += "</tr><tr>"
detail += "<th>Created</th><td>" + Created + "</td>"
detail += "</tr><tr>"
detail += "<th>Modified</th><td>" + Modified + "</td>"
detail += "</tr><tr>"
detail += "<th>Accessed</th><td>" + Accessed + "</td>"
detail += "</tr><tr>"
detail += "<th>Updated</th><td>" + Updated + "</td>"
detail += "</tr><tr>"
detail += "<th>Deleted</th><td>" + Deleted + "</td>"
detail += "</tr></table>"
Row = [
Created, detail, C, M, A, U, D, I, Name, Description,
Extension, Type, Status, Type_Description, Category,
Evidence_Object, Path, Sender, Recipients, Size, Created,
Modified, Accessed, Updated, Deleted, Internally_Created,
Attributes, Owner, Links, File_Count, Sector, ID,
Internal_ID, Internal_Parent, Dimension, SCPercent, Hash,
Hash_Set, Hash_Category, Report_Table, Comment, Metadata
]
if config.DEBUG:
print "\nTimeline data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i] + ':', assorted[i])
return Row
except TypeError:
print('TypeError')
pass
except:
traceback.print_exc(file=sys.stderr)
return None
if numlines > 2:
# Header and multiple lines, split up into files
f.seek(0)
firstline = f.readline().strip()
secondline = f.readline()
lineno = 1
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
targetdir = tmpdir + os.path.sep + os.path.dirname(fullpath)
if not os.path.exists(targetdir):
try:
os.makedirs(targetdir)
except OSError as exc:
if exc.errno != errno.EXIST:
raise
for line in f:
targetfile = fullpath + "_line_" + str(lineno).zfill(
len(str(numlines)))
lineno += 1
with open(tmpdir + targetfile, 'wb') as g:
g.write(firstline + '\n')
g.write(line)
recursive.call_uforia_recursive(config, rcontext, tmpdir,
os.path.dirname(fullpath))
# try:
# shutil.rmtree(tmpdir)
# except:
# traceback.print_exc(file=sys.stderr)
return None
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
# Try to parse RAR data
try:
# Set to full path of unrar.exe if it is not in PATH
rarfile.UNRAR_TOOL = config.UNRAR_TOOL
# Set up to 1 if you don't want to deal with decoding comments
# from unknown encoding. rarfile will try couple of common
# encodings in sequence.
rarfile.UNICODE_COMMENTS = 1
rar = rarfile.RarFile(fullpath)
assorted = [rar.namelist(), len(rar.namelist()),
rar.needs_password(), rar.comment]
# Get .rar's content metadata and store it in an dictionary.
# In the dictionary the key is the file name and
# the value is an other dict with its info.
content_info = {}
for info in rar.infolist():
content = {}
content["date_time"] = info.date_time
content["compress_size"] = info.compress_size
content["CRC"] = info.CRC
content["comment"] = info.comment
content["volume"] = info.volume
content["compress_type"] = info.compress_type
content["extract_version"] = info.extract_version
content["host_os"] = info.host_os
content["mode"] = info.mode
content["archival_time"] = info.arctime
content["is_directory"] = info.isdir()
content["needs_password"] = info.needs_password()
content_info[info.filename] = content
# Store content info in DB.
assorted.append(content_info)
del content_info
# Try to extract the content of the rar file.
try:
# Create a temporary directory
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
# Extract the rar file
rar.extractall(tmpdir)
recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath)
# Close the rar file
rar.close()
except:
traceback.print_exc(file=sys.stderr)
# Delete the temporary directory, proceed even if it causes
# an error
try:
pass
shutil.rmtree(tmpdir)
except:
traceback.print_exc(file=sys.stderr)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nRAR file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i] + ':', assorted[i])
return assorted
except:
traceback.print_exc(file=sys.stderr)
# Store values in database so not the whole application crashes
return None
def process(fullpath, config, rcontext, columns=None):
try:
# Open the zipfile
zip = zipfile.ZipFile(fullpath, mode='r')
# Get .zip metadata
assorted = [zip.namelist(), len(zip.namelist()), zipfile.ZIP_STORED,
zipfile.ZIP_DEFLATED, zip.debug, zip.comment]
# Get .zip's content metadata and store it in an dictionary.
# In the dictionary the key is the file name and
# the value is an other dict with its info.
content_info = {}
for info in zip.infolist():
content = {}
content["date_time"] = info.date_time
content["compress_type"] = info.compress_type
content["comment"] = info.comment
content["create_system"] = info.create_system
content["create_version"] = info.create_version
content["extract_version"] = info.extract_version
content["reserved"] = info.reserved
content["flag_bits"] = info.flag_bits
content["volume"] = info.volume
content["internal_attr"] = info.internal_attr
content["external_attr"] = info.external_attr
content["header_offset"] = info.header_offset
content["CRC"] = info.CRC
content["compress_size"] = info.compress_size
content["file_size"] = info.file_size
content["_raw_time"] = info._raw_time
# The extra tag needs to be encoded for JSON
if not info.extra:
content["extra"] = info.extra
else:
base64.b64encode(info.extra)
content_info[info.filename] = content
# Store content info in DB.
assorted.append(content_info)
del content_info
# Try to extract the content of the zip file.
try:
# Create a temporary directory
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
# Extract the zip file
zip.extractall(tmpdir)
recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath)
# Close the zip file
zip.close()
except:
traceback.print_exc(file=sys.stderr)
# Delete the temporary directory, proceed even if it causes
# an error
try:
pass
shutil.rmtree(tmpdir)
except:
traceback.print_exc(file=sys.stderr)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nZip file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i], assorted[i])
print
return assorted
except:
traceback.print_exc(file=sys.stderr)
return None
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
f = open(fullpath,'r')
if "/nontampered_" not in fullpath:
return None
if "Name;Description;Ext.;Type;Status;Type descr.;Category;Evidence object;Path;Sender;Recipients;Size;Created;Modified;Accessed;Record update;Deletion;Int. creation;Attr.;Owner;Links;File count;1st sector;ID;Int. ID;Int. parent;Dimens.;SC%;Hash;Hash Set;Hash Categ.;Report table;Comment;Metadata" not in f.read():
return None
else:
f.seek(0)
numlines = sum(1 for _ in f)
if numlines < 1:
# Empty file
return None
if numlines == 2:
# Header and single line, should go into the database
try:
f.seek(0)
firstline = f.readline().strip()
secondline = f.readline().strip()
itemlist = secondline.split(';')
Name = itemlist[0]
Description = itemlist[1] if itemlist[1] else None
Extension = itemlist[2] if itemlist[2] else None
Type = itemlist[3] if itemlist[3] else None
Status = itemlist[4] if itemlist[4] else None
Type_Description = itemlist[5] if itemlist[5] else None
Category = itemlist[6] if itemlist[6] else None
Evidence_Object = itemlist[7] if itemlist[7] else None
Path = itemlist[8] if itemlist[8] else None
Sender = itemlist[9] if itemlist[9] else None
Recipients = itemlist[10] if itemlist[10] else None
Size = itemlist[11] if itemlist[11] else None
RawDate = itemlist[12].split(' ')[0]
try:
Day,Month,Year = RawDate.split('-')
Date = '{:02}'.format(int(Day))+'-'+'{:02}'.format(int(Month))+'-'+'{:04}'.format(int(Year))
Time = itemlist[12].split(' ')[1]
Time += ":00" if len(Time) < 6 else Time
Created = datetime.datetime.fromtimestamp(time.mktime(time.strptime(Date+' '+Time,'%d-%m-%Y %H:%M:%S'))).strftime('%Y-%m-%d %H:%M:%S')
except ValueError:
Created = "1970-01-01 00:00:00"
RawDate = itemlist[13].split(' ')[0]
try:
Day,Month,Year = RawDate.split('-')
Date = '{:02}'.format(int(Day))+'-'+'{:02}'.format(int(Month))+'-'+'{:04}'.format(int(Year))
Time = itemlist[13].split(' ')[1]
Time += ":00" if len(Time) < 6 else Time
Modified = datetime.datetime.fromtimestamp(time.mktime(time.strptime(Date+' '+Time,'%d-%m-%Y %H:%M:%S'))).strftime('%Y-%m-%d %H:%M:%S')
except ValueError:
Modified = "1970-01-01 00:00:00"
RawDate = itemlist[14].split(' ')[0]
try:
Day,Month,Year = RawDate.split('-')
Date = '{:02}'.format(int(Day))+'-'+'{:02}'.format(int(Month))+'-'+'{:04}'.format(int(Year))
Time = itemlist[14].split(' ')[1]
Time += ":00" if len(Time) < 6 else Time
Accessed = datetime.datetime.fromtimestamp(time.mktime(time.strptime(Date+' '+Time,'%d-%m-%Y %H:%M:%S'))).strftime('%Y-%m-%d %H:%M:%S')
except ValueError:
Accessed = "1970-01-01 00:00:00"
RawDate = itemlist[15].split(' ')[0]
try:
Day,Month,Year = RawDate.split('-')
Date = '{:02}'.format(int(Day))+'-'+'{:02}'.format(int(Month))+'-'+'{:04}'.format(int(Year))
Time = itemlist[15].split(' ')[1]
Time += ":00" if len(Time) < 6 else Time
Updated = datetime.datetime.fromtimestamp(time.mktime(time.strptime(Date+' '+Time,'%d-%m-%Y %H:%M:%S'))).strftime('%Y-%m-%d %H:%M:%S')
except ValueError:
Updated = "1970-01-01 00:00:00"
RawDate = itemlist[16].split(' ')[0]
try:
Day,Month,Year = RawDate.split('-')
Date = '{:02}'.format(int(Day))+'-'+'{:02}'.format(int(Month))+'-'+'{:04}'.format(int(Year))
Time = itemlist[16].split(' ')[1]
Time += ":00" if len(Time) < 6 else Time
Deleted = datetime.datetime.fromtimestamp(time.mktime(time.strptime(Date+' '+Time,'%d-%m-%Y %H:%M:%S'))).strftime('%Y-%m-%d %H:%M:%S')
except ValueError:
Deleted = "1970-01-01 00:00:00"
RawDate = itemlist[17].split(' ')[0]
try:
Day,Month,Year = RawDate.split('-')
Date = '{:02}'.format(int(Day))+'-'+'{:02}'.format(int(Month))+'-'+'{:04}'.format(int(Year))
Time = itemlist[17].split(' ')[1]
Time += ":00" if len(Time) < 6 else Time
Internally_Created = datetime.datetime.fromtimestamp(time.mktime(time.strptime(Date+' '+Time,'%d-%m-%Y %H:%M:%S'))).strftime('%Y-%m-%d %H:%M:%S')
except ValueError:
Internally_Created = "1970-01-01 00:00:00"
Attributes = itemlist[18].strip() if itemlist[18] else None
Owner = itemlist[19].strip() if itemlist[19] else None
Links = itemlist[20].strip() if itemlist[20] else None
File_Count = itemlist[21].strip() if itemlist[21] else None
Sector = itemlist[22].strip() if itemlist[22] else None
ID = itemlist[23].strip() if itemlist[23] else None
Internal_ID = itemlist[24].strip() if itemlist[24] else None
Internal_Parent = itemlist[25].strip() if itemlist[25] else None
Dimension = itemlist[26].strip() if itemlist[26] else None
SCPercent = itemlist[27].strip() if itemlist[27] else None
Hash = itemlist[28].strip() if itemlist[28] else None
Hash_Set = itemlist[29].strip() if itemlist[29] else None
Hash_Category = itemlist[30].strip() if itemlist[30] else None
Report_Table = itemlist[31].strip() if itemlist[31] else None
Comment = itemlist[32].strip() if itemlist[32] else None
Metadata = itemlist[33].strip() if itemlist[33] else None
C,M,A,U,D,I = Created.split(' ')[0],Modified.split(' ')[0],Accessed.split(' ')[0],Updated.split(' ')[0],Deleted.split(' ')[0],Internally_Created.split(' ')[0]
detail = "<table><tr>"
detail += "<th>File</th><td>"+Path+"\\"+Name+"</td>"
detail += "</tr><tr>"
detail += "<th>Size</th><td>"+str(Size)+"</td>"
detail += "</tr><tr>"
detail += "<th>Created</th><td>"+Created+"</td>"
detail += "</tr><tr>"
detail += "<th>Modified</th><td>"+Modified+"</td>"
detail += "</tr><tr>"
detail += "<th>Accessed</th><td>"+Accessed+"</td>"
detail += "</tr><tr>"
detail += "<th>Updated</th><td>"+Updated+"</td>"
detail += "</tr><tr>"
detail += "<th>Deleted</th><td>"+Deleted+"</td>"
detail += "</tr></table>"
Row=[Created,detail,C,M,A,U,D,I,Name,Description,Extension,Type,Status,Type_Description,Category,Evidence_Object,Path,Sender,Recipients,Size,Created,Modified,Accessed,Updated,Deleted,Internally_Created,Attributes,Owner,Links,File_Count,Sector,ID,Internal_ID,Internal_Parent,Dimension,SCPercent,Hash,Hash_Set,Hash_Category,Report_Table,Comment,Metadata]
if config.DEBUG:
print "\nTimeline data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i] + ':', assorted[i])
return Row
except TypeError:
print('TypeError')
pass
except:
traceback.print_exc(file=sys.stderr)
return None
if numlines > 2:
# Header and multiple lines, split up into files
f.seek(0)
firstline = f.readline().strip()
secondline = f.readline()
lineno = 1
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
targetdir = tmpdir + os.path.sep + os.path.dirname(fullpath)
if not os.path.exists(targetdir):
try:
os.makedirs(targetdir)
except OSError as exc:
if exc.errno != errno.EXIST:
raise
for line in f:
targetfile = fullpath + "_line_" + str(lineno).zfill(len(str(numlines)))
lineno += 1
with open(tmpdir+targetfile,'wb') as g:
g.write(firstline+'\n')
g.write(line)
recursive.call_uforia_recursive(config,rcontext,tmpdir,os.path.dirname(fullpath))
# try:
# shutil.rmtree(tmpdir)
# except:
# traceback.print_exc(file=sys.stderr)
return None
def process(file, config, rcontext, columns=None):
fullpath = file.fullpath
try:
# Open the zipfile
zip = zipfile.ZipFile(fullpath, mode='r')
# Get .zip metadata
assorted = [
zip.namelist(),
len(zip.namelist()), zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED,
zip.debug, zip.comment
]
# Get .zip's content metadata and store it in an dictionary.
# In the dictionary the key is the file name and
# the value is an other dict with its info.
content_info = {}
for info in zip.infolist():
content = {}
content["date_time"] = info.date_time
content["compress_type"] = info.compress_type
content["comment"] = info.comment
content["create_system"] = info.create_system
content["create_version"] = info.create_version
content["extract_version"] = info.extract_version
content["reserved"] = info.reserved
content["flag_bits"] = info.flag_bits
content["volume"] = info.volume
content["internal_attr"] = info.internal_attr
content["external_attr"] = info.external_attr
content["header_offset"] = info.header_offset
content["CRC"] = info.CRC
content["compress_size"] = info.compress_size
content["file_size"] = info.file_size
content["_raw_time"] = info._raw_time
# The extra tag needs to be encoded for JSON
if not info.extra:
content["extra"] = info.extra
else:
base64.b64encode(info.extra)
content_info[info.filename] = content
# Store content info in DB.
assorted.append(content_info)
del content_info
# Try to extract the content of the zip file.
try:
# Create a temporary directory
tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR)
# Extract the zip file
zip.extractall(tmpdir)
recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath)
# Close the zip file
zip.close()
except:
traceback.print_exc(file=sys.stderr)
# Delete the temporary directory, proceed even if it causes
# an error
try:
pass
shutil.rmtree(tmpdir)
except:
traceback.print_exc(file=sys.stderr)
# Make sure we stored exactly the same amount of columns as
# specified!!
assert len(assorted) == len(columns)
# Print some data that is stored in the database if debug is true
if config.DEBUG:
print "\nZip file data:"
for i in range(0, len(assorted)):
print "%-18s %s" % (columns[i], assorted[i])
print
return assorted
except:
traceback.print_exc(file=sys.stderr)
return None
