def tika_extract(fullpath, context, metadata, config, rcontext): """ Use the Tika input stream and extract all embedded files (if possible). Invokes Uforia recursively over the extracted files. fullpath - Path of the file to extract context - The Tika parse context metadata - Tika metadata object oonfig - The Uforia configuration file rcontext - The Uforia recursion context variables """ # To skip recursive call if there are no files to extract extractor = tika.ParsingEmbeddedDocumentExtractor(context) needs_extraction = extractor.shouldParseEmbedded(metadata) if needs_extraction: # Call Uforia recursively on embedded files tempdir = None try: # Perform extraction tempdir = tempfile.mkdtemp(dir=config.EXTRACTDIR) _do_tika_extract(fullpath, tempdir) # Call Uforia again recursive.call_uforia_recursive(config, rcontext, tempdir, fullpath) except: traceback.print_exc(file=sys.stderr) finally: try: if tempdir: shutil.rmtree(tempdir) # delete directory except OSError as exc: traceback.print_exc(file=sys.stderr)
def process(file, config, rcontext, columns=None): fullpath = file.fullpath if file.btype.startswith("Microsoft Outlook email folder"): readpst_path = libutil.get_executable("libpst", "readpst") tempdir = None try: tempdir = tempfile.mkdtemp(dir=config.EXTRACTDIR) p = subprocess.Popen( [readpst_path, '-e', '-q', '-o', tempdir, fullpath]) err = p.communicate()[1] if err is not None: raise Exception("readpst failed to extract " + fullpath) recursive.call_uforia_recursive(config, rcontext, tempdir, fullpath) return [fullpath] except: traceback.print_exc(file=sys.stderr) return None finally: try: if tempdir: pass #shutil.rmtree(tempdir) # delete directory except OSError as exc: traceback.print_exc(file=sys.stderr)
def process(file, config, rcontext, columns=None): fullpath = file.fullpath # Try to parse 7z data try: # Get instance of 7z module zip_module = imp.load_source('7zfilerecursor', 'modules/application/' + 'x-7z-compressed/7zfilerecursor.py') file = open(fullpath, 'rb') assorted = _get_volume_descriptors(file) file.close() # Try to extract the content of the 7zip file. try: # Create a temporary directory tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) # Extract the 7zip file zip_module._extractall(fullpath, tmpdir) recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath) except: traceback.print_exc(file=sys.stderr) # Delete the temporary directory, proceed even if it causes # an error try: shutil.rmtree(tmpdir) except: traceback.print_exc(file=sys.stderr) # Make sure we stored exactly the same amount of columns as # specified!! assert len(assorted) == len(columns) return assorted except: traceback.print_exc(file=sys.stderr) # Store values in database so not the whole application crashes return None
def process(file, config, rcontext, columns=None): fullpath = file.fullpath # Try to parse 7z data try: # Get instance of 7z module zip_module = imp.load_source( '7zfilerecursor', 'modules/application/' + 'x-7z-compressed/7zfilerecursor.py') file = open(fullpath, 'rb') assorted = _get_volume_descriptors(file) file.close() # Try to extract the content of the 7zip file. try: # Create a temporary directory tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) # Extract the 7zip file zip_module._extractall(fullpath, tmpdir) recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath) except: traceback.print_exc(file=sys.stderr) # Delete the temporary directory, proceed even if it causes # an error try: shutil.rmtree(tmpdir) except: traceback.print_exc(file=sys.stderr) # Make sure we stored exactly the same amount of columns as # specified!! assert len(assorted) == len(columns) return assorted except: traceback.print_exc(file=sys.stderr) # Store values in database so not the whole application crashes return None
def process(file, config, rcontext, columns=None): fullpath = file.fullpath try: # Create a temporary directory tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) # Open gzip file for reading file = gzip.open(fullpath, 'rb') # Store gzip metadata values assorted = [file.extrabuf, file.extrasize, file.extrastart] # Read the uncompressed data file_content = file.read() # Write it to the temp folder uncompressed_file = open(tmpdir + os.path.sep + _uncompressed_filename(fullpath), "wb") uncompressed_file.write(file_content) # Close both files uncompressed_file.close() file.close() # Call Uforia recursively recursive.call_uforia_recursive(config, rcontext, tmpdir, os.path.dirname(fullpath)) # Delete the temporary directory, proceed even if it causes # an error try: shutil.rmtree(tmpdir) except: traceback.print_exc(file=sys.stderr) return assorted except: traceback.print_exc(file=sys.stderr) return None
def process(file, config, rcontext, columns=None): fullpath = file.fullpath try: # Create a temporary directory tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) # Open gzip file for reading file = gzip.open(fullpath, 'rb') # Store gzip metadata values assorted = [file.extrabuf, file.extrasize, file.extrastart] # Read the uncompressed data file_content = file.read() # Write it to the temp folder uncompressed_file = open( tmpdir + os.path.sep + _uncompressed_filename(fullpath), "wb") uncompressed_file.write(file_content) # Close both files uncompressed_file.close() file.close() # Call Uforia recursively recursive.call_uforia_recursive(config, rcontext, tmpdir, os.path.dirname(fullpath)) # Delete the temporary directory, proceed even if it causes # an error try: shutil.rmtree(tmpdir) except: traceback.print_exc(file=sys.stderr) return assorted except: traceback.print_exc(file=sys.stderr) return None
def xpdf_extract(fullpath, config, rcontext): """ Extract the images of the specified PDF file with xpdf_extract fullpath - Path of the pdf file to extract images from config - The Uforia configuration file rcontext - The Uforia recursion context variables """ tempdir = None try: # Perform extraction tempdir = tempfile.mkdtemp(dir=config.EXTRACTDIR) _do_xpdf_extract(fullpath, tempdir) # Call Uforia again recursive.call_uforia_recursive(config, rcontext, tempdir, fullpath) except: traceback.print_exc(file=sys.stderr) finally: try: if tempdir: shutil.rmtree(tempdir) # delete directory except OSError as exc: traceback.print_exc(file=sys.stderr)
def process(file, config, rcontext, columns=None): fullpath = file.fullpath if file.btype.startswith("Microsoft Outlook email folder"): readpst_path = libutil.get_executable("libpst", "readpst") tempdir = None try: tempdir = tempfile.mkdtemp(dir=config.EXTRACTDIR) p = subprocess.Popen([ readpst_path, '-e', '-q', '-o', tempdir, fullpath ]) err = p.communicate()[1] if err is not None: raise Exception("readpst failed to extract " + fullpath) recursive.call_uforia_recursive(config, rcontext, tempdir, fullpath) return [fullpath] except: traceback.print_exc(file=sys.stderr) return None finally: try: if tempdir: pass #shutil.rmtree(tempdir) # delete directory except OSError as exc: traceback.print_exc(file=sys.stderr)
def process(file, config, rcontext, columns=None): fullpath = file.fullpath # Try to parse 7z data try: seven_zip = py7zlib.Archive7z(open(fullpath, 'rb')) assorted = [seven_zip.getnames(), seven_zip.numfiles, seven_zip.solid, seven_zip.version] # Get .7zip's content metadata and store it in an dictionary. # In the dictionary the key is the file name and # the value is an other dict with its info. content_info = {} for member in seven_zip.getmembers(): content = {} content["is_emptystream"] = member.emptystream content["has_crc"] = member.checkcrc() content["digest"] = member.digest content["attributes"] = member.attributes content["compressed_size"] = member.compressed content["uncompressed_size"] = member.uncompressed content_info[member.filename] = content # Store content info in DB. assorted.append(content_info) del content_info # Try to extract the content of the 7zip file. try: # Create a temporary directory tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) # Extract the 7zip file _extractall(fullpath, tmpdir) recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath) except: traceback.print_exc(file=sys.stderr) # Delete the temporary directory, proceed even if it causes # an error try: pass shutil.rmtree(tmpdir) except: traceback.print_exc(file=sys.stderr) # Make sure we stored exactly the same amount of columns as # specified!! assert len(assorted) == len(columns) # Print some data that is stored in the database if debug is true if config.DEBUG: print "\n7z file data:" for i in range(0, len(assorted)): print "%-18s %s" % (columns[i] + ':', assorted[i]) return assorted except: traceback.print_exc(file=sys.stderr) # Store values in database so not the whole application crashes return None
def process(file, config, rcontext, columns=None): fullpath = file.fullpath f = open(fullpath, 'r') if "Date;Time;Called;Calling;Direction;Duration;ServiceCode;IMEI;CellID;SiteName;Suburb" not in f.read( ): return None else: f.seek(0) numlines = sum(1 for _ in f) if numlines < 1: # Empty file return None if numlines == 2: # Header and single line, should go into the database try: f.seek(0) firstline = f.readline().strip() itemlist = f.readline().split(';') Date = itemlist[0] RawTime = str(itemlist[1]) Time = RawTime.zfill(6) DateTime = datetime.datetime.fromtimestamp( time.mktime( time.strptime( Date + ' ' + Time, '%d/%m/%Y %H%M%S'))).strftime('%Y-%m-%d %H:%M:%S') From = itemlist[2].strip() if itemlist[2] else None To = itemlist[3].strip() if itemlist[3] else None Direction = itemlist[4].strip() if itemlist[4] else None Duration = itemlist[5].strip() if itemlist[5] else '0' ServiceCode = itemlist[6].strip() if itemlist[6] else 'No Code' IMEI = itemlist[7].strip() if itemlist[7] else 'No IMEI' CellID = itemlist[8].strip() if itemlist[8] else 'Unknown' SiteName = itemlist[9].strip() if itemlist[9] else 'Unknown' Suburb = itemlist[10].strip() if itemlist[10] else 'Unknown' Row = [ DateTime, From, To, Direction, Duration, ServiceCode, IMEI, CellID, SiteName, Suburb ] if config.DEBUG: print "\ncell phone data:" for i in range(0, len(assorted)): print "%-18s %s" % (columns[i] + ':', assorted[i]) return Row except TypeError: print('TypeError') pass except: traceback.print_exc(file=sys.stderr) return None if numlines > 2: # Header and multiple lines, split up into files f.seek(0) firstline = f.readline().strip() secondline = f.readline() lineno = 1 tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) targetdir = tmpdir + os.path.sep + os.path.dirname(fullpath) if not os.path.exists(targetdir): try: os.makedirs(targetdir) except OSError as exc: if exc.errno != errno.EXIST: raise for line in f: targetfile = fullpath + "_line_" + str(lineno).zfill( len(str(numlines))) lineno += 1 with open(tmpdir + targetfile, 'wb') as g: g.write(firstline + '\n') g.write(line) recursive.call_uforia_recursive(config, rcontext, tmpdir, os.path.dirname(fullpath)) try: shutil.rmtree(tmpdir) except: traceback.print_exc(file=sys.stderr) return None
def process(file, config, rcontext, columns=None): fullpath = file.fullpath try: # Open the tar file tar = tarfile.open(fullpath) # Get tar metadata assorted = [tar.getnames(), len(tar.getnames())] # Create an array with the the contents of the TarInfo structure member_info = [] for member in tar.getmembers(): member_dict = {} wanted_attributes = ['name', 'size', 'mtime', 'mode', 'type', 'linkname', 'uid', 'gid', 'uname', 'gname'] for attribute in wanted_attributes: member_dict[attribute] = getattr(member, attribute) member_info.append(member_dict) assorted.append(member_info) # Try to extract the content of the tar file. tmpdir = None try: # Create a temporary directory tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) # Extract the tar file tar.extractall(tmpdir) # Close the tar file tar.close() recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath) except: traceback.print_exc(file=sys.stderr) if tmpdir != None: # Delete the temporary directory, proceed even if it causes # an error. # Do not use shutils because it may cause permission denied # errors as tar preserves permissions. try: for root, dirs, files in os.walk(tmpdir, topdown=False): for name in files: filename = os.path.join(root, name) os.chmod(filename, stat.S_IWUSR) os.remove(filename) for name in dirs: os.rmdir(os.path.join(root, name)) except: traceback.print_exc(file=sys.stderr) # Make sure we stored exactly the same amount of columns as # specified!! assert len(assorted) == len(columns) # Print some data that is stored in the database if debug is true if config.DEBUG: print "\nTar file data:" for i in range(0, len(assorted)): print "%-18s %s" % (columns[i], assorted[i]) print return assorted except: traceback.print_exc(file=sys.stderr) return None
def process(file, config, rcontext, columns=None): fullpath = file.fullpath # Try to parse RAR data try: # Set to full path of unrar.exe if it is not in PATH rarfile.UNRAR_TOOL = config.UNRAR_TOOL # Set up to 1 if you don't want to deal with decoding comments # from unknown encoding. rarfile will try couple of common # encodings in sequence. rarfile.UNICODE_COMMENTS = 1 rar = rarfile.RarFile(fullpath) assorted = [ rar.namelist(), len(rar.namelist()), rar.needs_password(), rar.comment ] # Get .rar's content metadata and store it in an dictionary. # In the dictionary the key is the file name and # the value is an other dict with its info. content_info = {} for info in rar.infolist(): content = {} content["date_time"] = info.date_time content["compress_size"] = info.compress_size content["CRC"] = info.CRC content["comment"] = info.comment content["volume"] = info.volume content["compress_type"] = info.compress_type content["extract_version"] = info.extract_version content["host_os"] = info.host_os content["mode"] = info.mode content["archival_time"] = info.arctime content["is_directory"] = info.isdir() content["needs_password"] = info.needs_password() content_info[info.filename] = content # Store content info in DB. assorted.append(content_info) del content_info # Try to extract the content of the rar file. try: # Create a temporary directory tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) # Extract the rar file rar.extractall(tmpdir) recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath) # Close the rar file rar.close() except: traceback.print_exc(file=sys.stderr) # Delete the temporary directory, proceed even if it causes # an error try: pass shutil.rmtree(tmpdir) except: traceback.print_exc(file=sys.stderr) # Make sure we stored exactly the same amount of columns as # specified!! assert len(assorted) == len(columns) # Print some data that is stored in the database if debug is true if config.DEBUG: print "\nRAR file data:" for i in range(0, len(assorted)): print "%-18s %s" % (columns[i] + ':', assorted[i]) return assorted except: traceback.print_exc(file=sys.stderr) # Store values in database so not the whole application crashes return None
def process(fullpath, config, rcontext, columns=None): try: # Get instance of 7z module zip_module = imp.load_source('7zfilerecursor', 'modules/application/' + 'x-7z-compressed/7zfilerecursor.py') # Open cab file for reading file = open(fullpath, 'rb') # Add signature assorted = [file.read(4)] cabhdr = unpack('iiiiibbhhhhh', file.read(32)) # Add offset assorted.append(cabhdr[3]) # Add version version = "%d.%d" % (cabhdr[6], cabhdr[5]) assorted.append(version) # Add amount of folders assorted.append(cabhdr[7]) # Add amount of files assorted.append(cabhdr[8]) if cabhdr[9] > 3: print "CAB9 > 3" resv = unpack('hbb', file.read(4)) cabflr = unpack('ihh', file.read(8)) #Add OffsetFirstFile and Compression assorted.append(cabflr[0]) assorted.append(cabflr[2]) # Add None values to the database if cabflr is not correct if cabflr[2] >= 0: assorted.append(None) assorted.append(None) assorted.append(None) assorted.append(None) assorted.append(None) assorted.append(None) assorted.append(None) assorted.append(None) assorted.append(None) else: file.seek(cabflr[0]) cfdata = unpack('ibh', file.read(8)) # Add Checksum, SizeCompBytes, SizeUnCompBytes and PositionFirst assorted.append(cfdata[0]) assorted.append(cfdata[1]) assorted.append(cfdata[2]) assorted.append(file.tell()) # Add WinCEHeader assorted.append(file.read(4)) cehdr = unpack('iiiiiiiiiii', file.read(44)) # Add TargetArch assorted.append(cehdr[4]) minimum_ce_version = "%d.%d" % (cehdr[5], cehdr[6]) maximum_ce_version = "%d.%d" % (cehdr[7], cehdr[8]) minimum_build_number = "%d.%d" % (cehdr[9], cehdr[10]) assorted.append(minimum_ce_version) assorted.append(maximum_ce_version) assorted.append(minimum_build_number) # Try to extract the content of the 7zip file. try: # Get instance of 7z module zip_module = imp.load_source('7zfilerecursor', 'modules/application/' + 'x-7z-compressed/7zfilerecursor.py') # Create a temporary directory tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) # Extract the 7zip file zip_module._extractall(fullpath, tmpdir) recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath) except: traceback.print_exc(file=sys.stderr) # Delete the temporary directory, proceed even if it causes # an error try: pass shutil.rmtree(tmpdir) except: traceback.print_exc(file=sys.stderr) assert len(assorted) == len(columns) # Print some data that is stored in the database if debug is true if config.DEBUG: print "\nCab file data:" for i in range(0, len(assorted)): print "%-18s %s" % (columns[i] + ':', assorted[i]) return assorted except: traceback.print_exc(file=sys.stderr) # Store values in database so not the whole application crashes return None
def process(file, config, rcontext, columns=None): fullpath = file.fullpath if "Message-ID: " not in open(fullpath,'r').read(): return None # Try to parse rfc822 data try: # Get the e-mail headers from a file email_file = open(fullpath, 'r') msg = pyzmail.PyzMessage.factory(email_file) # find all attachments and save them to a temp folder tempdir = None attachments = [] try: tempdir = tempfile.mkdtemp(dir=config.EXTRACTDIR) for mailpart in msg.mailparts: if not mailpart.is_body: attachments.append(mailpart.filename) f = open(os.path.join(tempdir, mailpart.filename), 'wb') if mailpart.type.startswith('text/') and mailpart.charset is not None: f.write(mailpart.get_payload().decode(mailpart.charset)) else: f.write(mailpart.get_payload()) f.close() if len(attachments) > 0: recursive.call_uforia_recursive(config, rcontext, tempdir, fullpath) except: traceback.print_exc(file=sys.stderr) finally: try: if tempdir: shutil.rmtree(tempdir) # delete directory except OSError as exc: traceback.print_exc(file=sys.stderr) # Merge the receivers To = msg.get_decoded_header('To', None) XTo = msg.get_decoded_header('X-To', None) Cc = msg.get_decoded_header('Cc', None) XCc = msg.get_decoded_header('X-Cc', None) Bcc = msg.get_decoded_header('Bcc', None) XBcc = msg.get_decoded_header('X-Bcc', None) Date = datetime.datetime.fromtimestamp(int(email.utils.mktime_tz(email.utils.parsedate_tz(msg.get_decoded_header("Date", None))))).strftime('%Y-%m-%d %H:%M:%S') Subject = msg.get_decoded_header("Subject", None) From = msg.get_decoded_header("From", None) Received = msg.get_decoded_header("Received", None) MessageID = msg.get_decoded_header("Message-ID", None) Receivers = u'' for i in [To,XTo,Cc,XCc,Bcc,XBcc]: if i: Receivers += unicode(i)+', ' # Get most common headers assorted = [msg.get_decoded_header("Delivered-To", None), msg.get_decoded_header("Original-Recipient", None), Received, msg.get_decoded_header("Return-Path", None), msg.get_decoded_header("Received-SPF", None), msg.get_decoded_header("Authentication-Results", None), msg.get_decoded_header("DKIM-Signature", None), msg.get_decoded_header("DomainKey-Signature", None), msg.get_decoded_header("Organization", None), msg.get_decoded_header("MIME-Version", None), msg.get_decoded_header("List-Unsubscribe", None), msg.get_decoded_header("X-Received", None), msg.get_decoded_header("X-Priority", None), msg.get_decoded_header("X-MSMail-Priority", None), msg.get_decoded_header("X-Mailer", None), msg.get_decoded_header("X-MimeOLE", None), msg.get_decoded_header("X-Notifications", None), msg.get_decoded_header("X-Notification-ID", None), msg.get_decoded_header("X-Sender-ID", None), msg.get_decoded_header("X-Notification-Category", None), msg.get_decoded_header("X-Notification-Type", None), msg.get_decoded_header("X-UB", None), msg.get_decoded_header("Precedence", None), msg.get_decoded_header("Reply-To", None), msg.get_decoded_header("Auto-Submitted", None), MessageID, Date, Subject, From, Receivers, msg.get_decoded_header("Content-Type", None)] # Grab the common headers and all E-mail bodies Body = '' Headers = {'From':From,'Subject':Subject,'To':To,'XTo:':XTo,'Cc':Cc,'XCc':XCc,'Bcc':Bcc,'XBcc':XBcc,'Date':Date,'MessageID':MessageID,'Received':Received} for key in Headers: if Headers[key]: Body += key+': '+Headers[key]+'\n' Body += '\n' for mailpart in msg.mailparts: if mailpart.is_body: payload = mailpart.get_payload() try: Body += payload.decode('utf-8') Encoding = 'utf-8' except UnicodeError: try: Body += payload.decode('ISO-8859-1') Encoding = 'ISO-8859-1' except UnicodeError: Body += payload assorted.append(Body) assorted.append(','.join(attachments)) # Spam checking code - R. Broerze & A. Hamed if SPAMD_DOSPAMCHECK: try: raw_email = open(fullpath, 'r').read() try: full_email = raw_email.decode('utf-8') Encoding = 'utf-8' except UnicodeError: try: full_email = raw_email.decode('ISO-8859-1') Encoding = 'ISO-8859-1' except UnicodeError: full_email = raw_email sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((SPAMD_HOST, SPAMD_PORT)) data = 'REPORT SPAMC/1.2\r\n' data += 'Content-length: %d\r\n' % len(full_email.encode(Encoding)) data += 'User: %s\r\n\r\n' % SPAMD_USER data += full_email sock.sendall(data.encode(Encoding)); fd = sock.makefile('rb', 0) spamd_header = fd.readline() if spamd_header.find('EX_OK') == -1: if config.DEBUG: print('SpamCheck error') traceback.print_exc(file=sys.stderr) raise Exception spamd_score = fd.readline() spamd_score_splitted = spamd_score.split(";")[1].split("/")[0].strip() saveReport = False report = '' for line in fd.readlines(): if saveReport: report += line if line.startswith('----'): saveReport = True assorted.append(spamd_score_splitted) assorted.append(report) if float(spamd_score_splitted) > SPAMD_SPAMSCORELIMIT: assorted.append('yes') else: assorted.append('no') sock.close() except Exception: if config.DEBUG: print('SpamCheck error') traceback.print_exc(file=sys.stderr) assorted.append(None); assorted.append(None); assorted.append('unknown'); else: assorted.append(None); assorted.append(None); assorted.append('unknown'); # Make sure we stored exactly the same amount of columns as # specified!! assert len(assorted) == len(columns) # Print some data that is stored in the database if debug is true if config.DEBUG: print "\nrfc822 file data:" for i in range(0, len(assorted)): print "%-18s %s" % (columns[i] + ':', assorted[i]) return assorted except TypeError: print('TypeError') pass except: traceback.print_exc(file=sys.stderr) # Store values in database so not the whole application crashes return None
def process(fullpath, config, rcontext, columns=None): try: # Open the tar file tar = tarfile.open(fullpath) # Get tar metadata assorted = [tar.getnames(), len(tar.getnames())] # Create an array with the the contents of the TarInfo structure member_info = [] for member in tar.getmembers(): member_dict = {} wanted_attributes = ['name', 'size', 'mtime', 'mode', 'type', 'linkname', 'uid', 'gid', 'uname', 'gname'] for attribute in wanted_attributes: member_dict[attribute] = getattr(member, attribute) member_info.append(member_dict) assorted.append(member_info) # Try to extract the content of the tar file. tmpdir = None try: # Create a temporary directory tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) # Extract the tar file tar.extractall(tmpdir) # Close the tar file tar.close() recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath) except: traceback.print_exc(file=sys.stderr) if tmpdir != None: # Delete the temporary directory, proceed even if it causes # an error. # Do not use shutils because it may cause permission denied # errors as tar preserves permissions. try: for root, dirs, files in os.walk(tmpdir, topdown=False): for name in files: filename = os.path.join(root, name) os.chmod(filename, stat.S_IWUSR) os.remove(filename) for name in dirs: os.rmdir(os.path.join(root, name)) except: traceback.print_exc(file=sys.stderr) # Make sure we stored exactly the same amount of columns as # specified!! assert len(assorted) == len(columns) # Print some data that is stored in the database if debug is true if config.DEBUG: print "\nTar file data:" for i in range(0, len(assorted)): print "%-18s %s" % (columns[i], assorted[i]) print return assorted except: traceback.print_exc(file=sys.stderr) return None
def process(file, config, rcontext, columns=None): fullpath = file.fullpath if "Message-ID: " not in open(fullpath, 'r').read(): return None # Try to parse rfc822 data try: # Get the e-mail headers from a file email_file = open(fullpath, 'r') msg = pyzmail.PyzMessage.factory(email_file) # find all attachments and save them to a temp folder tempdir = None attachments = [] try: tempdir = tempfile.mkdtemp(dir=config.EXTRACTDIR) for mailpart in msg.mailparts: if not mailpart.is_body: attachments.append(mailpart.filename) f = open(os.path.join(tempdir, mailpart.filename), 'wb') if mailpart.type.startswith( 'text/') and mailpart.charset is not None: f.write(mailpart.get_payload().decode( mailpart.charset)) else: f.write(mailpart.get_payload()) f.close() if len(attachments) > 0: recursive.call_uforia_recursive(config, rcontext, tempdir, fullpath) except: traceback.print_exc(file=sys.stderr) finally: try: if tempdir: shutil.rmtree(tempdir) # delete directory except OSError as exc: traceback.print_exc(file=sys.stderr) # Merge the receivers To = msg.get_decoded_header('To', None) XTo = msg.get_decoded_header('X-To', None) Cc = msg.get_decoded_header('Cc', None) XCc = msg.get_decoded_header('X-Cc', None) Bcc = msg.get_decoded_header('Bcc', None) XBcc = msg.get_decoded_header('X-Bcc', None) Date = datetime.datetime.fromtimestamp( int( email.utils.mktime_tz( email.utils.parsedate_tz( msg.get_decoded_header( "Date", None))))).strftime('%Y-%m-%d %H:%M:%S') Subject = msg.get_decoded_header("Subject", None) From = msg.get_decoded_header("From", None) Received = msg.get_decoded_header("Received", None) MessageID = msg.get_decoded_header("Message-ID", None) Receivers = u'' for i in [To, XTo, Cc, XCc, Bcc, XBcc]: if i: Receivers += unicode(i) + ', ' # Get most common headers assorted = [ msg.get_decoded_header("Delivered-To", None), msg.get_decoded_header("Original-Recipient", None), Received, msg.get_decoded_header("Return-Path", None), msg.get_decoded_header("Received-SPF", None), msg.get_decoded_header("Authentication-Results", None), msg.get_decoded_header("DKIM-Signature", None), msg.get_decoded_header("DomainKey-Signature", None), msg.get_decoded_header("Organization", None), msg.get_decoded_header("MIME-Version", None), msg.get_decoded_header("List-Unsubscribe", None), msg.get_decoded_header("X-Received", None), msg.get_decoded_header("X-Priority", None), msg.get_decoded_header("X-MSMail-Priority", None), msg.get_decoded_header("X-Mailer", None), msg.get_decoded_header("X-MimeOLE", None), msg.get_decoded_header("X-Notifications", None), msg.get_decoded_header("X-Notification-ID", None), msg.get_decoded_header("X-Sender-ID", None), msg.get_decoded_header("X-Notification-Category", None), msg.get_decoded_header("X-Notification-Type", None), msg.get_decoded_header("X-UB", None), msg.get_decoded_header("Precedence", None), msg.get_decoded_header("Reply-To", None), msg.get_decoded_header("Auto-Submitted", None), MessageID, Date, Subject, From, Receivers, msg.get_decoded_header("Content-Type", None) ] # Grab the common headers and all E-mail bodies Body = '' Headers = { 'From': From, 'Subject': Subject, 'To': To, 'XTo:': XTo, 'Cc': Cc, 'XCc': XCc, 'Bcc': Bcc, 'XBcc': XBcc, 'Date': Date, 'MessageID': MessageID, 'Received': Received } for key in Headers: if Headers[key]: Body += key + ': ' + Headers[key] + '\n' Body += '\n' for mailpart in msg.mailparts: if mailpart.is_body: payload = mailpart.get_payload() try: Body += payload.decode('utf-8') Encoding = 'utf-8' except UnicodeError: try: Body += payload.decode('ISO-8859-1') Encoding = 'ISO-8859-1' except UnicodeError: Body += payload assorted.append(Body) assorted.append(','.join(attachments)) # Spam checking code - R. Broerze & A. Hamed if SPAMD_DOSPAMCHECK: try: raw_email = open(fullpath, 'r').read() try: full_email = raw_email.decode('utf-8') Encoding = 'utf-8' except UnicodeError: try: full_email = raw_email.decode('ISO-8859-1') Encoding = 'ISO-8859-1' except UnicodeError: full_email = raw_email sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((SPAMD_HOST, SPAMD_PORT)) data = 'REPORT SPAMC/1.2\r\n' data += 'Content-length: %d\r\n' % len( full_email.encode(Encoding)) data += 'User: %s\r\n\r\n' % SPAMD_USER data += full_email sock.sendall(data.encode(Encoding)) fd = sock.makefile('rb', 0) spamd_header = fd.readline() if spamd_header.find('EX_OK') == -1: if config.DEBUG: print('SpamCheck error') traceback.print_exc(file=sys.stderr) raise Exception spamd_score = fd.readline() spamd_score_splitted = spamd_score.split(";")[1].split( "/")[0].strip() saveReport = False report = '' for line in fd.readlines(): if saveReport: report += line if line.startswith('----'): saveReport = True assorted.append(spamd_score_splitted) assorted.append(report) if float(spamd_score_splitted) > SPAMD_SPAMSCORELIMIT: assorted.append('yes') else: assorted.append('no') sock.close() except Exception: if config.DEBUG: print('SpamCheck error') traceback.print_exc(file=sys.stderr) assorted.append(None) assorted.append(None) assorted.append('unknown') else: assorted.append(None) assorted.append(None) assorted.append('unknown') # Make sure we stored exactly the same amount of columns as # specified!! assert len(assorted) == len(columns) # Print some data that is stored in the database if debug is true if config.DEBUG: print "\nrfc822 file data:" for i in range(0, len(assorted)): print "%-18s %s" % (columns[i] + ':', assorted[i]) return assorted except TypeError: print('TypeError') pass except: traceback.print_exc(file=sys.stderr) # Store values in database so not the whole application crashes return None
def process(fullpath, config, rcontext, columns=None): try: # Get instance of 7z module zip_module = imp.load_source( '7zfilerecursor', 'modules/application/' + 'x-7z-compressed/7zfilerecursor.py') # Open cab file for reading file = open(fullpath, 'rb') # Add signature assorted = [file.read(4)] cabhdr = unpack('iiiiibbhhhhh', file.read(32)) # Add offset assorted.append(cabhdr[3]) # Add version version = "%d.%d" % (cabhdr[6], cabhdr[5]) assorted.append(version) # Add amount of folders assorted.append(cabhdr[7]) # Add amount of files assorted.append(cabhdr[8]) if cabhdr[9] > 3: print "CAB9 > 3" resv = unpack('hbb', file.read(4)) cabflr = unpack('ihh', file.read(8)) #Add OffsetFirstFile and Compression assorted.append(cabflr[0]) assorted.append(cabflr[2]) # Add None values to the database if cabflr is not correct if cabflr[2] >= 0: assorted.append(None) assorted.append(None) assorted.append(None) assorted.append(None) assorted.append(None) assorted.append(None) assorted.append(None) assorted.append(None) assorted.append(None) else: file.seek(cabflr[0]) cfdata = unpack('ibh', file.read(8)) # Add Checksum, SizeCompBytes, SizeUnCompBytes and PositionFirst assorted.append(cfdata[0]) assorted.append(cfdata[1]) assorted.append(cfdata[2]) assorted.append(file.tell()) # Add WinCEHeader assorted.append(file.read(4)) cehdr = unpack('iiiiiiiiiii', file.read(44)) # Add TargetArch assorted.append(cehdr[4]) minimum_ce_version = "%d.%d" % (cehdr[5], cehdr[6]) maximum_ce_version = "%d.%d" % (cehdr[7], cehdr[8]) minimum_build_number = "%d.%d" % (cehdr[9], cehdr[10]) assorted.append(minimum_ce_version) assorted.append(maximum_ce_version) assorted.append(minimum_build_number) # Try to extract the content of the 7zip file. try: # Get instance of 7z module zip_module = imp.load_source( '7zfilerecursor', 'modules/application/' + 'x-7z-compressed/7zfilerecursor.py') # Create a temporary directory tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) # Extract the 7zip file zip_module._extractall(fullpath, tmpdir) recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath) except: traceback.print_exc(file=sys.stderr) # Delete the temporary directory, proceed even if it causes # an error try: pass shutil.rmtree(tmpdir) except: traceback.print_exc(file=sys.stderr) assert len(assorted) == len(columns) # Print some data that is stored in the database if debug is true if config.DEBUG: print "\nCab file data:" for i in range(0, len(assorted)): print "%-18s %s" % (columns[i] + ':', assorted[i]) return assorted except: traceback.print_exc(file=sys.stderr) # Store values in database so not the whole application crashes return None
def process(file, config, rcontext, columns=None): fullpath = file.fullpath f = open(fullpath, 'r') if "/nontampered_" not in fullpath: return None if "Name;Description;Ext.;Type;Status;Type descr.;Category;Evidence object;Path;Sender;Recipients;Size;Created;Modified;Accessed;Record update;Deletion;Int. creation;Attr.;Owner;Links;File count;1st sector;ID;Int. ID;Int. parent;Dimens.;SC%;Hash;Hash Set;Hash Categ.;Report table;Comment;Metadata" not in f.read( ): return None else: f.seek(0) numlines = sum(1 for _ in f) if numlines < 1: # Empty file return None if numlines == 2: # Header and single line, should go into the database try: f.seek(0) firstline = f.readline().strip() secondline = f.readline().strip() itemlist = secondline.split(';') Name = itemlist[0] Description = itemlist[1] if itemlist[1] else None Extension = itemlist[2] if itemlist[2] else None Type = itemlist[3] if itemlist[3] else None Status = itemlist[4] if itemlist[4] else None Type_Description = itemlist[5] if itemlist[5] else None Category = itemlist[6] if itemlist[6] else None Evidence_Object = itemlist[7] if itemlist[7] else None Path = itemlist[8] if itemlist[8] else None Sender = itemlist[9] if itemlist[9] else None Recipients = itemlist[10] if itemlist[10] else None Size = itemlist[11] if itemlist[11] else None RawDate = itemlist[12].split(' ')[0] try: Day, Month, Year = RawDate.split('-') Date = '{:02}'.format(int(Day)) + '-' + '{:02}'.format( int(Month)) + '-' + '{:04}'.format(int(Year)) Time = itemlist[12].split(' ')[1] Time += ":00" if len(Time) < 6 else Time Created = datetime.datetime.fromtimestamp( time.mktime( time.strptime(Date + ' ' + Time, '%d-%m-%Y %H:%M:%S'))).strftime( '%Y-%m-%d %H:%M:%S') except ValueError: Created = "1970-01-01 00:00:00" RawDate = itemlist[13].split(' ')[0] try: Day, Month, Year = RawDate.split('-') Date = '{:02}'.format(int(Day)) + '-' + '{:02}'.format( int(Month)) + '-' + '{:04}'.format(int(Year)) Time = itemlist[13].split(' ')[1] Time += ":00" if len(Time) < 6 else Time Modified = datetime.datetime.fromtimestamp( time.mktime( time.strptime(Date + ' ' + Time, '%d-%m-%Y %H:%M:%S'))).strftime( '%Y-%m-%d %H:%M:%S') except ValueError: Modified = "1970-01-01 00:00:00" RawDate = itemlist[14].split(' ')[0] try: Day, Month, Year = RawDate.split('-') Date = '{:02}'.format(int(Day)) + '-' + '{:02}'.format( int(Month)) + '-' + '{:04}'.format(int(Year)) Time = itemlist[14].split(' ')[1] Time += ":00" if len(Time) < 6 else Time Accessed = datetime.datetime.fromtimestamp( time.mktime( time.strptime(Date + ' ' + Time, '%d-%m-%Y %H:%M:%S'))).strftime( '%Y-%m-%d %H:%M:%S') except ValueError: Accessed = "1970-01-01 00:00:00" RawDate = itemlist[15].split(' ')[0] try: Day, Month, Year = RawDate.split('-') Date = '{:02}'.format(int(Day)) + '-' + '{:02}'.format( int(Month)) + '-' + '{:04}'.format(int(Year)) Time = itemlist[15].split(' ')[1] Time += ":00" if len(Time) < 6 else Time Updated = datetime.datetime.fromtimestamp( time.mktime( time.strptime(Date + ' ' + Time, '%d-%m-%Y %H:%M:%S'))).strftime( '%Y-%m-%d %H:%M:%S') except ValueError: Updated = "1970-01-01 00:00:00" RawDate = itemlist[16].split(' ')[0] try: Day, Month, Year = RawDate.split('-') Date = '{:02}'.format(int(Day)) + '-' + '{:02}'.format( int(Month)) + '-' + '{:04}'.format(int(Year)) Time = itemlist[16].split(' ')[1] Time += ":00" if len(Time) < 6 else Time Deleted = datetime.datetime.fromtimestamp( time.mktime( time.strptime(Date + ' ' + Time, '%d-%m-%Y %H:%M:%S'))).strftime( '%Y-%m-%d %H:%M:%S') except ValueError: Deleted = "1970-01-01 00:00:00" RawDate = itemlist[17].split(' ')[0] try: Day, Month, Year = RawDate.split('-') Date = '{:02}'.format(int(Day)) + '-' + '{:02}'.format( int(Month)) + '-' + '{:04}'.format(int(Year)) Time = itemlist[17].split(' ')[1] Time += ":00" if len(Time) < 6 else Time Internally_Created = datetime.datetime.fromtimestamp( time.mktime( time.strptime(Date + ' ' + Time, '%d-%m-%Y %H:%M:%S'))).strftime( '%Y-%m-%d %H:%M:%S') except ValueError: Internally_Created = "1970-01-01 00:00:00" Attributes = itemlist[18].strip() if itemlist[18] else None Owner = itemlist[19].strip() if itemlist[19] else None Links = itemlist[20].strip() if itemlist[20] else None File_Count = itemlist[21].strip() if itemlist[21] else None Sector = itemlist[22].strip() if itemlist[22] else None ID = itemlist[23].strip() if itemlist[23] else None Internal_ID = itemlist[24].strip() if itemlist[24] else None Internal_Parent = itemlist[25].strip( ) if itemlist[25] else None Dimension = itemlist[26].strip() if itemlist[26] else None SCPercent = itemlist[27].strip() if itemlist[27] else None Hash = itemlist[28].strip() if itemlist[28] else None Hash_Set = itemlist[29].strip() if itemlist[29] else None Hash_Category = itemlist[30].strip() if itemlist[30] else None Report_Table = itemlist[31].strip() if itemlist[31] else None Comment = itemlist[32].strip() if itemlist[32] else None Metadata = itemlist[33].strip() if itemlist[33] else None C, M, A, U, D, I = Created.split(' ')[0], Modified.split( ' ')[0], Accessed.split(' ')[0], Updated.split( ' ')[0], Deleted.split( ' ')[0], Internally_Created.split(' ')[0] detail = "<table><tr>" detail += "<th>File</th><td>" + Path + "\\" + Name + "</td>" detail += "</tr><tr>" detail += "<th>Size</th><td>" + str(Size) + "</td>" detail += "</tr><tr>" detail += "<th>Created</th><td>" + Created + "</td>" detail += "</tr><tr>" detail += "<th>Modified</th><td>" + Modified + "</td>" detail += "</tr><tr>" detail += "<th>Accessed</th><td>" + Accessed + "</td>" detail += "</tr><tr>" detail += "<th>Updated</th><td>" + Updated + "</td>" detail += "</tr><tr>" detail += "<th>Deleted</th><td>" + Deleted + "</td>" detail += "</tr></table>" Row = [ Created, detail, C, M, A, U, D, I, Name, Description, Extension, Type, Status, Type_Description, Category, Evidence_Object, Path, Sender, Recipients, Size, Created, Modified, Accessed, Updated, Deleted, Internally_Created, Attributes, Owner, Links, File_Count, Sector, ID, Internal_ID, Internal_Parent, Dimension, SCPercent, Hash, Hash_Set, Hash_Category, Report_Table, Comment, Metadata ] if config.DEBUG: print "\nTimeline data:" for i in range(0, len(assorted)): print "%-18s %s" % (columns[i] + ':', assorted[i]) return Row except TypeError: print('TypeError') pass except: traceback.print_exc(file=sys.stderr) return None if numlines > 2: # Header and multiple lines, split up into files f.seek(0) firstline = f.readline().strip() secondline = f.readline() lineno = 1 tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) targetdir = tmpdir + os.path.sep + os.path.dirname(fullpath) if not os.path.exists(targetdir): try: os.makedirs(targetdir) except OSError as exc: if exc.errno != errno.EXIST: raise for line in f: targetfile = fullpath + "_line_" + str(lineno).zfill( len(str(numlines))) lineno += 1 with open(tmpdir + targetfile, 'wb') as g: g.write(firstline + '\n') g.write(line) recursive.call_uforia_recursive(config, rcontext, tmpdir, os.path.dirname(fullpath)) # try: # shutil.rmtree(tmpdir) # except: # traceback.print_exc(file=sys.stderr) return None
def process(file, config, rcontext, columns=None): fullpath = file.fullpath # Try to parse RAR data try: # Set to full path of unrar.exe if it is not in PATH rarfile.UNRAR_TOOL = config.UNRAR_TOOL # Set up to 1 if you don't want to deal with decoding comments # from unknown encoding. rarfile will try couple of common # encodings in sequence. rarfile.UNICODE_COMMENTS = 1 rar = rarfile.RarFile(fullpath) assorted = [rar.namelist(), len(rar.namelist()), rar.needs_password(), rar.comment] # Get .rar's content metadata and store it in an dictionary. # In the dictionary the key is the file name and # the value is an other dict with its info. content_info = {} for info in rar.infolist(): content = {} content["date_time"] = info.date_time content["compress_size"] = info.compress_size content["CRC"] = info.CRC content["comment"] = info.comment content["volume"] = info.volume content["compress_type"] = info.compress_type content["extract_version"] = info.extract_version content["host_os"] = info.host_os content["mode"] = info.mode content["archival_time"] = info.arctime content["is_directory"] = info.isdir() content["needs_password"] = info.needs_password() content_info[info.filename] = content # Store content info in DB. assorted.append(content_info) del content_info # Try to extract the content of the rar file. try: # Create a temporary directory tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) # Extract the rar file rar.extractall(tmpdir) recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath) # Close the rar file rar.close() except: traceback.print_exc(file=sys.stderr) # Delete the temporary directory, proceed even if it causes # an error try: pass shutil.rmtree(tmpdir) except: traceback.print_exc(file=sys.stderr) # Make sure we stored exactly the same amount of columns as # specified!! assert len(assorted) == len(columns) # Print some data that is stored in the database if debug is true if config.DEBUG: print "\nRAR file data:" for i in range(0, len(assorted)): print "%-18s %s" % (columns[i] + ':', assorted[i]) return assorted except: traceback.print_exc(file=sys.stderr) # Store values in database so not the whole application crashes return None
def process(fullpath, config, rcontext, columns=None): try: # Open the zipfile zip = zipfile.ZipFile(fullpath, mode='r') # Get .zip metadata assorted = [zip.namelist(), len(zip.namelist()), zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED, zip.debug, zip.comment] # Get .zip's content metadata and store it in an dictionary. # In the dictionary the key is the file name and # the value is an other dict with its info. content_info = {} for info in zip.infolist(): content = {} content["date_time"] = info.date_time content["compress_type"] = info.compress_type content["comment"] = info.comment content["create_system"] = info.create_system content["create_version"] = info.create_version content["extract_version"] = info.extract_version content["reserved"] = info.reserved content["flag_bits"] = info.flag_bits content["volume"] = info.volume content["internal_attr"] = info.internal_attr content["external_attr"] = info.external_attr content["header_offset"] = info.header_offset content["CRC"] = info.CRC content["compress_size"] = info.compress_size content["file_size"] = info.file_size content["_raw_time"] = info._raw_time # The extra tag needs to be encoded for JSON if not info.extra: content["extra"] = info.extra else: base64.b64encode(info.extra) content_info[info.filename] = content # Store content info in DB. assorted.append(content_info) del content_info # Try to extract the content of the zip file. try: # Create a temporary directory tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) # Extract the zip file zip.extractall(tmpdir) recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath) # Close the zip file zip.close() except: traceback.print_exc(file=sys.stderr) # Delete the temporary directory, proceed even if it causes # an error try: pass shutil.rmtree(tmpdir) except: traceback.print_exc(file=sys.stderr) # Make sure we stored exactly the same amount of columns as # specified!! assert len(assorted) == len(columns) # Print some data that is stored in the database if debug is true if config.DEBUG: print "\nZip file data:" for i in range(0, len(assorted)): print "%-18s %s" % (columns[i], assorted[i]) print return assorted except: traceback.print_exc(file=sys.stderr) return None
def process(file, config, rcontext, columns=None): fullpath = file.fullpath f = open(fullpath,'r') if "/nontampered_" not in fullpath: return None if "Name;Description;Ext.;Type;Status;Type descr.;Category;Evidence object;Path;Sender;Recipients;Size;Created;Modified;Accessed;Record update;Deletion;Int. creation;Attr.;Owner;Links;File count;1st sector;ID;Int. ID;Int. parent;Dimens.;SC%;Hash;Hash Set;Hash Categ.;Report table;Comment;Metadata" not in f.read(): return None else: f.seek(0) numlines = sum(1 for _ in f) if numlines < 1: # Empty file return None if numlines == 2: # Header and single line, should go into the database try: f.seek(0) firstline = f.readline().strip() secondline = f.readline().strip() itemlist = secondline.split(';') Name = itemlist[0] Description = itemlist[1] if itemlist[1] else None Extension = itemlist[2] if itemlist[2] else None Type = itemlist[3] if itemlist[3] else None Status = itemlist[4] if itemlist[4] else None Type_Description = itemlist[5] if itemlist[5] else None Category = itemlist[6] if itemlist[6] else None Evidence_Object = itemlist[7] if itemlist[7] else None Path = itemlist[8] if itemlist[8] else None Sender = itemlist[9] if itemlist[9] else None Recipients = itemlist[10] if itemlist[10] else None Size = itemlist[11] if itemlist[11] else None RawDate = itemlist[12].split(' ')[0] try: Day,Month,Year = RawDate.split('-') Date = '{:02}'.format(int(Day))+'-'+'{:02}'.format(int(Month))+'-'+'{:04}'.format(int(Year)) Time = itemlist[12].split(' ')[1] Time += ":00" if len(Time) < 6 else Time Created = datetime.datetime.fromtimestamp(time.mktime(time.strptime(Date+' '+Time,'%d-%m-%Y %H:%M:%S'))).strftime('%Y-%m-%d %H:%M:%S') except ValueError: Created = "1970-01-01 00:00:00" RawDate = itemlist[13].split(' ')[0] try: Day,Month,Year = RawDate.split('-') Date = '{:02}'.format(int(Day))+'-'+'{:02}'.format(int(Month))+'-'+'{:04}'.format(int(Year)) Time = itemlist[13].split(' ')[1] Time += ":00" if len(Time) < 6 else Time Modified = datetime.datetime.fromtimestamp(time.mktime(time.strptime(Date+' '+Time,'%d-%m-%Y %H:%M:%S'))).strftime('%Y-%m-%d %H:%M:%S') except ValueError: Modified = "1970-01-01 00:00:00" RawDate = itemlist[14].split(' ')[0] try: Day,Month,Year = RawDate.split('-') Date = '{:02}'.format(int(Day))+'-'+'{:02}'.format(int(Month))+'-'+'{:04}'.format(int(Year)) Time = itemlist[14].split(' ')[1] Time += ":00" if len(Time) < 6 else Time Accessed = datetime.datetime.fromtimestamp(time.mktime(time.strptime(Date+' '+Time,'%d-%m-%Y %H:%M:%S'))).strftime('%Y-%m-%d %H:%M:%S') except ValueError: Accessed = "1970-01-01 00:00:00" RawDate = itemlist[15].split(' ')[0] try: Day,Month,Year = RawDate.split('-') Date = '{:02}'.format(int(Day))+'-'+'{:02}'.format(int(Month))+'-'+'{:04}'.format(int(Year)) Time = itemlist[15].split(' ')[1] Time += ":00" if len(Time) < 6 else Time Updated = datetime.datetime.fromtimestamp(time.mktime(time.strptime(Date+' '+Time,'%d-%m-%Y %H:%M:%S'))).strftime('%Y-%m-%d %H:%M:%S') except ValueError: Updated = "1970-01-01 00:00:00" RawDate = itemlist[16].split(' ')[0] try: Day,Month,Year = RawDate.split('-') Date = '{:02}'.format(int(Day))+'-'+'{:02}'.format(int(Month))+'-'+'{:04}'.format(int(Year)) Time = itemlist[16].split(' ')[1] Time += ":00" if len(Time) < 6 else Time Deleted = datetime.datetime.fromtimestamp(time.mktime(time.strptime(Date+' '+Time,'%d-%m-%Y %H:%M:%S'))).strftime('%Y-%m-%d %H:%M:%S') except ValueError: Deleted = "1970-01-01 00:00:00" RawDate = itemlist[17].split(' ')[0] try: Day,Month,Year = RawDate.split('-') Date = '{:02}'.format(int(Day))+'-'+'{:02}'.format(int(Month))+'-'+'{:04}'.format(int(Year)) Time = itemlist[17].split(' ')[1] Time += ":00" if len(Time) < 6 else Time Internally_Created = datetime.datetime.fromtimestamp(time.mktime(time.strptime(Date+' '+Time,'%d-%m-%Y %H:%M:%S'))).strftime('%Y-%m-%d %H:%M:%S') except ValueError: Internally_Created = "1970-01-01 00:00:00" Attributes = itemlist[18].strip() if itemlist[18] else None Owner = itemlist[19].strip() if itemlist[19] else None Links = itemlist[20].strip() if itemlist[20] else None File_Count = itemlist[21].strip() if itemlist[21] else None Sector = itemlist[22].strip() if itemlist[22] else None ID = itemlist[23].strip() if itemlist[23] else None Internal_ID = itemlist[24].strip() if itemlist[24] else None Internal_Parent = itemlist[25].strip() if itemlist[25] else None Dimension = itemlist[26].strip() if itemlist[26] else None SCPercent = itemlist[27].strip() if itemlist[27] else None Hash = itemlist[28].strip() if itemlist[28] else None Hash_Set = itemlist[29].strip() if itemlist[29] else None Hash_Category = itemlist[30].strip() if itemlist[30] else None Report_Table = itemlist[31].strip() if itemlist[31] else None Comment = itemlist[32].strip() if itemlist[32] else None Metadata = itemlist[33].strip() if itemlist[33] else None C,M,A,U,D,I = Created.split(' ')[0],Modified.split(' ')[0],Accessed.split(' ')[0],Updated.split(' ')[0],Deleted.split(' ')[0],Internally_Created.split(' ')[0] detail = "<table><tr>" detail += "<th>File</th><td>"+Path+"\\"+Name+"</td>" detail += "</tr><tr>" detail += "<th>Size</th><td>"+str(Size)+"</td>" detail += "</tr><tr>" detail += "<th>Created</th><td>"+Created+"</td>" detail += "</tr><tr>" detail += "<th>Modified</th><td>"+Modified+"</td>" detail += "</tr><tr>" detail += "<th>Accessed</th><td>"+Accessed+"</td>" detail += "</tr><tr>" detail += "<th>Updated</th><td>"+Updated+"</td>" detail += "</tr><tr>" detail += "<th>Deleted</th><td>"+Deleted+"</td>" detail += "</tr></table>" Row=[Created,detail,C,M,A,U,D,I,Name,Description,Extension,Type,Status,Type_Description,Category,Evidence_Object,Path,Sender,Recipients,Size,Created,Modified,Accessed,Updated,Deleted,Internally_Created,Attributes,Owner,Links,File_Count,Sector,ID,Internal_ID,Internal_Parent,Dimension,SCPercent,Hash,Hash_Set,Hash_Category,Report_Table,Comment,Metadata] if config.DEBUG: print "\nTimeline data:" for i in range(0, len(assorted)): print "%-18s %s" % (columns[i] + ':', assorted[i]) return Row except TypeError: print('TypeError') pass except: traceback.print_exc(file=sys.stderr) return None if numlines > 2: # Header and multiple lines, split up into files f.seek(0) firstline = f.readline().strip() secondline = f.readline() lineno = 1 tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) targetdir = tmpdir + os.path.sep + os.path.dirname(fullpath) if not os.path.exists(targetdir): try: os.makedirs(targetdir) except OSError as exc: if exc.errno != errno.EXIST: raise for line in f: targetfile = fullpath + "_line_" + str(lineno).zfill(len(str(numlines))) lineno += 1 with open(tmpdir+targetfile,'wb') as g: g.write(firstline+'\n') g.write(line) recursive.call_uforia_recursive(config,rcontext,tmpdir,os.path.dirname(fullpath)) # try: # shutil.rmtree(tmpdir) # except: # traceback.print_exc(file=sys.stderr) return None
def process(file, config, rcontext, columns=None): fullpath = file.fullpath try: # Open the zipfile zip = zipfile.ZipFile(fullpath, mode='r') # Get .zip metadata assorted = [ zip.namelist(), len(zip.namelist()), zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED, zip.debug, zip.comment ] # Get .zip's content metadata and store it in an dictionary. # In the dictionary the key is the file name and # the value is an other dict with its info. content_info = {} for info in zip.infolist(): content = {} content["date_time"] = info.date_time content["compress_type"] = info.compress_type content["comment"] = info.comment content["create_system"] = info.create_system content["create_version"] = info.create_version content["extract_version"] = info.extract_version content["reserved"] = info.reserved content["flag_bits"] = info.flag_bits content["volume"] = info.volume content["internal_attr"] = info.internal_attr content["external_attr"] = info.external_attr content["header_offset"] = info.header_offset content["CRC"] = info.CRC content["compress_size"] = info.compress_size content["file_size"] = info.file_size content["_raw_time"] = info._raw_time # The extra tag needs to be encoded for JSON if not info.extra: content["extra"] = info.extra else: base64.b64encode(info.extra) content_info[info.filename] = content # Store content info in DB. assorted.append(content_info) del content_info # Try to extract the content of the zip file. try: # Create a temporary directory tmpdir = tempfile.mkdtemp("_uforiatmp", dir=config.EXTRACTDIR) # Extract the zip file zip.extractall(tmpdir) recursive.call_uforia_recursive(config, rcontext, tmpdir, fullpath) # Close the zip file zip.close() except: traceback.print_exc(file=sys.stderr) # Delete the temporary directory, proceed even if it causes # an error try: pass shutil.rmtree(tmpdir) except: traceback.print_exc(file=sys.stderr) # Make sure we stored exactly the same amount of columns as # specified!! assert len(assorted) == len(columns) # Print some data that is stored in the database if debug is true if config.DEBUG: print "\nZip file data:" for i in range(0, len(assorted)): print "%-18s %s" % (columns[i], assorted[i]) print return assorted except: traceback.print_exc(file=sys.stderr) return None