def get_unexpired(ids=None): """ return a list of query IDs that are still available on the drive the queries can be in various states of processing """ return [ f for f in readdir(Config.get('SPOOL_DIR')) if (f not in ['.', '..']) and (not ids or f in ids) and ( len(f) >= 32) ]
def merge(query_tuple): """ Runs in the 'io' worker merges multiple pcap results using wireshark's mergecap tool """ query = Query(qt=query_tuple) if not query.load(): Config.logger.debug("DEBUG: failed to load [{}]".format(query.id)) query.progress('merge', 'starting merge', Query.MERGE) files = [query.path(f) for f in readdir(query.job_path, endswith='.pcap')] if len(files) > 1: Config.logger.debug("Merging: {}".format(','.join(files))) merged_file = query.path('merged.tmp') cmd = ["/usr/sbin/mergecap", "-F", "pcap", "-w", merged_file] cmd.extend(files) from subprocess import call status_code= call(cmd) # Cleanup temporary files if status_code == 0: query.progress('merge', "merge complete, finalizing") # make the merged file available (rename is atomic) os.rename(merged_file, query.path('{}.pcap'.format(MERGED_NAME))) Config.logger.debug("Removing temp files: {}".format(str(files))) for item in files: os.remove(item) query.complete() else: query.error('merge', "{} returned {}".format(cmd, status_code)) elif files: os.rename(files[0], query.path('{}.pcap'.format(MERGED_NAME))) query.complete() else: query.error('merge', "Nothing to merge ?!?") query.save(to_file=True) cleanup.apply_async(queue='io')