def scan_inode_distributed(case, inode_id, scanners, cookie):
""" Schedules a job to scan the inode by an available worker.
Note this will actually happen out of process.
"""
Farm.post_job('Scan', dict(case=case, inode_id=inode_id, scanners=scanners), cookie)
return
#return scan_inode(case, inode_id, factories, cookie)
pdbh = DB.DBO()
## We require that all scanning calls have a valid cookie. The
## following helps to trap bugs by scanners not passing the cookie
## along.
if cookie == 0:
raise RuntimeError( "Invalid cookie")
## This is a cookie used to identify our requests so that we
## can check they have been done later.
pdbh.insert("jobs",
command = 'Scan',
arg1 = case,
arg2 = inode_id,
arg3 = ','.join([f.name for f in factories]),
cookie=cookie,
_fast = True
)
## This is running in the worker itself - we can not wake up the
## other workers from here.
Farm.wake_workers()
def schedule_inode_index_sql(case, sql, word_id, cookie='', unique=True):
dict_version = get_dict_version()
## If we want detailed scan we want to set bit 30 of the dict_version
if not unique:
desired_version = dict_version | 2**30
else:
desired_version = dict_version
## Now check the inode to see if its out dated
dbh = DB.DBO(case)
dbh2 = DB.DBO(case)
pdbh = DB.DBO()
dbh.execute("select inode_id from "
" inode where (inode.version < %r or "
" (inode.version & (pow(2,30) -1)) < %r ) "
" and (inode_id in (%s))" ,
dict_version, desired_version, sql)
for row in dbh:
dbh2.update('inode',
where = "inode_id = %s" % row['inode_id'],
desired_version = desired_version,
_fast=True)
pdbh.insert('jobs',
cookie = cookie,
command = 'Index',
_fast = True,
arg1 = case,
arg2 = row['inode_id'],
arg3 = desired_version) # The desired version we
# want to be in
## Wake the workers:
Farm.wake_workers()
def wait_for_scan(self, cookie):
""" Waits for scanners to complete """
pdbh = DB.DBO()
## Often this process owns a worker as well. In that case we can wake it up:
import pyflag.Farm as Farm
Farm.wake_workers()
## Wait until there are no more jobs left.
while 1:
pdbh.execute("select count(*) as total from jobs where cookie=%r and arg1=%r", (cookie,
self.environment._CASE))
row = pdbh.fetch()
if row and row['total']==0: break
time.sleep(1)
def wait_for_scan(self, cookie):
""" Waits for scanners to complete """
import pyflag.Farm as Farm
while Farm.get_cookie_reference(cookie) > 0:
time.sleep(0.5)
return
print "Waiting for cookie %s" % cookie
pdbh = DB.DBO()
## Often this process owns a worker as well. In that case we can wake it up:
import pyflag.Farm as Farm
#Farm.wake_workers()
## Wait until there are no more jobs left.
while 1:
pdbh.execute("select * from jobs where cookie=%r limit 1",
(cookie))
row = pdbh.fetch()
if not row: break
time.sleep(1)
def wait_for_scan(self, cookie):
""" Waits for scanners to complete """
pdbh = DB.DBO()
## Often this process owns a worker as well. In that case we can wake it up:
import pyflag.Farm as Farm
Farm.wake_workers()
## Wait until there are no more jobs left.
while 1:
pdbh.execute(
"select count(*) as total from jobs where cookie=%r and arg1=%r",
(cookie, self.environment._CASE))
row = pdbh.fetch()
if row and row['total'] == 0: break
time.sleep(1)
def schedule_index(case, inode_ids, word, word_type, unique=True):
""" This function schedules an indexing job on the inode specified with the word and index type specified.
We check to see if the index is already in the dictionary and if
said inode is up to the relevant dictionary version.
unique requests the indexer to produce a single hit for each inode
(i.e. it only shows unique hits). If unique is False the indexer
will generate offsets for all hits in this inode.
"""
word_id = insert_dictionary_word(word, word_type)
if type(inode_ids)!=type(list):
inode_ids = (inode_ids,)
for inode_id in inode_ids:
schedule_inode_index(case, inode_id, word_id)
## Wake the workers:
Farm.wake_workers()
def render_html(self, inode_id, table_renderer):
Farm.post_job("Export", dict(case=self.case, inode_id=inode_id))
filename, content_type, fd = table_renderer.make_archive_filename(inode_id)
result = "<a href='%s'>%s</a><br />" % (filename, fd.inode_id)
try:
filename = "inodes/%s_summary.html" % inode_id
fd.html_export
result += "<a href='%s'><img src=images/browse.png /></a>" % (filename,)
except AttributeError: pass
#if table_renderer.explain_inodes:
## Add a link to the explaination page:
filename = "inodes/%s_explain.html" % inode_id
result+="<a href='%s'><img src=images/question.png /></a>" %(filename,)
## Check if there are annotations for this
dbh = DB.DBO(self.case)
dbh.execute("select * from annotate where inode_id=%r", inode_id)
for row in dbh:
result += "<br>%s" % row['note']
return result
def wait_for_scan(self, cookie):
""" Waits for scanners to complete """
import pyflag.Farm as Farm
while Farm.get_cookie_reference(cookie)>0:
time.sleep(0.5)
return
print "Waiting for cookie %s" % cookie
pdbh = DB.DBO()
## Often this process owns a worker as well. In that case we can wake it up:
import pyflag.Farm as Farm
#Farm.wake_workers()
## Wait until there are no more jobs left.
while 1:
pdbh.execute("select * from jobs where cookie=%r limit 1", (cookie))
row = pdbh.fetch()
if not row: break
time.sleep(1)
said inode is up to the relevant dictionary version.
unique requests the indexer to produce a single hit for each inode
(i.e. it only shows unique hits). If unique is False the indexer
will generate offsets for all hits in this inode.
"""
word_id = insert_dictionary_word(word, word_type)
if type(inode_ids) != type(list):
inode_ids = (inode_ids, )
for inode_id in inode_ids:
schedule_inode_index(case, inode_id, word_id)
## Wake the workers:
Farm.wake_workers()
def count_outdated_inodes(case, sql, word_id=None, unique=True):
""" This function counts the number of inodes outstanding to be scanned.
sql is a subquery which is expected to return a list of inode_ids.
word_id is the word which we count the outdated inodes with
respect to. If word_id is None, or omitted we simply count all
inodes in the sql provided.
unique is the type of index (a unique index of a detailed index).
"""
dbh = DB.DBO(case)
import pyflag.AJAXUI as AJAXUI
UI.UI = AJAXUI.AJAXUI
elif config.THEME == "XML":
import pyflag.XMLUI as XMLUI
UI.UI = XMLUI.XMLUI
else:
import pyflag.HTMLUI as HTMLUI
UI.UI = HTMLUI.HTMLUI
## Start the workers
import pyflag.Farm as Farm
Farm.start_workers()
## Check the schema:
FlagFramework.check_schema()
try:
try:
Server(HandlerClass=FlagServerHandler)
except Exception, e:
pyflaglog.log(pyflaglog.ERROR, "Error %s" % e)
finally:
pyflaglog.log(pyflaglog.INFO, "Terminating PyFlag Server")
sys.exit(0)
def run(keepalive=None):
global last_mtime, offset, output_fd
create_output_file()
print "Created output_fd"
## Get the PCAPFS class and instantiate it:
pcapfs = Registry.FILESYSTEMS.dispatch("PCAP Filesystem")(config.case)
pcapfs.mount_point = config.mountpoint
pcapfs.VFSCreate(None,
"I%s" % config.iosource,
config.mountpoint,
directory=True)
pcap_dbh = DB.DBO(config.case)
pcap_dbh.mass_insert_start("pcap")
pcap_dbh.execute("select max(id) as m from pcap")
pcap_id = pcap_dbh.fetch()['m'] or 0
cookie, processor = pcapfs.make_processor(config.iosource, scanners)
last_time = 0
files_we_have = set()
update_files(files_we_have)
log_fd = open(config.log, "a")
last_mtime = os.stat(directory).st_mtime
while 1:
## Check that our parent process is still there:
fds = select.select([keepalive], [], [], 0)
if fds[0]:
data = os.read(fds[0][0], 1024)
if len(data) == 0:
pyflaglog.log(pyflaglog.INFO,
"Main process disappeared - closing children")
## Parent quit - kill our child
os.kill(pid, signal.SIGABRT)
os._exit(0)
t = os.stat(directory).st_mtime
if t >= last_mtime:
last_mtime = t
files = os.listdir(directory)
files.sort()
count = 0
if not os.access(config.lock, os.F_OK):
for f in files:
count += 1
if f in files_we_have: continue
## Detect if the lock file appeared:
if os.access(config.lock, os.F_OK): break
if (count % 10) == 0 and config.MAXIMUM_WORKER_MEMORY > 0:
Farm.check_mem(finish)
filename = "%s/%s" % (directory, f)
if config.log:
log_fd.write(f + "\n")
log_fd.flush()
files_we_have.add(f)
load_file(filename, processor, pcap_dbh)
last_time = time.time()
else:
print "Lock file found"
if config.single:
## Wait untill all our jobs are done
pdbh = DB.DBO()
while 1:
pdbh.execute(
"select count(*) as c from jobs where cookie = %r",
cookie)
row = pdbh.fetch()
if row and row['c'] > 0:
time.sleep(5)
continue
else:
break
sys.exit(0)
## We need to flush the decoder:
if time.time() - last_time > config.timeout:
print "Flushing reassembler"
processor.flush()
last_time = time.time()
print "%s: Sleeping for %s seconds" % (time.ctime(), config.sleep)
time.sleep(config.sleep)
output_fd.write(packet_data)
else:
offset += packet.caplen
if output_fd:
output_fd.flush()
pcap_dbh.delete("connection_details", where="inode_id is null")
pcap_dbh.mass_insert_commit()
last_mtime = 0
offset = 0
## Start up some workers if needed:
Farm.start_workers()
def update_files(files_we_have):
try:
log_fd = open(config.log)
print "Reading log file"
for l in log_fd:
files_we_have.add(l.strip())
print "Done - added %s files from log" % len(files_we_have)
log_fd.close()
except IOError:
pass
def run(keepalive=None):
def run(keepalive=None):
global last_mtime, offset, output_fd
create_output_file()
print "Created output_fd"
## Get the PCAPFS class and instantiate it:
pcapfs = Registry.FILESYSTEMS.dispatch("PCAP Filesystem")(config.case)
pcapfs.mount_point = config.mountpoint
pcapfs.VFSCreate(None, "I%s" % config.iosource, config.mountpoint,
directory=True)
pcap_dbh = DB.DBO(config.case)
pcap_dbh.mass_insert_start("pcap")
pcap_dbh.execute("select max(id) as m from pcap")
pcap_id = pcap_dbh.fetch()['m'] or 0
cookie, processor = pcapfs.make_processor(config.iosource, scanners)
last_time = 0
files_we_have = set()
update_files(files_we_have)
log_fd = open(config.log, "a")
last_mtime = os.stat(directory).st_mtime
while 1:
## Check that our parent process is still there:
fds = select.select([keepalive],[],[], 0)
if fds[0]:
data = os.read(fds[0][0],1024)
if len(data)==0:
pyflaglog.log(pyflaglog.INFO, "Main process disappeared - closing children")
## Parent quit - kill our child
os.kill(pid,signal.SIGABRT)
os._exit(0)
t = os.stat(directory).st_mtime
if t>=last_mtime:
last_mtime = t
files = os.listdir(directory)
files.sort()
count = 0
if not os.access(config.lock, os.F_OK):
for f in files:
count +=1
if f in files_we_have: continue
## Detect if the lock file appeared:
if os.access(config.lock, os.F_OK): break
if (count % 10) ==0 and config.MAXIMUM_WORKER_MEMORY > 0:
Farm.check_mem(finish)
filename = "%s/%s" % (directory,f)
if config.log:
log_fd.write(f+"\n")
log_fd.flush()
files_we_have.add(f)
load_file(filename, processor, pcap_dbh)
last_time = time.time()
else:
print "Lock file found"
if config.single:
## Wait untill all our jobs are done
pdbh = DB.DBO()
while 1:
pdbh.execute("select count(*) as c from jobs where cookie = %r", cookie)
row = pdbh.fetch()
if row and row['c'] >0:
time.sleep(5)
continue
else:
break
sys.exit(0)
## We need to flush the decoder:
if time.time() - last_time > config.timeout:
print "Flushing reassembler"
processor.flush()
last_time = time.time()
print "%s: Sleeping for %s seconds" % (time.ctime(), config.sleep)
time.sleep(config.sleep)
output_fd.write(packet_data)
else:
offset += packet.caplen
if output_fd:
output_fd.flush()
pcap_dbh.delete("connection_details",
where = "inode_id is null")
pcap_dbh.mass_insert_commit()
last_mtime = 0
offset = 0
## Start up some workers if needed:
Farm.start_workers()
def update_files(files_we_have):
try:
log_fd = open(config.log)
print "Reading log file"
for l in log_fd:
files_we_have.add(l.strip())
print "Done - added %s files from log" % len(files_we_have)
log_fd.close()
except IOError:
pass
def run(keepalive=None):
global last_mtime, offset, output_fd
flag = FlagFramework.Flag()
FlagFramework.GLOBAL_FLAG_OBJ = flag
#Set the UI module to produce HTML
if config.THEME == "AJAX":
import pyflag.AJAXUI as AJAXUI
UI.UI = AJAXUI.AJAXUI
elif config.THEME == "XML":
import pyflag.XMLUI as XMLUI
UI.UI = XMLUI.XMLUI
else:
import pyflag.HTMLUI as HTMLUI
UI.UI = HTMLUI.HTMLUI
## Start the workers
import pyflag.Farm as Farm
Farm.start_workers()
## Check the schema:
FlagFramework.check_schema()
try:
try:
Server(HandlerClass=FlagServerHandler)
except Exception, e:
pyflaglog.log(pyflaglog.ERROR, "Error %s" % e)
finally:
pyflaglog.log(pyflaglog.INFO, "Terminating PyFlag Server")
sys.exit(0)
