Ejemplo n.º 1
0
def scan_inode_distributed(case, inode_id, scanners, cookie):
    """ Schedules a job to scan the inode by an available worker.

    Note this will actually happen out of process.
    """
    Farm.post_job('Scan', dict(case=case, inode_id=inode_id, scanners=scanners), cookie)
    return

    #return scan_inode(case, inode_id, factories, cookie)
    pdbh = DB.DBO()

    ## We require that all scanning calls have a valid cookie. The
    ## following helps to trap bugs by scanners not passing the cookie
    ## along.
    if cookie == 0:
        raise RuntimeError( "Invalid cookie")
    ## This is a cookie used to identify our requests so that we
    ## can check they have been done later.
    pdbh.insert("jobs",
                command = 'Scan',
                arg1 = case,
                arg2 = inode_id,
                arg3 = ','.join([f.name for f in factories]),
                cookie=cookie,
                _fast = True
                )
    
    ## This is running in the worker itself - we can not wake up the
    ## other workers from here.
    Farm.wake_workers()
Ejemplo n.º 2
0
def schedule_inode_index_sql(case, sql, word_id, cookie='', unique=True):
    dict_version = get_dict_version()

    ## If we want detailed scan we want to set bit 30 of the dict_version
    if not unique:
        desired_version = dict_version | 2**30
    else:
        desired_version = dict_version

    ## Now check the inode to see if its out dated
    dbh = DB.DBO(case)
    dbh2 = DB.DBO(case)
    pdbh = DB.DBO()
    dbh.execute("select inode_id from "
                " inode where (inode.version < %r or "
                " (inode.version & (pow(2,30) -1)) < %r ) "
                " and (inode_id in (%s))" ,
                dict_version, desired_version, sql)
    
    for row in dbh:
        dbh2.update('inode',
                   where = "inode_id = %s" % row['inode_id'],
                   desired_version = desired_version,
                   _fast=True)

        pdbh.insert('jobs',
                    cookie = cookie,
                    command = 'Index',
                    _fast = True,
                    arg1 = case,
                    arg2 = row['inode_id'],
                    arg3 = desired_version) # The desired version we
                                            # want to be in
    ## Wake the workers:
    Farm.wake_workers()
Ejemplo n.º 3
0
    def wait_for_scan(self, cookie):
        """ Waits for scanners to complete """
        pdbh = DB.DBO()
        ## Often this process owns a worker as well. In that case we can wake it up:
        import pyflag.Farm as Farm
        Farm.wake_workers()
        
        ## Wait until there are no more jobs left.
        while 1:
            pdbh.execute("select count(*) as total from jobs where cookie=%r and arg1=%r", (cookie,
                         self.environment._CASE))
            row = pdbh.fetch()
            if row and row['total']==0: break

            time.sleep(1)
Ejemplo n.º 4
0
    def wait_for_scan(self, cookie):
        """ Waits for scanners to complete """
        import pyflag.Farm as Farm

        while Farm.get_cookie_reference(cookie) > 0:
            time.sleep(0.5)

        return

        print "Waiting for cookie %s" % cookie
        pdbh = DB.DBO()

        ## Often this process owns a worker as well. In that case we can wake it up:
        import pyflag.Farm as Farm

        #Farm.wake_workers()

        ## Wait until there are no more jobs left.
        while 1:
            pdbh.execute("select * from jobs where cookie=%r limit 1",
                         (cookie))
            row = pdbh.fetch()
            if not row: break

            time.sleep(1)
Ejemplo n.º 5
0
    def wait_for_scan(self, cookie):
        """ Waits for scanners to complete """
        pdbh = DB.DBO()
        ## Often this process owns a worker as well. In that case we can wake it up:
        import pyflag.Farm as Farm
        Farm.wake_workers()

        ## Wait until there are no more jobs left.
        while 1:
            pdbh.execute(
                "select count(*) as total from jobs where cookie=%r and arg1=%r",
                (cookie, self.environment._CASE))
            row = pdbh.fetch()
            if row and row['total'] == 0: break

            time.sleep(1)
Ejemplo n.º 6
0
def schedule_index(case, inode_ids, word, word_type, unique=True):
    """ This function schedules an indexing job on the inode specified with the word and index type specified.

    We check to see if the index is already in the dictionary and if
    said inode is up to the relevant dictionary version.

    unique requests the indexer to produce a single hit for each inode
    (i.e. it only shows unique hits). If unique is False the indexer
    will generate offsets for all hits in this inode.
    """
    word_id = insert_dictionary_word(word, word_type)

    if type(inode_ids)!=type(list):
        inode_ids = (inode_ids,)
        
    for inode_id in inode_ids:
        schedule_inode_index(case, inode_id, word_id)
        
    ## Wake the workers:
    Farm.wake_workers()
Ejemplo n.º 7
0
def render_html(self, inode_id, table_renderer):
    Farm.post_job("Export", dict(case=self.case, inode_id=inode_id))

    filename, content_type, fd = table_renderer.make_archive_filename(inode_id)
    result = "<a href='%s'>%s</a><br />" % (filename, fd.inode_id)
    
    try:
        filename = "inodes/%s_summary.html" % inode_id
        fd.html_export
        result += "<a href='%s'><img src=images/browse.png /></a>" % (filename,)
    except AttributeError: pass

    #if table_renderer.explain_inodes:
    ## Add a link to the explaination page:
    filename = "inodes/%s_explain.html" % inode_id
    result+="<a href='%s'><img src=images/question.png /></a>" %(filename,)

    ## Check if there are annotations for this
    dbh = DB.DBO(self.case)
    dbh.execute("select * from annotate where inode_id=%r", inode_id)
    for row in dbh:
        result += "<br>%s" % row['note']

    return result
Ejemplo n.º 8
0
    def wait_for_scan(self, cookie):
        """ Waits for scanners to complete """
        import pyflag.Farm as Farm

        while Farm.get_cookie_reference(cookie)>0:
            time.sleep(0.5)
        
        return
    
        print "Waiting for cookie %s" % cookie
        pdbh = DB.DBO()

        ## Often this process owns a worker as well. In that case we can wake it up:
        import pyflag.Farm as Farm
        
        #Farm.wake_workers()
        
        ## Wait until there are no more jobs left.
        while 1:
            pdbh.execute("select * from jobs where cookie=%r limit 1", (cookie))
            row = pdbh.fetch()
            if not row: break
            
            time.sleep(1)
Ejemplo n.º 9
0
    said inode is up to the relevant dictionary version.

    unique requests the indexer to produce a single hit for each inode
    (i.e. it only shows unique hits). If unique is False the indexer
    will generate offsets for all hits in this inode.
    """
    word_id = insert_dictionary_word(word, word_type)

    if type(inode_ids) != type(list):
        inode_ids = (inode_ids, )

    for inode_id in inode_ids:
        schedule_inode_index(case, inode_id, word_id)

    ## Wake the workers:
    Farm.wake_workers()


def count_outdated_inodes(case, sql, word_id=None, unique=True):
    """ This function counts the number of inodes outstanding to be scanned.

    sql is a subquery which is expected to return a list of inode_ids.

    word_id is the word which we count the outdated inodes with
    respect to. If word_id is None, or omitted we simply count all
    inodes in the sql provided.
    
    unique is the type of index (a unique index of a detailed index).
    """
    dbh = DB.DBO(case)
Ejemplo n.º 10
0
        import pyflag.AJAXUI as AJAXUI

        UI.UI = AJAXUI.AJAXUI
    elif config.THEME == "XML":
        import pyflag.XMLUI as XMLUI

        UI.UI = XMLUI.XMLUI
    else:
        import pyflag.HTMLUI as HTMLUI

        UI.UI = HTMLUI.HTMLUI

    ## Start the workers
    import pyflag.Farm as Farm

    Farm.start_workers()

    ## Check the schema:
    FlagFramework.check_schema()

    try:
        try:
            Server(HandlerClass=FlagServerHandler)

        except Exception, e:
            pyflaglog.log(pyflaglog.ERROR, "Error %s" % e)

    finally:
        pyflaglog.log(pyflaglog.INFO, "Terminating PyFlag Server")
        sys.exit(0)
Ejemplo n.º 11
0
def run(keepalive=None):
    global last_mtime, offset, output_fd

    create_output_file()

    print "Created output_fd"
    ## Get the PCAPFS class and instantiate it:
    pcapfs = Registry.FILESYSTEMS.dispatch("PCAP Filesystem")(config.case)
    pcapfs.mount_point = config.mountpoint
    pcapfs.VFSCreate(None,
                     "I%s" % config.iosource,
                     config.mountpoint,
                     directory=True)

    pcap_dbh = DB.DBO(config.case)
    pcap_dbh.mass_insert_start("pcap")

    pcap_dbh.execute("select max(id) as m from pcap")
    pcap_id = pcap_dbh.fetch()['m'] or 0
    cookie, processor = pcapfs.make_processor(config.iosource, scanners)

    last_time = 0

    files_we_have = set()
    update_files(files_we_have)
    log_fd = open(config.log, "a")
    last_mtime = os.stat(directory).st_mtime

    while 1:
        ## Check that our parent process is still there:
        fds = select.select([keepalive], [], [], 0)
        if fds[0]:
            data = os.read(fds[0][0], 1024)
            if len(data) == 0:
                pyflaglog.log(pyflaglog.INFO,
                              "Main process disappeared - closing children")
                ## Parent quit - kill our child
                os.kill(pid, signal.SIGABRT)
                os._exit(0)

        t = os.stat(directory).st_mtime
        if t >= last_mtime:
            last_mtime = t
            files = os.listdir(directory)
            files.sort()
            count = 0

            if not os.access(config.lock, os.F_OK):
                for f in files:
                    count += 1
                    if f in files_we_have: continue

                    ## Detect if the lock file appeared:
                    if os.access(config.lock, os.F_OK): break

                    if (count % 10) == 0 and config.MAXIMUM_WORKER_MEMORY > 0:
                        Farm.check_mem(finish)

                    filename = "%s/%s" % (directory, f)
                    if config.log:
                        log_fd.write(f + "\n")
                        log_fd.flush()
                        files_we_have.add(f)

                    load_file(filename, processor, pcap_dbh)

                    last_time = time.time()
            else:
                print "Lock file found"

            if config.single:
                ## Wait untill all our jobs are done
                pdbh = DB.DBO()
                while 1:
                    pdbh.execute(
                        "select count(*) as c from jobs where cookie = %r",
                        cookie)
                    row = pdbh.fetch()
                    if row and row['c'] > 0:
                        time.sleep(5)
                        continue
                    else:
                        break

                sys.exit(0)

            ## We need to flush the decoder:
            if time.time() - last_time > config.timeout:
                print "Flushing reassembler"
                processor.flush()
                last_time = time.time()

        print "%s: Sleeping for %s seconds" % (time.ctime(), config.sleep)
        time.sleep(config.sleep)
Ejemplo n.º 12
0
            output_fd.write(packet_data)
        else:
            offset += packet.caplen

    if output_fd:
        output_fd.flush()

    pcap_dbh.delete("connection_details", where="inode_id is null")
    pcap_dbh.mass_insert_commit()


last_mtime = 0
offset = 0

## Start up some workers if needed:
Farm.start_workers()


def update_files(files_we_have):
    try:
        log_fd = open(config.log)
        print "Reading log file"
        for l in log_fd:
            files_we_have.add(l.strip())
        print "Done - added %s files from log" % len(files_we_have)
        log_fd.close()
    except IOError:
        pass


def run(keepalive=None):
Ejemplo n.º 13
0
def run(keepalive=None):
    global last_mtime, offset, output_fd

    create_output_file()

    print "Created output_fd"
    ## Get the PCAPFS class and instantiate it:
    pcapfs = Registry.FILESYSTEMS.dispatch("PCAP Filesystem")(config.case)
    pcapfs.mount_point = config.mountpoint
    pcapfs.VFSCreate(None, "I%s" % config.iosource, config.mountpoint, 
                     directory=True)

    pcap_dbh = DB.DBO(config.case)
    pcap_dbh.mass_insert_start("pcap")

    pcap_dbh.execute("select max(id) as m from pcap")
    pcap_id = pcap_dbh.fetch()['m'] or 0
    cookie, processor = pcapfs.make_processor(config.iosource, scanners)

    last_time = 0

    files_we_have = set()
    update_files(files_we_have)
    log_fd = open(config.log, "a")
    last_mtime = os.stat(directory).st_mtime

    while 1:
        ## Check that our parent process is still there:
        fds = select.select([keepalive],[],[], 0)
        if fds[0]:
            data = os.read(fds[0][0],1024)
            if len(data)==0:
                pyflaglog.log(pyflaglog.INFO, "Main process disappeared - closing children")
                ## Parent quit - kill our child
                os.kill(pid,signal.SIGABRT)
                os._exit(0)
        
        t = os.stat(directory).st_mtime
        if t>=last_mtime:
            last_mtime = t
            files = os.listdir(directory)
            files.sort()
            count = 0

            if not os.access(config.lock, os.F_OK):
                for f in files:
                   count +=1
                   if f in files_we_have: continue

                   ## Detect if the lock file appeared:
                   if os.access(config.lock, os.F_OK): break

                   if (count % 10) ==0 and config.MAXIMUM_WORKER_MEMORY > 0:
                       Farm.check_mem(finish)

                   filename = "%s/%s" % (directory,f)
                   if config.log:
                       log_fd.write(f+"\n")
                       log_fd.flush()
                       files_we_have.add(f)

                   load_file(filename, processor, pcap_dbh)

                   last_time = time.time()
            else:
               print "Lock file found"

            if config.single:
                ## Wait untill all our jobs are done
                pdbh = DB.DBO()
                while 1:
                    pdbh.execute("select count(*) as c from jobs where cookie = %r", cookie)
                    row = pdbh.fetch()
                    if row and row['c'] >0:
                        time.sleep(5)
                        continue
                    else:
                        break

                sys.exit(0)

            ## We need to flush the decoder:
            if time.time() - last_time > config.timeout:
                print "Flushing reassembler"
                processor.flush()
                last_time = time.time()

        print "%s: Sleeping for %s seconds" % (time.ctime(), config.sleep)
        time.sleep(config.sleep)
Ejemplo n.º 14
0
            output_fd.write(packet_data)
        else:
            offset += packet.caplen

    if output_fd:
        output_fd.flush()
        
    pcap_dbh.delete("connection_details",
                    where = "inode_id is null")
    pcap_dbh.mass_insert_commit() 

last_mtime = 0
offset = 0

## Start up some workers if needed:
Farm.start_workers()

def update_files(files_we_have):
    try:
        log_fd = open(config.log)
        print "Reading log file"
        for l in log_fd:
            files_we_have.add(l.strip())
        print "Done - added %s files from log" % len(files_we_have)
        log_fd.close()
    except IOError:
        pass

    
def run(keepalive=None):
    global last_mtime, offset, output_fd
Ejemplo n.º 15
0
    flag = FlagFramework.Flag()
    FlagFramework.GLOBAL_FLAG_OBJ = flag
    #Set the UI module to produce HTML
    if config.THEME == "AJAX":
        import pyflag.AJAXUI as AJAXUI
        UI.UI = AJAXUI.AJAXUI
    elif config.THEME == "XML":
        import pyflag.XMLUI as XMLUI
        UI.UI = XMLUI.XMLUI
    else:
        import pyflag.HTMLUI as HTMLUI
        UI.UI = HTMLUI.HTMLUI

    ## Start the workers
    import pyflag.Farm as Farm
    Farm.start_workers()

    ## Check the schema:
    FlagFramework.check_schema()

    try:
        try:
            Server(HandlerClass=FlagServerHandler)

        except Exception, e:
            pyflaglog.log(pyflaglog.ERROR, "Error %s" % e)

    finally:
        pyflaglog.log(pyflaglog.INFO, "Terminating PyFlag Server")
        sys.exit(0)