def pane_cb(path, result):
query['order'] = 'Filename'
if path == '': path = '/'
## If we are asked to show a file, we will show the
## contents of the directory the file is in:
fsfd = FileSystem.DBFS(query["case"])
if not fsfd.isdir(path):
path = os.path.dirname(path)
self.make_table_widget(
['URN', 'Name', 'Size', 'Modified'],
query,
result,
where=DB.expand(
"path=%r and (isnull(type) or type!='directory')",
(path)),
)
result.toolbar(text=DB.expand("Scan %s", path),
icon="examine.png",
link=query_type(family="Load Data",
report="ScanFS",
path=path,
case=query['case']),
pane='popup')
def longls(self,path='/', dirs = None):
dbh=DB.DBO(self.case)
if self.isdir(path):
## If we are listing a directory, we list the files inside the directory
if not path.endswith('/'):
path=path+'/'
where = DB.expand(" path=%r " ,path)
else:
## We are listing the exact file specified:
where = DB.expand(" path=%r and name=%r", (
FlagFramework.normpath(posixpath.dirname(path)+'/'),
posixpath.basename(path)))
mode =''
if(dirs == 1):
mode=" and file.mode like 'd%'"
elif(dirs == 0):
mode=" and file.mode like 'r%'"
dbh.execute("select * from file where %s %s", (where, mode))
result = [dent for dent in dbh]
for dent in result:
if dent['inode']:
dbh.execute("select * from inode where inode = %r", dent['inode'])
data = dbh.fetch()
if data:
dent.update(data)
return result
def pane_cb(path, tmp):
query['order'] = 'Filename'
## If we are asked to show a file, we will show the
## contents of the directory the file is in:
fsfd = FileSystem.DBFS(query["case"])
if not fsfd.isdir(path):
path = os.path.dirname(path)
tmp.table(
elements=[
InodeIDType(case=query['case']),
FilenameType(basename=True, case=query['case']),
DeletedType(),
IntegerType('File Size', 'size'),
TimestampType('Last Modified', 'mtime'),
StringType('Mode', 'mode', table='file')
],
table='inode',
where=DB.expand("file.path=%r and file.mode!='d/d'",
(path + '/')),
case=query['case'],
pagesize=10,
filter="filter2",
)
target = tmp.defaults.get('open_tree', '/')
tmp.toolbar(text=DB.expand("Scan %s", target),
icon="examine.png",
link=query_type(family="Load Data",
report="ScanFS",
path=target,
case=query['case']),
pane='popup')
def longls(self, path='/', dirs=None):
dbh = DB.DBO(self.case)
if self.isdir(path):
## If we are listing a directory, we list the files inside the directory
if not path.endswith('/'):
path = path + '/'
where = DB.expand(" path=%r ", path)
else:
## We are listing the exact file specified:
where = DB.expand(
" path=%r and name=%r",
(FlagFramework.normpath(posixpath.dirname(path) + '/'),
posixpath.basename(path)))
mode = ''
if (dirs == 1):
mode = " and file.mode like 'd%'"
elif (dirs == 0):
mode = " and file.mode like 'r%'"
dbh.execute("select * from file where %s %s", (where, mode))
result = [dent for dent in dbh]
for dent in result:
if dent['inode']:
dbh.execute("select * from inode where inode = %r",
dent['inode'])
data = dbh.fetch()
if data:
dent.update(data)
return result
def pane_cb(path,tmp):
query['order']='Filename'
## If we are asked to show a file, we will show the
## contents of the directory the file is in:
fsfd = FileSystem.DBFS( query["case"])
if not fsfd.isdir(path):
path=os.path.dirname(path)
tmp.table(
elements = [ InodeIDType(case=query['case']),
FilenameType(basename=True, case=query['case']),
DeletedType(),
IntegerType('File Size','size'),
TimestampType('Last Modified','mtime'),
StringType('Mode','mode', table='file') ],
table='inode',
where=DB.expand("file.path=%r and file.mode!='d/d'", (path+'/')),
case=query['case'],
pagesize=10,
filter="filter2",
)
target = tmp.defaults.get('open_tree','/')
tmp.toolbar(text=DB.expand("Scan %s",target),
icon="examine.png",
link=query_type(family="Load Data", report="ScanFS",
path=target,
case=query['case']), pane='popup'
)
def __str__(self):
postfix = ''
## Some tags are never allowed to be outputted
if self.name not in self.allowable_tags:
if self.name in self.forbidden_tag:
return ''
#print "Rejected tag %s" % self.name
return self.innerHTML()
if self.name == 'head':
self.children = [self.header,] + self.children
elif self.name =='body':
self.children = [self.body_extra, ] + self.children
## Frames without src are filtered because IE Whinges:
if self.name == 'iframe' and 'src' not in self.attributes:
return ''
attributes = "".join([" %s='%s'" % (k,v) for k,v \
in self.attributes.items() if k in \
self.allowable_attributes])
if 'style' in self.attributes:
attributes += ' style=%r' % self.css_filter(self.attributes['style'] or '')
if 'http-equiv' in self.attributes:
if self.attributes['http-equiv'].lower() == "content-type":
## PF _always_ outputs in utf8
attributes += ' http-equiv = "Content-Type" content="text/html; charset=UTF-8"'
if 'src' in self.attributes:
attributes += ' src=%s' % self.resolve_reference(self.attributes['src'])
try:
if 'href' in self.attributes:
if self.name == 'link':
attributes += " href=%s" % self.resolve_reference(self.attributes['href'], 'text/css')
else:
attributes += DB.expand(' href="javascript: alert(%r)"',
iri_to_uri(DB.expand("%s",self.attributes['href'])[:100]))
postfix = self.mark_link(self.attributes['href'])
except: pass
## CSS needs to be filtered extra well
if self.name == 'style':
return expand("<style %s>%s</style>" , (attributes,
self.css_filter(self.innerHTML())))
if self.type == 'selfclose':
return expand("<%s%s/>%s" , (self.name, attributes, postfix))
else:
return expand("<%s%s>%s</%s>%s", (self.name, attributes,
self.innerHTML(),
self.name,postfix))
class Index(Farm.Task):
""" A task to index an inode with the dictionary """
def run(self, case, inode_id, *args):
global INDEX
if not INDEX: reindex()
try:
desired_version = int(args[0])
except:
desired_version = INDEX_VERSION
## Did they want a detailed index or a unique index?
unique = desired_version < 2**30
## In unique mode we want to generate one hit per scan job per
## word
if unique:
INDEX.clear_set()
pyflaglog.log(
pyflaglog.VERBOSE_DEBUG,
"Indexing inode_id %s (version %s)" % (inode_id, desired_version))
fsfd = FileSystem.DBFS(case)
fd = fsfd.open(inode_id=inode_id)
buff_offset = 0
dbh = DB.DBO(case)
## Clear old hits:
dbh.check_index("LogicalIndexOffsets", "inode_id")
dbh.delete("LogicalIndexOffsets",
where=DB.expand("inode_id = %r", inode_id))
## Get ready for scan
dbh.mass_insert_start("LogicalIndexOffsets")
while 1:
data = fd.read(1024 * 1024)
if len(data) == 0: break
for offset, matches in INDEX.index_buffer(data, unique=unique):
for id, length in matches:
dbh.mass_insert(inode_id=inode_id,
word_id=id,
offset=offset + buff_offset,
length=length)
buff_offset += len(data)
dbh.mass_insert_commit()
## Update the version
dbh.update("inode",
where=DB.expand('inode_id = %r', inode_id),
version=desired_version)
def execute(self):
for iosource in self.args:
dbh = DB.DBO(self.environment._CASE)
dbh2 = dbh.clone()
dbh.delete('inode', where=DB.expand("inode like 'I%s|%%'", iosource))
dbh.execute("select * from filesystems where iosource = %r", iosource)
for row in dbh:
dbh2.delete('file', where=DB.expand("path like '%s%%'", iosource))
dbh.delete("iosources", where=DB.expand("name=%r", iosource))
yield "Removed IOSource %s" % iosource
def run(self, case, inode_id, *args):
global INDEX
if not INDEX: reindex()
try:
desired_version = int(args[0])
except:
desired_version = INDEX_VERSION
## Did they want a detailed index or a unique index?
unique = desired_version < 2**30
## In unique mode we want to generate one hit per scan job per
## word
if unique:
INDEX.clear_set()
pyflaglog.log(pyflaglog.VERBOSE_DEBUG, "Indexing inode_id %s (version %s)" % (inode_id, desired_version))
fsfd = FileSystem.DBFS(case)
fd = fsfd.open(inode_id=inode_id)
buff_offset = 0
dbh = DB.DBO(case)
## Clear old hits:
dbh.check_index("LogicalIndexOffsets", "inode_id")
dbh.delete("LogicalIndexOffsets", where=DB.expand("inode_id = %r",
inode_id))
## Get ready for scan
dbh.mass_insert_start("LogicalIndexOffsets")
while 1:
data = fd.read(1024*1024)
if len(data)==0: break
for offset, matches in INDEX.index_buffer(data, unique = unique):
for id, length in matches:
dbh.mass_insert(
inode_id = inode_id,
word_id = id,
offset = offset + buff_offset,
length = length)
buff_offset += len(data)
dbh.mass_insert_commit()
## Update the version
dbh.update("inode",
where = DB.expand('inode_id = %r', inode_id),
version = desired_version)
def execute(self):
for iosource in self.args:
dbh = DB.DBO(self.environment._CASE)
dbh2 = dbh.clone()
dbh.delete('inode',
where=DB.expand("inode like 'I%s|%%'", iosource))
dbh.execute("select * from filesystems where iosource = %r",
iosource)
for row in dbh:
dbh2.delete('file',
where=DB.expand("path like '%s%%'", iosource))
dbh.delete("iosources", where=DB.expand("name=%r", iosource))
yield "Removed IOSource %s" % iosource
class ThumbnailType(InodeIDType):
""" A Column showing thumbnails of inodes """
def __init__(self, name='Thumbnail', **args ):
InodeIDType.__init__(self, **args)
self.fsfd = FileSystem.DBFS(self.case)
self.name = name
def select(self):
return "%s.inode_id" % self.table
## When exporting to html we need to export the thumbnail too:
def render_html(self, inode_id, table_renderer):
ct=''
try:
fd = self.fsfd.open(inode_id = inode_id)
image = Graph.Thumbnailer(fd, 200)
inode_filename, ct, fd = table_renderer.make_archive_filename(inode_id)
filename, ct, fd = table_renderer.make_archive_filename(inode_id, directory = "thumbnails/")
table_renderer.add_file_from_string(filename,
image.display())
except IOError,e:
print e
return "<a href=%r ><img src='images/broken.png' /></a>" % inode_filename
InodeIDType.render_html(self, inode_id, table_renderer)
table_renderer.add_file_to_archive(inode_id)
return DB.expand("<a href=%r type=%r ><img src=%r /></a>",
(inode_filename, ct, filename))
def form(self, query, result):
result.textfield("Inode ID", 'inode_id')
dbh = DB.DBO(query['case'])
try:
result.selector("Table Name", 'table_name', DB.expand('select name as `key`,name as value from sqlite where inode_id=%r', query['inode_id']), case=query['case'])
except KeyError, e:
pass
def display(self, query, result):
path = query['path']
key = query['key']
result.heading("Registry Key Contents")
result.text(DB.expand("Key %s/%s:", (path, key)),
style='red',
font='typewriter')
dbh = DB.DBO(query['case'])
def hexdump(query, out):
""" Show the hexdump for the key """
dbh.execute(
"select value from reg where path=%r and reg_key=%r limit 1",
(path, key))
row = dbh.fetch()
if row:
HexDump(row['value'], out).dump()
return out
def strings(query, out):
""" Draw the strings in the key """
out.para("not implimented yet")
return out
def stats(query, out):
""" display stats on a key """
out.para("not implemented yet")
return out
result.notebook(names=["HexDump", "Strings", "Statistics"],
callbacks=[hexdump, strings, stats],
context="display_mode")
def drop_table(case, name):
""" Drops the log table tablename """
if not name: return
dbh = DB.DBO(case)
pyflaglog.log(pyflaglog.DEBUG, "Dropping log table %s in case %s" % (name, case))
dbh.execute("select * from log_tables where table_name = %r limit 1" , name)
row = dbh.fetch()
## Table not found
if not row:
return
preset = row['preset']
## Get the driver for this table:
log = load_preset(case, preset)
log.drop(name)
## Ask the driver to remove its table:
dbh.delete("log_tables",
where= DB.expand("table_name = %r ", name));
## Make sure that the reports get all reset
FlagFramework.reset_all(family='Load Data', report="Load Preset Log File",
table = name, case=case)
def __init__(self, case, fd, inode):
File.__init__(self, case, fd, inode)
# strategy: must determine basepath from parent, get our path
# from db and then return the file:
## Note this _must_ work because we can only ever be called on
## a mounted iosource - it is an error otherwise:
basepath = fd.io.directory
self.case = case
dbh = DB.DBO(case)
dbh.check_index("file", "inode")
dbh.execute("select path,name from file where inode=%r limit 1", (inode))
row = dbh.fetch()
path = row["path"]
mount_point = fd.io.mount_point
## Prune the path down to the mount point:
if path[: len(mount_point)] != mount_point:
raise RuntimeError(DB.expand("Something went wrong - %s should be mounted on %s", (path, mount_point)))
path = path[len(mount_point) :]
path = basepath + "/" + path + "/" + row["name"]
if not path.startswith(posixpath.normpath(config.UPLOADDIR)):
path = FlagFramework.sane_join(config.UPLOADDIR, path)
if os.path.isdir(path):
self.fd = StringIO.StringIO("")
else:
self.fd = open(path, "r")
s = os.stat(path)
self.size = s.st_size
def pane_cb(path, result):
tlds = path.split("/")
try:
result.defaults.set('filter', DB.expand('TLD = %r and "Content Type" contains html',tlds[1]))
Reports.CaseTableReports.display(self, query, result)
except IndexError:
result.para("Click on a TLD to view all URLs from that TLD")
def pane_cb(path, result):
if not path.endswith('/'): path=path+'/'
result.heading("Path is %s" % path)
case = query['case']
dbh = DB.DBO(case)
fsfd = Registry.FILESYSTEMS.dispatch(query['fstype'])(case)
## Try to see if the directory is already loaded:
dbh.execute("select * from file where path=%r and name=''", path)
if not dbh.fetch():
fsfd.load(mount_point = query['mount_point'], iosource_name= query['iosource'],
directory = path)
## Now display the table
result.table(
elements = [ InodeIDType(case=query['case']),
FilenameType(case=query['case']),
DeletedType(),
IntegerType(name='File Size',column='size'),
TimestampType(name = 'Last Modified',column = 'mtime'),
],
table='inode',
where=DB.expand("file.path=%r and file.mode!='d/d'",(path)),
case = query['case'],
pagesize=10,
)
def glob_sql(pattern):
path, name = posixpath.split(pattern)
if globbing_re.search(path):
path_sql = "path rlike '^%s/?$'" % translate(path)
else:
## Ensure that path has a / at the end:
if not path.endswith("/"): path = path + '/'
path_sql = "path='%s'" % path
if globbing_re.search(name):
name_sql = "name rlike '^%s$'" % translate(name)
else:
name_sql = DB.expand("name=%r", name)
if name and path:
sql = "select concat(path,name) as path from file where %s and %s group by file.path,file.name" % (
path_sql, name_sql)
elif name:
sql = "select concat(path,name) as path from file where %s group by file.path,file.name" % name_sql
elif path:
#sql = "%s and name=''" % path_sql
sql = "select path from file where %s group by file.path" % path_sql
else:
## Dont return anything for an empty glob
sql = "select * from file where 1=0"
return sql
def explain(self, query, result):
name = self.fd.name
## Trim the upload directory if present
if name.startswith(config.UPLOADDIR):
name = name[len(config.UPLOADDIR) :]
result.row("Filename", DB.expand("%s", name), **{"class": "explainrow"})
def get_factories(case, scanners):
""" Scanner factories are obtained from the Store or created as
required. Scanners is a list in the form case:scanner
"""
## Ensure dependencies are satisfied
scanners = ScannerUtils.fill_in_dependancies(scanners)
## First prepare the required factories:
result = []
for scanner in scanners:
key = DB.expand("%s:%s", (case, scanner))
try:
f = factories.get(key)
except KeyError:
try:
cls = Registry.SCANNERS.dispatch(scanner)
except:
# pyflaglog.log(pyflaglog.WARNING, "Unable to find scanner for %s", scanner)
continue
# Instatiate it:
import pyflag.FileSystem as FileSystem
f = cls(FileSystem.DBFS(case))
## Initialise it:
f.prepare()
## Store it:
factories.put(f, key=key)
result.append(f)
return result
def analyse(self, query):
context = self.get_context(query)
word_id = Indexing.insert_dictionary_word(query['word'], query['type'])
pdbh = DB.DBO()
sql = DB.expand("select inode.inode_id as `inode_id` "\
"%s where (%s) and (%s)", (context.get('tables',''),
context.get('inode_sql','1'),
context.get('where','1')))
Indexing.schedule_inode_index_sql(query['case'],
sql,
word_id,
query['cookie'],
unique=True)
## Now wait here until everyone is finished:
while 1:
pdbh.execute("select count(*) as c from jobs where cookie=%r",
query['cookie'])
row = pdbh.fetch()
self.rows_left = row['c']
if row['c'] == 0: break
time.sleep(1)
return 1
def glob_sql(pattern):
path,name = posixpath.split(pattern)
if globbing_re.search(path):
path_sql = "path rlike '^%s/?$'" % translate(path)
else:
## Ensure that path has a / at the end:
if not path.endswith("/"): path=path+'/'
path_sql = "path='%s'" % path
if globbing_re.search(name):
name_sql = "name rlike '^%s$'" % translate(name)
else:
name_sql = DB.expand("name=%r", name)
if name and path:
sql = "select concat(path,name) as path from file where %s and %s group by file.path,file.name" % (path_sql,name_sql)
elif name:
sql = "select concat(path,name) as path from file where %s group by file.path,file.name" % name_sql
elif path:
#sql = "%s and name=''" % path_sql
sql = "select path from file where %s group by file.path" % path_sql
else:
## Dont return anything for an empty glob
sql = "select * from file where 1=0"
return sql
def display(self,query,result):
path=query['path']
key=query['key']
result.heading("Registry Key Contents")
result.text(DB.expand("Key %s/%s:", (path,key)),style='red',font='typewriter')
dbh=DB.DBO(query['case'])
def hexdump(query,out):
""" Show the hexdump for the key """
dbh.execute("select value from reg where path=%r and reg_key=%r limit 1",(path,key))
row=dbh.fetch()
if row:
HexDump(row['value'],out).dump()
return out
def strings(query,out):
""" Draw the strings in the key """
out.para("not implimented yet")
return out
def stats(query,out):
""" display stats on a key """
out.para("not implemented yet")
return out
result.notebook(
names=["HexDump","Strings","Statistics"],
callbacks=[hexdump,strings,stats],
context="display_mode"
)
def explain(self, query, result):
name = self.fd.name
## Trim the upload directory if present
if name.startswith(config.UPLOADDIR):
name = name[len(config.UPLOADDIR):]
result.row("Filename",DB.expand("%s", name), **{'class':'explainrow'})
def finish(self):
self.dbh.mass_insert_commit()
## Update the version
self.dbh.update("inode",
where = DB.expand('inode_id = %r', self.inode_id),
version = INDEX_VERSION)
del self.dbh
def finish(self):
self.dbh.mass_insert_commit()
## Update the version
self.dbh.update("inode",
where=DB.expand('inode_id = %r', self.inode_id),
version=INDEX_VERSION)
del self.dbh
def insert_into_table(mode ,root ,name):
rel_root = FlagFramework.normpath(DB.expand("%s/%s/" ,
(mount_point, root[len(path):])))
try:
s=os.lstat(os.path.join(root,name))
except OSError:
pyflaglog.log(pyflaglog.WARNING, DB.expand("Unable to stat %s - mount the directory with the uid option", root))
return
inode = DB.expand("I%s|M%s", (iosource_name, s.st_ino))
dbh_inode.insert('inode',
inode = inode,
uid = s.st_uid,
gid = s.st_gid,
_mtime = "from_unixtime(%s)" % s.st_mtime,
_atime = "from_unixtime(%s)" % s.st_atime,
_ctime = "from_unixtime(%s)" % s.st_ctime,
status = 'alloc',
mode = str(oct(s.st_mode)),
size = s.st_size,
_fast=True)
inode_id = dbh_inode.autoincrement()
dbh_file.mass_insert(inode_id = inode_id,
inode = inode,
mode = mode,
status = 'alloc',
path = rel_root,
name = name)
## If needed schedule inode for scanning:
if scanners and mode=='r/r':
pdbh.mass_insert(
command = 'Scan',
arg1 = self.case,
arg2 = inode,
arg3= scanner_string,
cookie=cookie,
)
## Fixme - handle symlinks
try:
link=os.readlink(DB.expand("%s/%s", (root,name)).encode("utf8"))
except OSError:
link=''
def drop_preset(preset):
""" Drops the specified preset name """
pyflaglog.log(pyflaglog.DEBUG, "Droppping preset %s" % preset)
for case, table in find_tables(preset):
drop_table(case, table)
dbh = DB.DBO()
if preset:
dbh.delete("log_presets", where=DB.expand("name = %r",preset))
def insert_whois_cache(sql_ip, id, ipinfo):
dbh = DB.DBO()
dbh.insert("whois_cache",
_ip=sql_ip,
id=id,
_geoip_city=DB.expand(
"(select id from geoip_city where city=%r "
"limit 1)", (ipinfo.get('city', 'Unknown'), )) or '',
_geoip_country=DB.expand(
"(select id from geoip_country where country"
"=%r limit 1)", (ipinfo.get("country_code3", "---"), )),
_geoip_org=DB.expand(
"(select id from geoip_org where org"
"=%r limit 1)", (ipinfo.get("org", "Unknown"), )),
_geoip_isp=DB.expand(
"(select id from geoip_isp where isp"
"=%r limit 1)", (ipinfo.get("isp", "Unknown"), )),
_fast=True)
def drop_preset(preset):
""" Drops the specified preset name """
pyflaglog.log(pyflaglog.DEBUG, "Droppping preset %s" % preset)
for case, table in find_tables(preset):
drop_table(case, table)
dbh = DB.DBO()
if preset:
dbh.delete("log_presets", where=DB.expand("name = %r", preset))
def operator_hit(self, column, operator, arg):
""" Search for a hit in the dictionary """
## Try to work out if we need to reindex:
reindex = False
dbh = DB.DBO()
dbh.execute("select id from dictionary where word = %r limit 1", arg)
row = dbh.fetch()
if self.ui:
## If the word is not in the dictionary, we definitely want to reindex
if not row:
count, total, tables, sql = self.outstanding_inodes()
message = "Word %s is not in the dictionary" % arg
## If the word is in the dictionary we want to know how may
## inodes are outdated
else:
count, total, tables, sql = self.outstanding_inodes(
word_id=row['id'])
message = "There are some inodes which are not up to date"
## Any inodes to process?
if count > 0:
reindex = True
## We do not need to reindex - just do it
if not reindex:
return DB.expand(
"(%s = %s)",
(self.escape_column_name(self.column), row.get('id', 0)))
## Allow the user to reindex the currently selected set of
## inodes with a new dictionary based on the new word
self.ui.heading(message)
self.ui.para(
"This will affect %s inodes and require rescanning %s bytes" %
(count, total))
## Make up the link for the use:
context = FlagFramework.STORE.put(
dict(tables=tables,
inode_sql=sql,
previous_query=self.ui.defaults,
target='parent_pane',
where=self.table_where_clause))
link = query_type(report="Add Word",
family="Keyword Indexing",
case=self.case,
context=context,
word=arg)
self.ui.link("Click here to scan these inodes", link, pane='self')
## Ok - Show the error to the user:
raise self.ui
def insert_whois_cache(sql_ip, id, ipinfo):
dbh = DB.DBO()
dbh.insert("whois_cache",
_ip = sql_ip,
id = id,
_geoip_city = DB.expand("(select id from geoip_city where city=%r "
"limit 1)", (ipinfo.get('city','Unknown'),)) or '',
_geoip_country = DB.expand("(select id from geoip_country where country"
"=%r limit 1)", (ipinfo.get("country_code3","---"),)),
_geoip_org = DB.expand("(select id from geoip_org where org"
"=%r limit 1)", (ipinfo.get("org","Unknown"),)),
_geoip_isp = DB.expand("(select id from geoip_isp where isp"
"=%r limit 1)", (ipinfo.get("isp","Unknown"),)),
_fast = True
)
def pane_cb(path,result):
query['order']='Filename'
if path=='': path='/'
## If we are asked to show a file, we will show the
## contents of the directory the file is in:
fsfd = FileSystem.DBFS( query["case"])
if not fsfd.isdir(path):
path=os.path.dirname(path)
self.make_table_widget(['URN','Name',
'Size','Modified'],
query, result,
where=DB.expand("path=%r and (isnull(type) or type!='directory')", (path)),)
result.toolbar(text=DB.expand("Scan %s",path),
icon="examine.png",
link=query_type(family="Load Data", report="ScanFS",
path=path,
case=query['case']), pane='popup'
)
def form(self, query, result):
result.textfield("Inode ID", "inode_id")
dbh = DB.DBO(query["case"])
try:
result.selector(
"Table Name",
"table_name",
DB.expand("select name as `key`,name as value from sqlite where inode_id=%r", query["inode_id"]),
case=query["case"],
)
except KeyError, e:
pass
def longls(self,path='/', dirs = None):
dbh=DB.DBO(self.case)
if self.isdir(path):
## If we are listing a directory, we list the files inside the directory
where = DB.expand(" path=%r " ,path)
else:
## We are listing the exact file specified:
where = DB.expand(" path=%r and name=%r", (
FlagFramework.normpath(posixpath.dirname(path)),
posixpath.basename(path)))
## Only list directories
if dirs:
where += " and isnull(inode_id) "
else:
where += " and not isnull(inode_id) "
dbh.execute("select * from vfs where %s group by inode_id,path,name", (where))
result = [dent for dent in dbh]
return result
def operator_hit(self, column, operator, arg):
""" Search for a hit in the dictionary """
## Try to work out if we need to reindex:
reindex = False
dbh = DB.DBO()
dbh.execute("select id from dictionary where word = %r limit 1", arg)
row = dbh.fetch()
if self.ui:
## If the word is not in the dictionary, we definitely want to reindex
if not row:
count, total, tables, sql = self.outstanding_inodes()
message = "Word %s is not in the dictionary" % arg
## If the word is in the dictionary we want to know how may
## inodes are outdated
else:
count, total, tables, sql = self.outstanding_inodes(word_id = row['id'])
message = "There are some inodes which are not up to date"
## Any inodes to process?
if count > 0:
reindex = True
## We do not need to reindex - just do it
if not reindex:
return DB.expand("(%s = %s)",
(self.escape_column_name(self.column),
row.get('id',0)))
## Allow the user to reindex the currently selected set of
## inodes with a new dictionary based on the new word
self.ui.heading(message)
self.ui.para("This will affect %s inodes and require rescanning %s bytes" % (count,total))
## Make up the link for the use:
context = FlagFramework.STORE.put(dict(tables = tables,
inode_sql = sql,
previous_query = self.ui.defaults,
target = 'parent_pane',
where = self.table_where_clause
))
link = query_type(report = "Add Word", family = "Keyword Indexing",
case = self.case,
context = context,
word = arg)
self.ui.link("Click here to scan these inodes", link,
pane = 'self')
## Ok - Show the error to the user:
raise self.ui
def delete_case(case):
""" A helper function which deletes the case """
dbh = DB.DBO(None)
## Broadcast that the case is about to be dropped (This broadcasts
## to the workers)
dbh.insert('jobs',command = "DropCase", state='broadcast', arg1=case, cookie=0, _fast = True)
## This sends an event to our process:
post_event('reset', case)
## Remove any jobs that may be outstanding (dont touch the
## currently processing jobs)
dbh.delete('jobs',DB.expand("arg1=%r and state='pending' " , case),
_fast= True)
## Now wait until there are no more processing jobs:
total_time = 0
while 1:
dbh.execute("select * from jobs where arg1=%r and state='processing' limit 1", case)
row = dbh.fetch()
if row:
time.sleep(2)
total_time += 2
if total_time > 20:
pyflaglog.log(pyflaglog.WARNING,"Outstanding jobs remain in %s. Removing the case anyway." % case)
dbh.execute("delete from jobs where arg1=%r and state='processing'",case)
break
pyflaglog.log(pyflaglog.INFO, "Waiting for outstanding jobs in case %r to be completed" % case)
else:
break
try:
#Delete the case from the database
dbh.delete('meta',DB.expand("property='flag_db' and value=%r" , case),
_fast=True)
dbh.execute("drop database if exists `%s`" ,case)
except DB.DBError,e:
pass
def form(self, query, result):
result.textfield("Inode ID", 'inode_id')
dbh = DB.DBO(query['case'])
try:
result.selector(
"Table Name",
'table_name',
DB.expand(
'select name as `key`,name as value from sqlite where inode_id=%r',
query['inode_id']),
case=query['case'])
except KeyError, e:
pass
def add_inode(self, fd, offset):
""" This is called to allow the Carver to add VFS inodes.
Returns the new inode_id.
"""
## Calculate the length of the new file
length = self.get_length(fd,offset)
new_inode = "%s|o%s:%s" % (self.fd.inode, offset, length)
path, inode, inode_id = self.fsfd.lookup(inode_id = self.fd.inode_id)
name = DB.expand("%s/%s", (path, self.make_filename(offset)))
## By default we just add a VFS Inode for it.
new_inode_id = self.fsfd.VFSCreate(None,
new_inode,
name,
size = length,
)
pyflaglog.log(pyflaglog.DEBUG, DB.expand("Added Carved inode %s (id %s) as %s",
(new_inode, new_inode_id,
name)))
self.add_type_info(new_inode_id)
def list_hits(case, inode_id, word, start=None, end=None):
""" Returns a generator of hits of the word within the inode
between offset start and end (these are inode offsets."""
dbh = DB.DBO(case)
pdbh = DB.DBO()
pdbh.execute("select id from dictionary where word = %r limit 1" , word)
row = pdbh.fetch()
if not row:
raise RuntimeError("Word queried (%s) is not in the dictionary???" % word)
ranges = ''
if start!=None:
ranges += DB.expand("and offset >= %r",(start,))
if end!=None:
ranges += DB.expand("and offset < %r", (end,))
id = row['id']
dbh.execute("select offset,length from LogicalIndexOffsets where "
"inode_id = %r and word_id = %r %s order by offset", inode_id, id, ranges)
return dbh
def render_page(self, page_name, page_number, elements, row_generator):
## We must have an InodeID in there
inode = None
for e in elements:
if isinstance(e, InodeIDType):
inode = e
break
## Should not happen
if not inode:
raise RuntimeError("You must have Inodes in your table")
start_value = None
end_value = None
self.row_count = 0
tmp = []
result = '<table>'
for row in row_generator:
value = row[inode.name]
cell_ui = value
tmp.append(cell_ui.__str__())
end_value = row[elements[self.order].name]
if not start_value:
start_value = end_value
if len(tmp) >= 5:
result += self.render_row(tmp)
tmp = []
self.row_count +=1
if self.row_count > self.pagesize:
break
result += self.render_row( tmp)
dbh = DB.DBO(self.case)
dbh.delete("reporting", where=DB.expand("page_name = %r", page_name))
dbh.insert("reporting",
start_value = start_value,
end_value = end_value,
page_name = page_name,
description = self.description)
return self.header % {'toolbar': self.navigation_buttons(page_number),
'title': self.description or "PyFlag Gallery Export",
} + \
result + """</tbody></table>
def render_page(self, page_name, page_number, elements, row_generator):
## We must have an InodeID in there
inode = None
for e in elements:
if isinstance(e, AFF4URN):
inode = e
break
## Should not happen
if not inode:
raise RuntimeError("You must have Inodes in your table")
start_value = None
end_value = None
self.row_count = 0
tmp = []
result = '<table>'
for row in row_generator:
value = row[inode.name]
cell_ui = value
tmp.append(cell_ui.__str__())
end_value = row[elements[self.order].name]
if not start_value:
start_value = end_value
if len(tmp) >= 5:
result += self.render_row(tmp)
tmp = []
self.row_count +=1
if self.row_count > self.pagesize:
break
result += self.render_row( tmp)
dbh = DB.DBO(self.case)
dbh.delete("reporting", where=DB.expand("page_name = %r", page_name))
dbh.insert("reporting",
start_value = start_value,
end_value = end_value,
page_name = page_name,
description = self.description)
return self.header % {'toolbar': self.navigation_buttons(page_number),
'title': self.description or "PyFlag Gallery Export",
} + \
result + """</tbody></table>
def mass_annotate(new_query, result):
if not new_query.has_key("annotate_inode"):
result.heading("Error")
result.para("You must select some inodes to annotate by checking their checkboxes")
return
if new_query.has_key("__submit__"):
tag = new_query.get("annotate_text","Tag")
category = new_query.get("annotate_category", "Note")
if new_query.has_key("new_annotate_category"):
category = new_query['new_annotate_category']
dbh = DB.DBO(case)
## First delete old annotations if present
for inode_id in new_query.getarray("annotate_inode"):
dbh.delete("annotate", where=DB.expand("inode_id=%r", inode_id))
## Now insert new ones
for inode_id in new_query.getarray("annotate_inode"):
dbh.insert("annotate",
inode_id = inode_id,
note = tag,
category = category,
)
del query["annotate_inode"]
query.set("annotate_text", tag)
query.set("annotate_category", category)
result.refresh(0, query, pane="parent_pane")
fsfd = FileSystem.DBFS(case)
new_query.default('annotate_text',query.get("annotate_text","Tag"))
new_query.default("annotate_category", query.get("annotate_category","Note"))
result.decoration='naked'
result.heading("Set Annotation Text")
result.start_form(new_query, pane='self')
result.textarea("Annotation Text",'annotate_text')
TableActions.selector_display(None, "Category", "annotate_category",
result=result, table = 'annotate',
field='category', case=case,
default='Note')
result.end_table()
result.end_form()
result.para("The following inodes will be annotated:")
result.start_table(**{'class':'GeneralTable'})
for inode_id in new_query.getarray("annotate_inode"):
path, inode, inode_id = fsfd.lookup(inode_id = inode_id)
result.row(inode, path)
def mass_annotate(new_query, result):
if not new_query.has_key("annotate_inode"):
result.heading("Error")
result.para("You must select some inodes to annotate by checking their checkboxes")
return
if new_query.has_key("__submit__"):
tag = new_query.get("annotate_text","Tag")
category = new_query.get("annotate_category", "Note")
if new_query.has_key("new_annotate_category"):
category = new_query['new_annotate_category']
dbh = DB.DBO(case)
## First delete old annotations if present
for inode_id in new_query.getarray("annotate_inode"):
dbh.delete("annotate", where=DB.expand("inode_id=%r", inode_id))
## Now insert new ones
for inode_id in new_query.getarray("annotate_inode"):
dbh.insert("annotate",
inode_id = inode_id,
note = tag,
category = category,
)
del query["annotate_inode"]
query.set("annotate_text", tag)
query.set("annotate_category", category)
result.refresh(0, query, pane="parent_pane")
fsfd = FileSystem.DBFS(case)
new_query.default('annotate_text',query.get("annotate_text","Tag"))
new_query.default("annotate_category", query.get("annotate_category","Note"))
result.decoration='naked'
result.heading("Set Annotation Text")
result.start_form(new_query, pane='self')
result.textarea("Annotation Text",'annotate_text')
TableActions.selector_display(None, "Category", "annotate_category",
result=result, table = 'annotate',
field='category', case=case,
default='Note')
result.end_table()
result.end_form()
result.para("The following inodes will be annotated:")
result.start_table(**{'class':'GeneralTable'})
for inode_id in new_query.getarray("annotate_inode"):
path, inode, inode_id = fsfd.lookup(inode_id = inode_id)
result.row(inode, path)
def insert_into_table(mode, root, name):
rel_root = FlagFramework.normpath(DB.expand("%s/%s/", (mount_point, root[len(path) :])))
try:
s = os.lstat(os.path.join(root, name))
except OSError:
pyflaglog.log(
pyflaglog.WARNING, DB.expand("Unable to stat %s - mount the directory with the uid option", root)
)
return
inode = DB.expand("I%s|M%s", (iosource_name, s.st_ino))
dbh_inode.insert(
"inode",
inode=inode,
uid=s.st_uid,
gid=s.st_gid,
_mtime="from_unixtime(%s)" % s.st_mtime,
_atime="from_unixtime(%s)" % s.st_atime,
_ctime="from_unixtime(%s)" % s.st_ctime,
status="alloc",
mode=str(oct(s.st_mode)),
size=s.st_size,
_fast=True,
)
inode_id = dbh_inode.autoincrement()
dbh_file.mass_insert(inode_id=inode_id, inode=inode, mode=mode, status="alloc", path=rel_root, name=name)
## If needed schedule inode for scanning:
if scanners and mode == "r/r":
pdbh.mass_insert(command="Scan", arg1=self.case, arg2=inode, arg3=scanner_string, cookie=cookie)
## Fixme - handle symlinks
try:
link = os.readlink(DB.expand("%s/%s", (root, name)).encode("utf8"))
except OSError:
link = ""
def external_process(self, fd):
pyflaglog.log(pyflaglog.DEBUG, "Opening %s for SQLite scanning",
self.inode)
filename = CacheManager.MANAGER.provide_cache_filename(
self.case, self.fd.inode)
db = sqlite.connect(filename)
ldbh = db.cursor()
ldbh2 = db.cursor()
ldbh.execute("select * From sqlite_master")
dbh = DB.DBO(self.case)
dbh.cursor.ignore_warnings = True
for row in ldbh:
if row[0] == 'table':
dbh.insert('sqlite',
name=row[1],
inode_id=self.fd.inode_id,
definition=row[4])
try:
case_table = build_case_table("sqlite", row[4])
except RuntimeError, e:
pyflaglog.log(pyflaglog.WARNING, e)
continue
## Create our copy of this table (if its not
## already there
try:
case_table.create(dbh)
except Exception, e:
pass
## Insert all the data into our copy of the table
ldbh2.execute("select * from %s" % row[1])
dbh.mass_insert_start(case_table.name)
for row in ldbh2:
args = {'inode_id': self.fd.inode_id}
for i in range(len(row)):
if row[i] != None:
if case_table.sql_filters[i]:
args["_" + case_table.
column_names[i]] = DB.expand(
case_table.sql_filters[i], row[i])
else:
args[case_table.column_names[i]] = row[i]
dbh.mass_insert(**args)
dbh.mass_insert_commit()
def log(level,message, *args):
""" Prints the message out only if the configured verbosity is higher than the message's level."""
if config.LOG_LEVEL >= level:
try:
log_fd = open(config.LOGFILE,"ab")
except Exception,e:
log_fd = sys.stderr
import pyflag.DB as DB
try:
string = DB.expand("%s(%s): %s" % (os.getpid(),lookup[level],message), args)
except Exception,e:
log_fd.write("%s\n" % e)
string = message
def set_filter(self, query,result):
## If we get here - the word is ok
filter_expression = DB.expand("Word = '%s'", (query['indexing_word']))
try:
query.set(self.filter, "%s and %s" % (query[self.filter],
filter_expression))
except KeyError:
query.set(self.filter, filter_expression)
## Check that we can filter on the word as all:
element = WordColumn(case = self.case, filter=self.filter,
where = self.where)
element.ui = result
element.elements = self.elements
element.operator_hit("Word", "=", query['indexing_word'])
def form(self,query,result):
try:
## Draw the form for each scan group:
result.text(DB.expand("Scanning Inode %s", (query['inode'])))
groups = []
for cls in ScannerUtils.scan_groups_gen():
try:
drawer = cls.Drawer()
if drawer.group in groups: continue
groups.append(drawer.group)
drawer.form(query,result)
except RuntimeError:
pass
result.checkbox('Click here when finished','final','ok')
except KeyError:
return result
def external_process(self, fd):
pyflaglog.log(pyflaglog.DEBUG, "Opening %s for SQLite scanning", self.inode)
filename = CacheManager.MANAGER.provide_cache_filename(self.case, self.fd.inode)
db = sqlite.connect(filename)
ldbh = db.cursor()
ldbh2 = db.cursor()
ldbh.execute("select * From sqlite_master")
dbh = DB.DBO(self.case)
dbh.cursor.ignore_warnings = True
for row in ldbh:
if row[0]=='table':
dbh.insert('sqlite',
name = row[1],
inode_id = self.fd.inode_id,
definition = row[4])
try:
case_table = build_case_table("sqlite",
row[4])
except RuntimeError,e:
pyflaglog.log(pyflaglog.WARNING, e)
continue
## Create our copy of this table (if its not
## already there
try:
case_table.create(dbh)
except Exception,e:
pass
## Insert all the data into our copy of the table
ldbh2.execute("select * from %s" % row[1])
dbh.mass_insert_start(case_table.name)
for row in ldbh2:
args = {'inode_id': self.fd.inode_id}
for i in range(len(row)):
if row[i]!=None:
if case_table.sql_filters[i]:
args["_"+case_table.column_names[i]] = DB.expand(
case_table.sql_filters[i],row[i])
else:
args[case_table.column_names[i]] = row[i]
dbh.mass_insert(**args)
dbh.mass_insert_commit()
def display(self,query,result):
## This indexing will be done in process (i.e. not
## distributable) because its exactly one job:
task = Index()
task.run(query['case'], query['inode_id'], 2**30 + int(query['word_id']))
case = query['case']
result.table(
elements = [ InodeIDType(case=case),
OffsetType(case=case),
DataPreview(name='Preview', case=case),
],
table = 'LogicalIndexOffsets',
where = DB.expand('inode.inode_id = %r and LogicalIndexOffsets.word_id= %r',
(query['inode_id'],query['word_id'])),
case =case,
)
def display(self, query, result):
## The dictionary is site wide and lives in the FlagDB
dbh = DB.DBO()
## class_override the class variable:
try:
if len(query['class_override']) > 3:
del query['class']
query['class'] = query['class_override']
del query['class_override']
except KeyError:
pass
status = ''
## Do we need to add a new entry:
try:
if len(query['word']) < 3:
raise DB.DBError(
"Word is too short to index, minimum of 3 letter words")
if query['action'] == 'insert':
try:
if len(query['class']) < 3:
raise DB.DBError(
"Class name is too short, minimum of 3 letter words are used as class names"
)
except KeyError:
status = "Classification missing or too short"
raise
Indexing.insert_dictionary_word(query['word'], query['type'],
query['class'])
status = "Added word %s to dictionary" % query['word']
elif query['action'] == 'delete':
dbh.delete("dictionary",
where=DB.expand("word=%b and type=%r",
(query['word'], query['type'])))
status = "Deleted word %s from dictionary" % query['word']
except KeyError, e:
pass
def pane_cb(path,tmp):
fsfd = FileSystem.DBFS( query["case"])
if not fsfd.isdir(path):
path=posixpath.dirname(path)
new_query = make_new_query(query, path + '/')
tmp.table(
elements = [ InodeIDType(case=query['case']),
FilenameType(basename=True, case=query['case'],
link = new_query,
link_pane = 'parent'),
IntegerType('File Size','size'),
],
table='inode',
where=DB.expand("file.path=%r and file.mode!='d/d'", (path+'/')),
case=query['case'],
pagesize=10,
filter="filter2",
)
def render_pane(self, branch, query, result):
## We may only draw on the pane that belongs to us:
if branch[0] != self.name:
return
if len(branch)==1:
result.heading("Show file types")
result.text("This statistic allows different file types to be examined")
else:
t = branch[1].replace("__",'/')
result.table(
elements = [ AFF4URN(case = self.case),
FilenameType(case = self.case, link_pane='main'),
IntegerType('Size','size', table='vfs'),
TimestampType('Timestamp','mtime', table='vfs'),
StringType('Mime', 'mime', table='type')],
table = 'type',
where = DB.expand('type.type=%r ', t),
case = self.case,
)
def store_key(nk_key, path):
if not nk_key: return
key_name = nk_key['key_name'].__unicode__()
regi_handle.mass_insert(dirname=path, basename=key_name)
new_path = DB.expand("%s/%s/", (path, nk_key['key_name']))
new_path = FlagFramework.normpath(new_path)
## Store all the values:
for v in nk_key.values():
reg_handle.mass_insert(inode_id=inode_id,
path=new_path,
offset=v['data']['offs_data'],
_modified="from_unixtime(%d)" %
nk_key['WriteTS'].get_value(),
type=v['data']['val_type'],
reg_key=v['keyname'],
value=v['data'])
for k in nk_key.keys():
store_key(k, new_path)
