def generate(target, configfile, database, all_forks=False):
"""Main function for the graph subcommand.
"""
# In here, a file is any file on the filesystem. A binary is a file, that
# gets executed. A process is a system-level task, identified by its pid
# (pids don't get reused in the database).
# What I call program is the couple (process, binary), so forking creates a
# new program (with the same binary) and exec'ing creates a new program as
# well (with the same process)
# Because of this, fork+exec will create an intermediate program that
# doesn't do anything (new process but still old binary). If that program
# doesn't do anything worth showing on the graph, it will be erased, unless
# all_forks is True (--all-forks).
# Reads package ownership from the configuration
if not configfile.is_file():
logging.critical("Configuration file does not exist!\n"
"Did you forget to run 'reprozip trace'?\n"
"If not, you might want to use --dir to specify an "
"alternate location.")
sys.exit(1)
runs, packages, other_files = load_config(configfile, canonical=False)
packages = dict((f.path, pkg) for pkg in packages for f in pkg.files)
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
# This is a bit weird. We need to iterate on all types of events at the
# same time, ordering by timestamp, so we decorate-sort-undecorate
# Decoration adds timestamp (for sorting) and tags by event type, one of
# 'process', 'open' or 'exec'
# Reads processes from the database
process_cursor = conn.cursor()
process_rows = process_cursor.execute(
'''
SELECT id, parent, timestamp
FROM processes
ORDER BY id
''')
processes = {}
all_programs = []
# ... and opened files...
file_cursor = conn.cursor()
file_rows = file_cursor.execute(
'''
SELECT name, timestamp, mode, process
FROM opened_files
ORDER BY id
''')
binaries = set()
files = OrderedSet()
edges = OrderedSet()
# ... as well as executed files.
exec_cursor = conn.cursor()
exec_rows = exec_cursor.execute(
'''
SELECT name, timestamp, process, argv
FROM executed_files
ORDER BY id
''')
# Loop on all event lists
logging.info("Getting all events from database...")
rows = heapq.merge(((r[2], 'process', r) for r in process_rows),
((r[1], 'open', r) for r in file_rows),
((r[1], 'exec', r) for r in exec_rows))
for ts, event_type, data in rows:
if event_type == 'process':
r_id, r_parent, r_timestamp = data
if r_parent is not None:
parent = processes[r_parent]
binary = parent.binary
else:
parent = None
binary = None
p = Process(r_id,
parent,
r_timestamp,
False,
binary,
C_INITIAL if r_parent is None else C_FORK)
processes[r_id] = p
all_programs.append(p)
elif event_type == 'open':
r_name, r_timestamp, r_mode, r_process = data
r_name = PosixPath(r_name)
if r_mode != FILE_WDIR:
process = processes[r_process]
files.add(r_name)
edges.add((process, r_name, r_mode, None))
elif event_type == 'exec':
r_name, r_timestamp, r_process, r_argv = data
r_name = PosixPath(r_name)
process = processes[r_process]
binaries.add(r_name)
# Here we split this process in two "programs", unless the previous
# one hasn't done anything since it was created via fork()
if not all_forks and not process.acted:
process.binary = r_name
process.created = C_FORKEXEC
process.acted = True
else:
process = Process(process.pid,
process,
r_timestamp,
True, # Hides exec only once
r_name,
C_EXEC)
all_programs.append(process)
processes[r_process] = process
argv = tuple(r_argv.split('\0'))
if not argv[-1]:
argv = argv[:-1]
edges.add((process, r_name, None, argv))
process_cursor.close()
file_cursor.close()
conn.close()
# Puts files in packages
logging.info("Organizes packages...")
package_files = {}
other_files = []
for f in files:
pkg = packages.get(f)
if pkg is not None:
package_files.setdefault((pkg.name, pkg.version), []).append(f)
else:
other_files.append(f)
# Writes DOT file
with target.open('w', encoding='utf-8', newline='\n') as fp:
fp.write('digraph G {\n /* programs */\n node [shape=box];\n')
# Programs
logging.info("Writing programs...")
for program in all_programs:
fp.write(' prog%d [label="%s (%d)"];\n' % (
id(program), program.binary or "-", program.pid))
if program.parent is not None:
reason = ''
if program.created == C_FORK:
reason = "fork"
elif program.created == C_EXEC:
reason = "exec"
elif program.created == C_FORKEXEC:
reason = "fork+exec"
fp.write(' prog%d -> prog%d [label="%s"];\n' % (
id(program.parent), id(program), reason))
fp.write('\n node [shape=ellipse];\n\n /* system packages */\n')
# Files from packages
logging.info("Writing packages...")
for i, ((name, version), files) in enumerate(iteritems(package_files)):
fp.write(' subgraph cluster%d {\n label=' % i)
if version:
fp.write('"%s %s";\n' % (escape(name), escape(version)))
else:
fp.write('"%s";\n' % escape(name))
for f in files:
fp.write(' "%s";\n' % escape(unicode_(f)))
fp.write(' }\n')
fp.write('\n /* other files */\n')
# Other files
logging.info("Writing other files...")
for f in other_files:
fp.write(' "%s"\n' % escape(unicode_(f)))
fp.write('\n')
# Edges
logging.info("Connecting edges...")
for prog, f, mode, argv in edges:
if mode is None:
fp.write(' "%s" -> prog%d [color=blue, label="%s"];\n' % (
escape(unicode_(f)),
id(prog),
escape(' '.join(argv))))
elif mode & FILE_WRITE:
fp.write(' prog%d -> "%s" [color=red];\n' % (
id(prog), escape(unicode_(f))))
elif mode & FILE_READ:
fp.write(' "%s" -> prog%d [color=green];\n' % (
escape(unicode_(f)), id(prog)))
fp.write('}\n')
def generate(target,
configfile,
database,
all_forks=False,
graph_format='dot',
level_pkgs='file',
level_processes='thread',
level_other_files='all',
regex_filters=None,
regex_replaces=None,
aggregates=None):
"""Main function for the graph subcommand.
"""
try:
graph_format = {
'dot': FORMAT_DOT,
'DOT': FORMAT_DOT,
'json': FORMAT_JSON,
'JSON': FORMAT_JSON
}[graph_format]
except KeyError:
logging.critical("Unknown output format %r", graph_format)
sys.exit(1)
level_pkgs, level_processes, level_other_files, file_depth = \
parse_levels(level_pkgs, level_processes, level_other_files)
# Reads package ownership from the configuration
if not configfile.is_file():
logging.critical("Configuration file does not exist!\n"
"Did you forget to run 'reprozip trace'?\n"
"If not, you might want to use --dir to specify an "
"alternate location.")
sys.exit(1)
config = load_config(configfile, canonical=False)
inputs_outputs = dict(
(f.path, n) for n, f in iteritems(config.inputs_outputs))
has_thread_flag = config.format_version >= LooseVersion('0.7')
runs, files, edges = read_events(database, all_forks, has_thread_flag)
# Label the runs
if len(runs) != len(config.runs):
logging.warning("Configuration file doesn't list the same number of "
"runs we found in the database!")
else:
for config_run, run in izip(config.runs, runs):
run.name = config_run['id']
# Apply regexes
ignore = [
lambda path, r=re.compile(p): r.search(path) is not None
for p in regex_filters or []
]
replace = [
lambda path, r=re.compile(p): r.sub(repl, path)
for p, repl in regex_replaces or []
]
def filefilter(path):
pathuni = unicode_(path)
if any(f(pathuni) for f in ignore):
logging.debug("IGN %s", pathuni)
return None
if not (replace or aggregates):
return path
for fi in replace:
pathuni_ = fi(pathuni)
if pathuni_ != pathuni:
logging.debug("SUB %s -> %s", pathuni, pathuni_)
pathuni = pathuni_
for prefix in aggregates or []:
if pathuni.startswith(prefix):
logging.debug("AGG %s -> %s", pathuni, prefix)
pathuni = prefix
break
return PosixPath(pathuni)
files_new = set()
for fi in files:
fi = filefilter(fi)
if fi is not None:
files_new.add(fi)
files = files_new
edges_new = OrderedSet()
for prog, fi, mode, argv in edges:
fi = filefilter(fi)
if fi is not None:
edges_new.add((prog, fi, mode, argv))
edges = edges_new
# Puts files in packages
package_map = {}
if level_pkgs == LVL_PKG_IGNORE:
packages = []
other_files = files
else:
logging.info("Organizes packages...")
file2package = dict(
(f.path, pkg) for pkg in config.packages for f in pkg.files)
packages = {}
other_files = []
for fi in files:
pkg = file2package.get(fi)
if pkg is not None:
package = packages.get(pkg.name)
if package is None:
package = Package(pkg.name, pkg.version)
packages[pkg.name] = package
package.files.add(fi)
package_map[fi] = package
else:
other_files.append(fi)
packages = sorted(itervalues(packages), key=lambda pkg: pkg.name)
for i, pkg in enumerate(packages):
pkg.id = i
# Filter other files
if level_other_files == LVL_OTHER_ALL and file_depth is not None:
other_files = set(
PosixPath(*f.components[:file_depth + 1]) for f in other_files)
edges = OrderedSet((prog, f if f in package_map else PosixPath(
*f.components[:file_depth + 1]), mode, argv)
for prog, f, mode, argv in edges)
else:
if level_other_files == LVL_OTHER_IO:
other_files = set(f for f in other_files if f in inputs_outputs)
edges = [(prog, f, mode, argv) for prog, f, mode, argv in edges
if f in package_map or f in other_files]
elif level_other_files == LVL_OTHER_NO:
other_files = set()
edges = [(prog, f, mode, argv) for prog, f, mode, argv in edges
if f in package_map]
args = (target, runs, packages, other_files, package_map, edges,
inputs_outputs, level_pkgs, level_processes, level_other_files)
if graph_format == FORMAT_DOT:
graph_dot(*args)
elif graph_format == FORMAT_JSON:
graph_json(*args)
else:
assert False
def read_events(database, all_forks, has_thread_flag):
# In here, a file is any file on the filesystem. A binary is a file, that
# gets executed. A process is a system-level task, identified by its pid
# (pids don't get reused in the database).
# What I call program is the couple (process, binary), so forking creates a
# new program (with the same binary) and exec'ing creates a new program as
# well (with the same process)
# Because of this, fork+exec will create an intermediate program that
# doesn't do anything (new process but still old binary). If that program
# doesn't do anything worth showing on the graph, it will be erased, unless
# all_forks is True (--all-forks).
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
# This is a bit weird. We need to iterate on all types of events at the
# same time, ordering by timestamp, so we decorate-sort-undecorate
# Decoration adds timestamp (for sorting) and tags by event type, one of
# 'process', 'open' or 'exec'
# Reads processes from the database
process_cursor = conn.cursor()
if has_thread_flag:
sql = '''
SELECT id, parent, timestamp, is_thread
FROM processes
ORDER BY id
'''
else:
sql = '''
SELECT id, parent, timestamp, 0 as is_thread
FROM processes
ORDER BY id
'''
process_rows = process_cursor.execute(sql)
processes = {}
all_programs = []
# ... and opened files...
file_cursor = conn.cursor()
file_rows = file_cursor.execute('''
SELECT name, timestamp, mode, process, is_directory
FROM opened_files
ORDER BY id
''')
binaries = set()
files = set()
edges = OrderedSet()
# ... as well as executed files.
exec_cursor = conn.cursor()
exec_rows = exec_cursor.execute('''
SELECT name, timestamp, process, argv
FROM executed_files
ORDER BY id
''')
# Loop on all event lists
logging.info("Getting all events from database...")
rows = heapq.merge(((r[2], 'process', r) for r in process_rows),
((r[1], 'open', r) for r in file_rows),
((r[1], 'exec', r) for r in exec_rows))
runs = []
run = None
for ts, event_type, data in rows:
if event_type == 'process':
r_id, r_parent, r_timestamp, r_thread = data
logging.debug("Process %d created (parent %r)", r_id, r_parent)
if r_parent is not None:
parent = processes[r_parent]
binary = parent.binary
else:
run = Run(len(runs))
runs.append(run)
parent = None
binary = None
process = Process(r_id, run, parent, r_timestamp, r_thread, False,
binary,
C_INITIAL if r_parent is None else C_FORK)
processes[r_id] = process
all_programs.append(process)
run.processes.append(process)
elif event_type == 'open':
r_name, r_timestamp, r_mode, r_process, r_directory = data
r_name = normalize_path(r_name)
logging.debug("File open: %s, process %d", r_name, r_process)
if not (r_mode & FILE_WDIR or r_directory):
process = processes[r_process]
files.add(r_name)
edges.add((process, r_name, r_mode, None))
elif event_type == 'exec':
r_name, r_timestamp, r_process, r_argv = data
r_name = normalize_path(r_name)
argv = tuple(r_argv.split('\0'))
if not argv[-1]:
argv = argv[:-1]
logging.debug("File exec: %s, process %d", r_name, r_process)
process = processes[r_process]
binaries.add(r_name)
# Here we split this process in two "programs", unless the previous
# one hasn't done anything since it was created via fork()
if not all_forks and not process.acted:
process.binary = r_name
process.created = C_FORKEXEC
process.acted = True
else:
process = Process(
process.pid,
run,
process,
r_timestamp,
False,
True, # Hides exec only once
r_name,
C_EXEC)
all_programs.append(process)
processes[r_process] = process
run.processes.append(process)
files.add(r_name)
edges.add((process, r_name, None, argv))
process_cursor.close()
file_cursor.close()
exec_cursor.close()
conn.close()
return runs, files, edges
def generate(target, directory, all_forks=False):
"""Main function for the graph subcommand.
"""
# In here, a file is any file on the filesystem. A binary is a file, that
# gets executed. A process is a system-level task, identified by its pid
# (pids don't get reused in the database).
# What I call program is the couple (process, binary), so forking creates a
# new program (with the same binary) and exec'ing creates a new program as
# well (with the same process)
# Because of this, fork+exec will create an intermediate program that
# doesn't do anything (new process but still old binary). If that program
# doesn't do anything worth showing on the graph, it will be erased, unless
# all_forks is True (--all-forks).
database = directory / 'trace.sqlite3'
# Reads package ownership from the configuration
configfile = directory / 'config.yml'
if not configfile.is_file():
logging.critical("Configuration file does not exist!\n"
"Did you forget to run 'reprozip trace'?\n"
"If not, you might want to use --dir to specify an "
"alternate location.")
sys.exit(1)
runs, packages, other_files, patterns = load_config(configfile,
canonical=False)
packages = dict((f.path, pkg) for pkg in packages for f in pkg.files)
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
# This is a bit weird. We need to iterate on all types of events at the
# same time, ordering by timestamp, so we decorate-sort-undecorate
# Decoration adds timestamp (for sorting) and tags by event type, one of
# 'process', 'open' or 'exec'
# Reads processes from the database
process_cursor = conn.cursor()
process_rows = process_cursor.execute(
'''
SELECT id, parent, timestamp
FROM processes
ORDER BY id
''')
processes = {}
all_programs = []
# ... and opened files...
file_cursor = conn.cursor()
file_rows = file_cursor.execute(
'''
SELECT name, timestamp, mode, process
FROM opened_files
ORDER BY id
''')
binaries = set()
files = OrderedSet()
edges = OrderedSet()
# ... as well as executed files.
exec_cursor = conn.cursor()
exec_rows = exec_cursor.execute(
'''
SELECT name, timestamp, process, argv
FROM executed_files
ORDER BY id
''')
# Loop on all event lists
logging.info("Getting all events from database...")
rows = heapq.merge(((r[2], 'process', r) for r in process_rows),
((r[1], 'open', r) for r in file_rows),
((r[1], 'exec', r) for r in exec_rows))
for ts, event_type, data in rows:
if event_type == 'process':
r_id, r_parent, r_timestamp = data
if r_parent is not None:
parent = processes[r_parent]
binary = parent.binary
else:
parent = None
binary = None
p = Process(r_id,
parent,
r_timestamp,
False,
binary,
C_INITIAL if r_parent is None else C_FORK)
processes[r_id] = p
all_programs.append(p)
elif event_type == 'open':
r_name, r_timestamp, r_mode, r_process = data
r_name = PosixPath(r_name)
if r_mode != FILE_WDIR:
process = processes[r_process]
files.add(r_name)
edges.add((process, r_name, r_mode, None))
elif event_type == 'exec':
r_name, r_timestamp, r_process, r_argv = data
r_name = PosixPath(r_name)
process = processes[r_process]
binaries.add(r_name)
# Here we split this process in two "programs", unless the previous
# one hasn't done anything since it was created via fork()
if not all_forks and not process.acted:
process.binary = r_name
process.created = C_FORKEXEC
process.acted = True
else:
process = Process(process.pid,
process,
r_timestamp,
True, # Hides exec only once
r_name,
C_EXEC)
all_programs.append(process)
processes[r_process] = process
argv = tuple(r_argv.split('\0'))
if not argv[-1]:
argv = argv[:-1]
edges.add((process, r_name, None, argv))
process_cursor.close()
file_cursor.close()
conn.close()
# Puts files in packages
logging.info("Organizes packages...")
package_files = {}
other_files = []
for f in files:
pkg = packages.get(f)
if pkg is not None:
package_files.setdefault((pkg.name, pkg.version), []).append(f)
else:
other_files.append(f)
# Writes DOT file
with target.open('w', encoding='utf-8', newline='\n') as fp:
fp.write('digraph G {\n /* programs */\n node [shape=box];\n')
# Programs
logging.info("Writing programs...")
for program in all_programs:
fp.write(' prog%d [label="%s (%d)"];\n' % (
id(program), program.binary or "-", program.pid))
if program.parent is not None:
reason = ''
if program.created == C_FORK:
reason = "fork"
elif program.created == C_EXEC:
reason = "exec"
elif program.created == C_FORKEXEC:
reason = "fork+exec"
fp.write(' prog%d -> prog%d [label="%s"];\n' % (
id(program.parent), id(program), reason))
fp.write('\n node [shape=ellipse];\n\n /* system packages */\n')
# Files from packages
logging.info("Writing packages...")
for i, ((name, version), files) in enumerate(iteritems(package_files)):
fp.write(' subgraph cluster%d {\n label=' % i)
if version:
fp.write('"%s %s";\n' % (escape(name), escape(version)))
else:
fp.write('"%s";\n' % escape(name))
for f in files:
fp.write(' "%s";\n' % escape(unicode_(f)))
fp.write(' }\n')
fp.write('\n /* other files */\n')
# Other files
logging.info("Writing other files...")
for f in other_files:
fp.write(' "%s"\n' % escape(unicode_(f)))
fp.write('\n')
# Edges
logging.info("Connecting edges...")
for prog, f, mode, argv in edges:
if mode is None:
fp.write(' "%s" -> prog%d [color=blue, label="%s"];\n' % (
escape(unicode_(f)),
id(prog),
escape(' '.join(argv))))
elif mode & FILE_WRITE:
fp.write(' prog%d -> "%s" [color=red];\n' % (
id(prog), escape(unicode_(f))))
elif mode & FILE_READ:
fp.write(' "%s" -> prog%d [color=green];\n' % (
escape(unicode_(f)), id(prog)))
fp.write('}\n')
def generate(target, configfile, database, all_forks=False, graph_format='dot',
level_pkgs='file', level_processes='thread',
level_other_files='all',
regex_filters=None, regex_replaces=None, aggregates=None):
"""Main function for the graph subcommand.
"""
try:
graph_format = {'dot': FORMAT_DOT, 'DOT': FORMAT_DOT,
'json': FORMAT_JSON, 'JSON': FORMAT_JSON}[graph_format]
except KeyError:
logging.critical("Unknown output format %r", graph_format)
sys.exit(1)
level_pkgs, level_processes, level_other_files, file_depth = \
parse_levels(level_pkgs, level_processes, level_other_files)
# Reads package ownership from the configuration
if not configfile.is_file():
logging.critical("Configuration file does not exist!\n"
"Did you forget to run 'reprozip trace'?\n"
"If not, you might want to use --dir to specify an "
"alternate location.")
sys.exit(1)
config = load_config(configfile, canonical=False)
inputs_outputs = dict((f.path, n)
for n, f in iteritems(config.inputs_outputs))
has_thread_flag = config.format_version >= LooseVersion('0.7')
runs, files, edges = read_events(database, all_forks,
has_thread_flag)
# Label the runs
if len(runs) != len(config.runs):
logging.warning("Configuration file doesn't list the same number of "
"runs we found in the database!")
else:
for config_run, run in izip(config.runs, runs):
run.name = config_run['id']
# Apply regexes
ignore = [lambda path, r=re.compile(p): r.search(path) is not None
for p in regex_filters or []]
replace = [lambda path, r=re.compile(p): r.sub(repl, path)
for p, repl in regex_replaces or []]
def filefilter(path):
pathuni = unicode_(path)
if any(f(pathuni) for f in ignore):
logging.debug("IGN %s", pathuni)
return None
if not (replace or aggregates):
return path
for fi in replace:
pathuni_ = fi(pathuni)
if pathuni_ != pathuni:
logging.debug("SUB %s -> %s", pathuni, pathuni_)
pathuni = pathuni_
for prefix in aggregates or []:
if pathuni.startswith(prefix):
logging.debug("AGG %s -> %s", pathuni, prefix)
pathuni = prefix
break
return PosixPath(pathuni)
files_new = set()
for fi in files:
fi = filefilter(fi)
if fi is not None:
files_new.add(fi)
files = files_new
edges_new = OrderedSet()
for prog, fi, mode, argv in edges:
fi = filefilter(fi)
if fi is not None:
edges_new.add((prog, fi, mode, argv))
edges = edges_new
# Puts files in packages
package_map = {}
if level_pkgs == LVL_PKG_IGNORE:
packages = []
other_files = files
else:
logging.info("Organizes packages...")
file2package = dict((f.path, pkg)
for pkg in config.packages for f in pkg.files)
packages = {}
other_files = []
for fi in files:
pkg = file2package.get(fi)
if pkg is not None:
package = packages.get(pkg.name)
if package is None:
package = Package(pkg.name, pkg.version)
packages[pkg.name] = package
package.files.add(fi)
package_map[fi] = package
else:
other_files.append(fi)
packages = sorted(itervalues(packages), key=lambda pkg: pkg.name)
for i, pkg in enumerate(packages):
pkg.id = i
# Filter other files
if level_other_files == LVL_OTHER_ALL and file_depth is not None:
other_files = set(PosixPath(*f.components[:file_depth + 1])
for f in other_files)
edges = OrderedSet((prog,
f if f in package_map
else PosixPath(*f.components[:file_depth + 1]),
mode,
argv)
for prog, f, mode, argv in edges)
else:
if level_other_files == LVL_OTHER_IO:
other_files = set(f for f in other_files if f in inputs_outputs)
edges = [(prog, f, mode, argv)
for prog, f, mode, argv in edges
if f in package_map or f in other_files]
elif level_other_files == LVL_OTHER_NO:
other_files = set()
edges = [(prog, f, mode, argv)
for prog, f, mode, argv in edges
if f in package_map]
args = (target, runs, packages, other_files, package_map, edges,
inputs_outputs, level_pkgs, level_processes, level_other_files)
if graph_format == FORMAT_DOT:
graph_dot(*args)
elif graph_format == FORMAT_JSON:
graph_json(*args)
else:
assert False
def read_events(database, all_forks, has_thread_flag):
# In here, a file is any file on the filesystem. A binary is a file, that
# gets executed. A process is a system-level task, identified by its pid
# (pids don't get reused in the database).
# What I call program is the couple (process, binary), so forking creates a
# new program (with the same binary) and exec'ing creates a new program as
# well (with the same process)
# Because of this, fork+exec will create an intermediate program that
# doesn't do anything (new process but still old binary). If that program
# doesn't do anything worth showing on the graph, it will be erased, unless
# all_forks is True (--all-forks).
if PY3:
# On PY3, connect() only accepts unicode
conn = sqlite3.connect(str(database))
else:
conn = sqlite3.connect(database.path)
conn.row_factory = sqlite3.Row
# This is a bit weird. We need to iterate on all types of events at the
# same time, ordering by timestamp, so we decorate-sort-undecorate
# Decoration adds timestamp (for sorting) and tags by event type, one of
# 'process', 'open' or 'exec'
# Reads processes from the database
process_cursor = conn.cursor()
if has_thread_flag:
sql = '''
SELECT id, parent, timestamp, is_thread
FROM processes
ORDER BY id
'''
else:
sql = '''
SELECT id, parent, timestamp, 0 as is_thread
FROM processes
ORDER BY id
'''
process_rows = process_cursor.execute(sql)
processes = {}
all_programs = []
# ... and opened files...
file_cursor = conn.cursor()
file_rows = file_cursor.execute(
'''
SELECT name, timestamp, mode, process, is_directory
FROM opened_files
ORDER BY id
''')
binaries = set()
files = set()
edges = OrderedSet()
# ... as well as executed files.
exec_cursor = conn.cursor()
exec_rows = exec_cursor.execute(
'''
SELECT name, timestamp, process, argv
FROM executed_files
ORDER BY id
''')
# Loop on all event lists
logging.info("Getting all events from database...")
rows = heapq.merge(((r[2], 'process', r) for r in process_rows),
((r[1], 'open', r) for r in file_rows),
((r[1], 'exec', r) for r in exec_rows))
runs = []
run = None
for ts, event_type, data in rows:
if event_type == 'process':
r_id, r_parent, r_timestamp, r_thread = data
logging.debug("Process %d created (parent %r)", r_id, r_parent)
if r_parent is not None:
parent = processes[r_parent]
binary = parent.binary
else:
run = Run(len(runs))
runs.append(run)
parent = None
binary = None
process = Process(r_id,
run,
parent,
r_timestamp,
r_thread,
False,
binary,
C_INITIAL if r_parent is None else C_FORK)
processes[r_id] = process
all_programs.append(process)
run.processes.append(process)
elif event_type == 'open':
r_name, r_timestamp, r_mode, r_process, r_directory = data
r_name = normalize_path(r_name)
logging.debug("File open: %s, process %d", r_name, r_process)
if not (r_mode & FILE_WDIR or r_directory):
process = processes[r_process]
files.add(r_name)
edges.add((process, r_name, r_mode, None))
elif event_type == 'exec':
r_name, r_timestamp, r_process, r_argv = data
r_name = normalize_path(r_name)
argv = tuple(r_argv.split('\0'))
if not argv[-1]:
argv = argv[:-1]
logging.debug("File exec: %s, process %d", r_name, r_process)
process = processes[r_process]
binaries.add(r_name)
# Here we split this process in two "programs", unless the previous
# one hasn't done anything since it was created via fork()
if not all_forks and not process.acted:
process.binary = r_name
process.created = C_FORKEXEC
process.acted = True
else:
process = Process(process.pid,
run,
process,
r_timestamp,
False,
True, # Hides exec only once
r_name,
C_EXEC)
all_programs.append(process)
processes[r_process] = process
run.processes.append(process)
files.add(r_name)
edges.add((process, r_name, None, argv))
process_cursor.close()
file_cursor.close()
exec_cursor.close()
conn.close()
return runs, files, edges
