def list_all_lind_paths(startingdir='/'):
"""
<Purpose>
Returns a list of all files / dirs in the lind fs. Each has an absolute
name. This is similar to 'find startingdir'.
<Arguments>
startingdir: the path to start with.
<Exceptions>
If startingdir is not a dir, this is a SyscallError.
<Side Effects>
None
<Returns>
A list of strings that correspond to absolute names. For example:
['/','/etc/','/etc/passwd']
"""
# BUG: This code may need to be revisited with symlinks...
# this will throw exceptions when given bad data...
lind_test_server.chdir_syscall(startingdir)
return _find_all_paths_recursively(startingdir)
def list_all_lind_paths(startingdir='/'):
"""
<Purpose>
Returns a list of all files / dirs in the lind fs. Each has an absolute
name. This is similar to 'find startingdir'.
<Arguments>
startingdir: the path to start with.
<Exceptions>
If startingdir is not a dir, this is a SyscallError.
<Side Effects>
None
<Returns>
A list of strings that correspond to absolute names. For example:
['/','/etc/','/etc/passwd']
"""
# BUG: This code may need to be revisited with symlinks...
# this will throw exceptions when given bad data...
lind_test_server.chdir_syscall(startingdir)
return _find_all_paths_recursively(startingdir)
def cd_cmd(dir): assert(dir) try: lind_test_server.chdir_syscall(dir) except lind_test_server.SyscallError, e: print "Could not cd. Error: %s" % e
def cp_recursive(source, dest):
new_file_name = dest.split("/")[-1]#get just the new file's name
# check if the source file is a directory
if(not os.path.isdir(source)):
# keep same name if none specified
if not new_file_name:
new_file_name = source.split("/")[-1]
# not a directory, just make sure the path exists, then copy the file into the virtual FS
path_str = check_path_exists(dest)# we should create the path if it doesnt exist (but print a warning)!
copy_file(source, path_str, new_file_name)
else:
# if it IS a dir we need to do a little more work.
path_str = check_path_exists(dest)#add the path up to the directory
lind_test_server.chdir_syscall(path_str)#chdir to the end of that new path
mode = stat.S_IMODE(os.stat(source).st_mode)
#print "in cp " + source + "/" + new_file_name + "..." + str(mode) + "..." + str(S_IRWXU)
try:
lind_test_server.chdir_syscall(new_file_name)#chdir to that new directory
except lind_test_server.SyscallError:
lind_test_server.mkdir_syscall(new_file_name, mode)#make a directory for the new directory
lind_test_server.chdir_syscall(new_file_name)#chdir to that new directory
# then we loop through each of the subfiles of the directory, and recursively call the cp function
# keeping track of both the native file location as well as the virtual file location!
for c in os.listdir(source):
cp_recursive(source + "/" + c, dest + "/" + c)
def check_path_exists(path):
path_list = path.split("/")[0:-1]#get each directory in the path, excluding the new file's name
lind_test_server.chdir_syscall("/")#start at root
path_str = "/"#this string will hold the whole path of the destination file (excluding the filename itself)
# loop through each dir in the path
for p in path_list:
if(p == ""): # ignore the empty string returned from splitting root dir '/'
continue
path_str += (p + "/")#c oncat the dir to the full path string
# try to chdir (if it fails, means the directory doesnt exist)
try:
lind_test_server.chdir_syscall(p)
# if (when) the exception is caught, create a new dir and chdir to it
except lind_test_server.SyscallError:
# this seems to cause an assertion to fail!
#try: # get the mode if the dir exists in the fs
# mode = stat.S_IMODE(os.stat(path_str).st_mode)
#except OSError: # if not set to default 777
# mode = lind_test_server.S_IRWXA
mode = lind_test_server.S_IRWXA
lind_test_server.mkdir_syscall(p, mode)
lind_test_server.chdir_syscall(p)
return path_str
def generate_fs(syscalls, home_env=None):
"""
<Purpose>
This is the only public function of this module. It takes a list of
Syscall objects as an argument and generates a Lind fs based on the
information gathered from these system calls.
<Arguments>
syscalls:
A list of Syscall objects, representing system calls parsed from a trace
file.
home_env:
The HOME environment variable extracted from the traced execve sytem call.
<Exceptions>
None
<Side Effects>
Generates a set of files that make up the Lind file system.
<Returns>
None
"""
# load an initial lind file system.
lind_test_server._blank_fs_init()
# Keep track of all the file paths dealt with and make a note of whether the
# file was added to the lind fs or not. Example: {"filename.txt":True,
# "filename2.txt":False} where True indicates the file was added into the lind
# fs and False means the file was seen but wasn't added to the lind fs. We
# keep track of all the paths met so far for two reasons. Firstly, we want to
# construct the INITIAL file system state, so we only deal with the first
# occurence of each path. For example consider the following sequence of
# traced system calls example:
#
# 11443 open("test.txt", O_RDONLY) = -1 ENOENT (No such file or directory)
# ...
# 11443 open("test.txt", O_RDONLY) = 3
#
# In this example when the "test.txt" path is met for the first time, the file
# does not exist (indicated by: -1 ENOENT). Subsequently, the "test.txt" path
# is met again and in this case the file does exist. In this case we do NOT
# want to include the "test.txt" file in the lind fs, because the initial file
# system state did not include this file, even though the file is eventually
# created.
#
# The second reason we keep track of all the paths met and whether they were
# added in the lind fs is to confirm whether the lind fs was constructed
# successfully.
seen_paths = {}
# a list of system calls that include a filepath in their arguments. We only
# care about these system calls so only the actions representing of these
# system calls will be examined.
syscalls_with_path = ['open', 'creat', 'statfs', 'access', 'stat', 'link',
'unlink', 'chdir', 'rmdir', 'mkdir']
for action in actions:
# the general format of an action is the following:
# (syscall_name, (arguments tuple), (return tuple))
#
# Example successful syscall action:
# ('open_syscall', ('syscalls.txt', ['O_RDONLY', 'O_CREAT'],
# ['S_IWUSR', 'S_IRUSR', 'S_IWGRP', 'S_IRGRP', 'S_IROTH']),
# (3, None))
#
# Example unsuccessful syscall actions:
# ('access_syscall', ('/etc/ld.so.nohwcap', ['F_OK']),
# (-1, 'ENOENT'))
# ('mkdir_syscall', ('syscalls_dir', ['S_IRWXU', 'S_IRWXG',
# 'S_IXOTH', 'S_IROTH']), (-1, 'EEXIST'))
syscall_name, syscall_parameters, syscall_result = action
if DEBUG:
print "Action:", action
# each syscall name should end with _syscall
if not syscall_name.endswith("_syscall"):
raise Exception("Unexpected name of system call '" +
syscall_name + "'")
# remove the _syscall part from the syscall name
syscall_name = syscall_name[:syscall_name.find("_syscall")]
if syscall_name in syscalls_with_path:
# TODO: I should consider O_CREAT O_EXECL and O_TRUNC flags.
# check if the system call is open and whether it includes the O_CREAT
# flag. If it does, set the o_creat flag to True, otherwise set it to
# False. The o_creat flag will be used when trying to add a file, to
# indicate how to handle the case the actual file is missing.
# Specifically, when trying to add a file into the lind fs that does not
# exist in the local fs, an exception will be raised, unless the o_creat
# flag is set to True, in which case a warning will be printed instead.
o_creat = False
if syscall_name == "open":
open_flags = action[1][1]
if "O_CREAT" in open_flags:
o_creat = True
# Similarly, if the system call is creat, set the o_creat flag to True
if syscall_name == "creat":
o_creat = True
# get the file path from the action
path = action[1][0]
# We want the initial file system state. So deal only with the earliest
# syscall pertaining to a path.
if path not in seen_paths:
# if the syscall was successful, copy file/dir into the lind fs.
if syscall_result != (-1, 'ENOENT'):
path, path_added = _copy_path_into_lind(path, home_path, o_creat)
# remember this path was seen and whether it was added to the lind fs.
seen_paths[path] = path_added
else:
# remember this path was seen but not added to the lind fs.
seen_paths[path] = False
# the link syscall contains a second path.
if syscall_name == 'link':
# ('link_syscall', ('syscalls.txt', 'syscalls.link'), (0, None))
path2 = action[1][1]
if path2 not in seen_paths:
# if we got an exists error, the file must have been there
# already, so we add it in the lind fs.
if syscall_result == (-1, 'EEXIST'):
path2, path_added = _copy_path_into_lind(path2, home_path, o_creat)
# remember this path was seen and whether it was added to the lind fs.
seen_paths[path2] = path_added
else:
# remember this path was seen but not added to the lind fs.
seen_paths[path2] = False
"""
TODO: Can add support for symlink here. Slightly more tricky due to
return part and error messages.
"""
# change the home directory and the lind current directory so that future
# references to relative paths will be handled correctly.
if syscall_name == 'chdir':
home_path = os.path.join(home_path, path)
home_path = os.path.normpath(home_path)
lind_test_server.chdir_syscall(home_path)
if DEBUG:
print
print "Seen Paths"
print seen_paths
# all actions are now read. Let's confirm that lind fs is as expected.
all_lind_paths = list_all_lind_paths()
if DEBUG:
print
print "Lind Paths"
print all_lind_paths
for seen_path in seen_paths:
# convert to lind absolute path.
abs_seen_path = seen_path
if not abs_seen_path.startswith("/"):
abs_seen_path = "/" + abs_seen_path
abs_seen_path = os.path.normpath(abs_seen_path)
# skip the root
if abs_seen_path == "/":
continue
if seen_paths[seen_path]:
# the file should be in lind fs.
if abs_seen_path not in all_lind_paths:
raise Exception("Expected file '" + abs_seen_path +
"' not found in Lind fs")
else:
# file should not be in lind fs.
if abs_seen_path in all_lind_paths:
raise Exception("Unexpected file '" + abs_seen_path + "' in Lind fs")
from lind_fs_constants import *
import lind_test_server
# Let's add a few directories to the system and see if it works...
lind_test_server._blank_fs_init()
lind_test_server.mkdir_syscall('/bar',S_IRWXA)
lind_test_server.mkdir_syscall('/bar/baz',S_IRWXA)
lind_test_server.mkdir_syscall('/bar/baz/yaargh',0)
lind_test_server.access_syscall('bar',F_OK)
# go to bar and look for baz...
lind_test_server.chdir_syscall('bar')
lind_test_server.access_syscall('baz',F_OK)
# go to back up and look bar...
lind_test_server.chdir_syscall('..')
lind_test_server.access_syscall('bar',F_OK)
# go to the yaargh dir
lind_test_server.chdir_syscall('/bar/baz/yaargh')
lind_test_server.access_syscall('../../baz',F_OK)
def generate_fs(actions, trace_path):
# Relative paths found in actions are related to this HOME_PATH. It is initially
# set to an empty string which ultimately translates to the current directory.
home_path = ''
# read the trace and examine the execve syscall to see if the HOME environment
# variable is set to somewhere else. This usually happens when running
# benchmarks. The reason to do this is because actions referring to files
# using relative paths, might refere to these files relative to the HOME
# variable defined in the execve syscall.
fh = open(trace_path, "r")
# the execve syscall is the first action of the trace file
execve_line = fh.readline()
fh.close()
# If the 'HOME' variable is defined in the execve line, the HOME_PATH
# variable will be set to the path of 'HOME'.
if execve_line.find("execve(") != -1:
execve_parts = execve_line.split(", ")
# the parameter of the HOME variable in the execve syscall has this format:
# "HOME=/home/savvas/tests/" including the double quotes.
for part in execve_parts:
if part.startswith("\"HOME="):
part = part.strip("\"")
assert (part.startswith("HOME="))
home_path = part[part.find("HOME=") + 5:]
# load an initial lind file system.
lind_test_server._blank_fs_init()
# Keep track of all the file paths dealt with and make a note of whether the
# file was added to the lind fs or not. Example: {"filename.txt":True,
# "filename2.txt":False} where True indicates the file was added into the lind
# fs and False means the file was seen but wasn't added to the lind fs. We
# keep track of all the paths met so far for two reasons. Firstly, we want to
# construct the INITIAL file system state, so we only deal with the first
# occurence of each path. For example consider the following sequence of
# traced system calls example:
#
# 11443 open("test.txt", O_RDONLY) = -1 ENOENT (No such file or directory)
# ...
# 11443 open("test.txt", O_RDONLY) = 3
#
# In this example when the "test.txt" path is met for the first time, the file
# does not exist (indicated by: -1 ENOENT). Subsequently, the "test.txt" path
# is met again and in this case the file does exist. In this case we do NOT
# want to include the "test.txt" file in the lind fs, because the initial file
# system state did not include this file, even though the file is eventually
# created.
#
# The second reason we keep track of all the paths met and whether they were
# added in the lind fs is to confirm whether the lind fs was constructed
# successfully.
seen_paths = {}
# a list of system calls that include a filepath in their arguments. We only
# care about these system calls so only the actions representing of these
# system calls will be examined.
syscalls_with_path = [
'open', 'creat', 'statfs', 'access', 'stat', 'link', 'unlink', 'chdir',
'rmdir', 'mkdir'
]
for action in actions:
# the general format of an action is the following:
# (syscall_name, (arguments tuple), (return tuple))
#
# Example successful syscall action:
# ('open_syscall', ('syscalls.txt', ['O_RDONLY', 'O_CREAT'],
# ['S_IWUSR', 'S_IRUSR', 'S_IWGRP', 'S_IRGRP', 'S_IROTH']),
# (3, None))
#
# Example unsuccessful syscall actions:
# ('access_syscall', ('/etc/ld.so.nohwcap', ['F_OK']),
# (-1, 'ENOENT'))
# ('mkdir_syscall', ('syscalls_dir', ['S_IRWXU', 'S_IRWXG',
# 'S_IXOTH', 'S_IROTH']), (-1, 'EEXIST'))
syscall_name, syscall_parameters, syscall_result = action
if DEBUG:
print "Action:", action
# each syscall name should end with _syscall
if not syscall_name.endswith("_syscall"):
raise Exception("Unexpected name of system call '" + syscall_name +
"'")
# remove the _syscall part from the syscall name
syscall_name = syscall_name[:syscall_name.find("_syscall")]
if syscall_name in syscalls_with_path:
# TODO: I should consider O_CREAT O_EXECL and O_TRUNC flags.
# check if the system call is open and whether it includes the O_CREAT
# flag. If it does, set the o_creat flag to True, otherwise set it to
# False. The o_creat flag will be used when trying to add a file, to
# indicate how to handle the case the actual file is missing.
# Specifically, when trying to add a file into the lind fs that does not
# exist in the local fs, an exception will be raised, unless the o_creat
# flag is set to True, in which case a warning will be printed instead.
o_creat = False
if syscall_name == "open":
open_flags = action[1][1]
if "O_CREAT" in open_flags:
o_creat = True
# Similarly, if the system call is creat, set the o_creat flag to True
if syscall_name == "creat":
o_creat = True
# get the file path from the action
path = action[1][0]
# We want the initial file system state. So deal only with the earliest
# syscall pertaining to a path.
if path not in seen_paths:
# if the syscall was successful, copy file/dir into the lind fs.
if syscall_result != (-1, 'ENOENT'):
path, path_added = _copy_path_into_lind(
path, home_path, o_creat)
# remember this path was seen and whether it was added to the lind fs.
seen_paths[path] = path_added
else:
# remember this path was seen but not added to the lind fs.
seen_paths[path] = False
# the link syscall contains a second path.
if syscall_name == 'link':
# ('link_syscall', ('syscalls.txt', 'syscalls.link'), (0, None))
path2 = action[1][1]
if path2 not in seen_paths:
# if we got an exists error, the file must have been there
# already, so we add it in the lind fs.
if syscall_result == (-1, 'EEXIST'):
path2, path_added = _copy_path_into_lind(
path2, home_path, o_creat)
# remember this path was seen and whether it was added to the lind fs.
seen_paths[path2] = path_added
else:
# remember this path was seen but not added to the lind fs.
seen_paths[path2] = False
"""
TODO: Can add support for symlink here. Slightly more tricky due to
return part and error messages.
"""
# change the home directory and the lind current directory so that future
# references to relative paths will be handled correctly.
if syscall_name == 'chdir':
home_path = os.path.join(home_path, path)
home_path = os.path.normpath(home_path)
lind_test_server.chdir_syscall(home_path)
if DEBUG:
print
print "Seen Paths"
print seen_paths
# all actions are now read. Let's confirm that lind fs is as expected.
all_lind_paths = list_all_lind_paths()
if DEBUG:
print
print "Lind Paths"
print all_lind_paths
for seen_path in seen_paths:
# convert to lind absolute path.
abs_seen_path = seen_path
if not abs_seen_path.startswith("/"):
abs_seen_path = "/" + abs_seen_path
abs_seen_path = os.path.normpath(abs_seen_path)
# skip the root
if abs_seen_path == "/":
continue
if seen_paths[seen_path]:
# the file should be in lind fs.
if abs_seen_path not in all_lind_paths:
raise Exception("Expected file '" + abs_seen_path +
"' not found in Lind fs")
else:
# file should not be in lind fs.
if abs_seen_path in all_lind_paths:
raise Exception("Unexpected file '" + abs_seen_path +
"' in Lind fs")