def parse_line(line):
try:
toret = Struct()
# Split the line, the format being 'HH:MM:SS.nnnnn syscall(args...) = RETVALUE ERRORCODE (Error String)'
m = re.search(r'^([0-9:\.]+) ([^(]+)(\(.*\)) += ([xa-f\-0-9]+|\?) ?(E[^ ]* \([^\(\)]*\)|\([^\(\)]*\))?$', line)
# Convert time into a numerical value
time = line[m.start(1) : m.end(1)]
toret.str_time = time
time = time.split(':')
toret.time = int(time[0]) * 60.0 * 60.0 + int(time[1]) * 60.0 + float(time[2])
toret.syscall = line[m.start(2) : m.end(2)]
toret.ret = line[m.start(4) : m.end(4)]
return_explanation = line[m.start(5) : m.end(5)]
if return_explanation.startswith("E"):
toret.err = return_explanation
else:
toret.return_explanation = return_explanation
# The arguments part looks something like '(20, "hello", "world", 3)'
args = csv.reader([line[m.start(3):m.end(3)]], delimiter=',', quotechar='"').next()
# Now args is ['(20', ' "hello"', ' "world"', ' 3)']
args = [x[1:] for x in args]
args[len(args) - 1] = args[len(args) - 1][:-1]
toret.args = args
return toret
except AttributeError as err:
for innocent_line in ['+++ exited with', ' --- SIG', '<unfinished ...>', ' = ? <unavailable>', 'ptrace(SYSCALL):No such process']:
if line.find(innocent_line) != -1:
return False
print line
raise err
def get_deps(self, ops): last_sync = None for i in range(0, len(ops)): ops[i].hidden_dependencies = set() ops[i].hidden_twojournalfs_stuff = Struct(reverse_fsync_dependencies = set()) if last_sync != None: ops[i].hidden_dependencies.add(last_sync) if ops[i].op in ['sync', 'stdout']: last_sync = i else: assert ops[i].op in ['truncate', 'write', 'delete_dir_entry', 'create_dir_entry'] if ops[i].op == 'sync': for j in range(i - 1, -1, -1): if ops[j].op in ['sync', 'write']: i_final = ops[i].offset + ops[i].count i_initial = ops[i].offset j_final = ops[j].offset + ops[j].count j_initial = ops[j].offset if ops[j].op == 'sync': if not ops[j].inode == ops[i].inode: continue # If j-sync overlaps i-sync if j_initial <= i_initial and j_final >= i_final: break elif ops[j].op == 'truncate': if not ops[j].inode == ops[i].inode: continue assert ops[i].hidden_micro_op.hidden_parsed_line.syscall in ['fsync', 'fdatasync', 'sync'] ops[i].hidden_dependencies.add(j) ops[j].hidden_twojournalfs_stuff.reverse_fsync_dependencies.add(i) elif ops[j].op == 'write': if not ops[j].inode == ops[i].inode: continue # If j_initial is within i's range if j_initial >= i_initial and j_initial <= i_final: if not (j_final >= i_initial and j_final <= i_final): if not 'warned_xxxx1' in globals(): print '----------------------------------------------------------' print 'WARNING: not (j_final >= i_initial and j_final <= i_final)' traceback.print_stack(file = sys.stdout) print '----------------------------------------------------------' globals()['warned_xxxx1'] = 1 ops[i].hidden_dependencies.add(j) ops[j].hidden_twojournalfs_stuff.reverse_fsync_dependencies.add(i) else: if (j_final >= i_initial and j_final <= i_final): if not 'warned_xxxx2' in globals(): print '----------------------------------------------------------' print 'WARNING: (j_final >= i_initial and j_final <= i_final)' traceback.print_stack(file = sys.stdout) print '----------------------------------------------------------' globals()['warned_xxxx2'] = 1 elif ops[j].op in ['create_dir_entry', 'delete_dir_entry']: if not ops[j].parent == ops[i].inode: continue assert ops[i].hidden_micro_op.hidden_parsed_line.syscall in ['fsync', 'sync'] ops[i].hidden_dependencies.add(j) ops[j].hidden_twojournalfs_stuff.reverse_fsync_dependencies.add(i) else: assert ops[j].op == 'stdout'
def save(self, i):
assert self.fs_initialized
self.saved[int(i)] = copy.deepcopy(
Struct(micro_ops=self.micro_ops,
micro_end=self.__micro_end,
disk_end=self.__disk_end,
test_suite=self.test_suite))
def unlink_disk_ops(parent, inode, name, size, hardlinks, entry_type = Struct.TYPE_FILE, atomicity_prefix = ''): toret = [] if hardlinks == 1: toret += trunc_disk_ops(inode, size, 0, atomicity_prefix = atomicity_prefix) if len(toret) > 0: toret[-1].atomicity = atomicity_prefix + 'fully truncated' disk_op = Struct(op = 'delete_dir_entry', parent = parent, entry = name, inode = inode, entry_type = entry_type) toret.append(disk_op) return toret
def link_disk_ops(parent,
inode,
name,
mode=None,
entry_type=Struct.TYPE_FILE):
return [
Struct(op='create_dir_entry',
parent=parent,
entry=name,
inode=inode,
mode=mode,
entry_type=entry_type)
]
def set_fs(self, fs):
all_diskops = []
for micro_op_id in range(0, len(self.micro_ops)):
fs.get_disk_ops(self.micro_ops[micro_op_id])
if micro_op_id == self.__micro_end:
self.__disk_end = len(
self.micro_ops[micro_op_id].hidden_disk_ops) - 1
cnt = 0
for disk_op in self.micro_ops[micro_op_id].hidden_disk_ops:
disk_op.hidden_omitted = False
disk_op.hidden_id = cnt
disk_op.hidden_micro_op = self.micro_ops[micro_op_id]
cnt += 1
all_diskops += self.micro_ops[micro_op_id].hidden_disk_ops
for i in range(0, len(all_diskops)):
if all_diskops[i].op == 'stdout':
all_diskops[i] = Struct(op='write',
inode=-1,
offset=0,
count=1,
hidden_actual_op=all_diskops[i])
self.test_suite = auto_test.ALCTestSuite(all_diskops)
for i in range(0, len(all_diskops)):
if all_diskops[i].op == 'write' and all_diskops[i].inode == -1:
all_diskops[i] = all_diskops[i].hidden_actual_op
fs.get_deps(all_diskops)
dependency_tuples = []
for i in range(0, len(all_diskops)):
for j in sorted(list(all_diskops[i].hidden_dependencies)):
dependency_tuples.append((i, j))
self.test_suite.add_deps_to_ops(dependency_tuples)
self.fs_initialized = True
self.saved = dict()
self.save(0)
def trunc_disk_ops(inode,
initial_size,
final_size,
append_micro_op=None,
atomicity_prefix=''):
toret = []
if initial_size == final_size:
print 'Warning: trunc_disk_ops called for the same initial and final size, ' + str(
initial_size)
return toret
# If we are making the file smaller, follow the same algorithm
# as making the file bigger. But, exchange the initial_size and
# final_size in the beginning, and then reverse the final
# output list.
invert = False
if initial_size > final_size:
t = initial_size
initial_size = final_size
final_size = t
invert = True
if append_micro_op:
assert not invert
assert append_micro_op.inode == inode
assert append_micro_op.offset == initial_size
assert append_micro_op.count == (final_size - initial_size)
splits = self.splits
split_mode = self.split_mode
start = initial_size
remaining = final_size - initial_size
if split_mode == 'count':
per_slice_size = int(math.ceil(float(remaining) / splits))
end = 0
while remaining > 0:
if split_mode == 'aligned':
count = min(splits - (start % splits), remaining)
else:
count = min(per_slice_size, remaining)
end = count + start
if invert:
# Actually truncate. Final operation (if there is no size-splitting) for size-decreasing truncates.
atomicity = atomicity_prefix
if remaining != count:
atomicity += ' semi-truncated (' + str(
splits) + ' ' + split_mode + ' splits)'
disk_op = Struct(op='truncate',
inode=inode,
initial_size=start,
final_size=end,
atomicity=atomicity)
toret.append(disk_op)
if append_micro_op:
# Write zeros
atomicity = atomicity_prefix + 'zero written'
if remaining != count:
atomicity += ' semi-expanded (' + str(
splits) + ' ' + split_mode + ' splits)'
disk_op = Struct(op = 'write', inode = inode, offset = start, dump_offset = 0, count = count, \
dump_file = None, override_data = None, special_write = 'ZEROS', atomicity = atomicity)
toret.append(disk_op)
if not invert:
# Write garbage
atomicity = atomicity_prefix + 'garbage written'
if remaining != count:
atomicity += ' semi-expanded (' + str(
splits) + ' ' + split_mode + ' splits)'
disk_op = Struct(op = 'write', inode = inode, offset = start, dump_offset = 0, count = count, \
dump_file = None, override_data = None, special_write = 'GARBAGE', atomicity = atomicity)
toret.append(disk_op)
if (not invert) and not append_micro_op:
# Write zeros. Final operation (if there is no size-splitting) for size-increasing truncate.
atomicity = atomicity_prefix
if remaining != count:
atomicity += 'semi-expanded (' + str(
splits) + ' ' + split_mode + ' splits)'
disk_op = Struct(op = 'write', inode = inode, offset = start, dump_offset = 0, count = count, \
dump_file = None, override_data = None, special_write = 'ZEROS', atomicity = atomicity)
toret.append(disk_op)
if append_micro_op:
# Write data. Final operation (if there is no size-splitting) for appends.
atomicity = atomicity_prefix
if remaining != count:
atomicity += 'semi-expanded (' + str(
splits) + ' ' + split_mode + ' splits)'
dump_offset = append_micro_op.dump_offset + (
start - append_micro_op.offset)
disk_op = Struct(op = 'write', inode = inode, offset = start, dump_offset = dump_offset, count = count, \
dump_file = append_micro_op.dump_file, special_write = None, atomicity = atomicity)
toret.append(disk_op)
remaining -= count
start = end
assert end == final_size
if invert == True:
toret.reverse()
for disk_op in toret:
t = disk_op.initial_size
disk_op.initial_size = disk_op.final_size
disk_op.final_size = t
return toret
def get_disk_ops(self, line):
splits = self.splits
split_mode = self.split_mode
def trunc_disk_ops(inode,
initial_size,
final_size,
append_micro_op=None,
atomicity_prefix=''):
toret = []
if initial_size == final_size:
print 'Warning: trunc_disk_ops called for the same initial and final size, ' + str(
initial_size)
return toret
# If we are making the file smaller, follow the same algorithm
# as making the file bigger. But, exchange the initial_size and
# final_size in the beginning, and then reverse the final
# output list.
invert = False
if initial_size > final_size:
t = initial_size
initial_size = final_size
final_size = t
invert = True
if append_micro_op:
assert not invert
assert append_micro_op.inode == inode
assert append_micro_op.offset == initial_size
assert append_micro_op.count == (final_size - initial_size)
splits = self.splits
split_mode = self.split_mode
start = initial_size
remaining = final_size - initial_size
if split_mode == 'count':
per_slice_size = int(math.ceil(float(remaining) / splits))
end = 0
while remaining > 0:
if split_mode == 'aligned':
count = min(splits - (start % splits), remaining)
else:
count = min(per_slice_size, remaining)
end = count + start
if invert:
# Actually truncate. Final operation (if there is no size-splitting) for size-decreasing truncates.
atomicity = atomicity_prefix
if remaining != count:
atomicity += ' semi-truncated (' + str(
splits) + ' ' + split_mode + ' splits)'
disk_op = Struct(op='truncate',
inode=inode,
initial_size=start,
final_size=end,
atomicity=atomicity)
toret.append(disk_op)
if append_micro_op:
# Write zeros
atomicity = atomicity_prefix + 'zero written'
if remaining != count:
atomicity += ' semi-expanded (' + str(
splits) + ' ' + split_mode + ' splits)'
disk_op = Struct(op = 'write', inode = inode, offset = start, dump_offset = 0, count = count, \
dump_file = None, override_data = None, special_write = 'ZEROS', atomicity = atomicity)
toret.append(disk_op)
if not invert:
# Write garbage
atomicity = atomicity_prefix + 'garbage written'
if remaining != count:
atomicity += ' semi-expanded (' + str(
splits) + ' ' + split_mode + ' splits)'
disk_op = Struct(op = 'write', inode = inode, offset = start, dump_offset = 0, count = count, \
dump_file = None, override_data = None, special_write = 'GARBAGE', atomicity = atomicity)
toret.append(disk_op)
if (not invert) and not append_micro_op:
# Write zeros. Final operation (if there is no size-splitting) for size-increasing truncate.
atomicity = atomicity_prefix
if remaining != count:
atomicity += 'semi-expanded (' + str(
splits) + ' ' + split_mode + ' splits)'
disk_op = Struct(op = 'write', inode = inode, offset = start, dump_offset = 0, count = count, \
dump_file = None, override_data = None, special_write = 'ZEROS', atomicity = atomicity)
toret.append(disk_op)
if append_micro_op:
# Write data. Final operation (if there is no size-splitting) for appends.
atomicity = atomicity_prefix
if remaining != count:
atomicity += 'semi-expanded (' + str(
splits) + ' ' + split_mode + ' splits)'
dump_offset = append_micro_op.dump_offset + (
start - append_micro_op.offset)
disk_op = Struct(op = 'write', inode = inode, offset = start, dump_offset = dump_offset, count = count, \
dump_file = append_micro_op.dump_file, special_write = None, atomicity = atomicity)
toret.append(disk_op)
remaining -= count
start = end
assert end == final_size
if invert == True:
toret.reverse()
for disk_op in toret:
t = disk_op.initial_size
disk_op.initial_size = disk_op.final_size
disk_op.final_size = t
return toret
def unlink_disk_ops(parent,
inode,
name,
size,
hardlinks,
entry_type=Struct.TYPE_FILE,
atomicity_prefix=''):
toret = []
if hardlinks == 1:
toret += trunc_disk_ops(inode,
size,
0,
atomicity_prefix=atomicity_prefix)
if len(toret) > 0:
toret[-1].atomicity = atomicity_prefix + 'fully truncated'
disk_op = Struct(op='delete_dir_entry',
parent=parent,
entry=name,
inode=inode,
entry_type=entry_type)
toret.append(disk_op)
return toret
def link_disk_ops(parent,
inode,
name,
mode=None,
entry_type=Struct.TYPE_FILE):
return [
Struct(op='create_dir_entry',
parent=parent,
entry=name,
inode=inode,
mode=mode,
entry_type=entry_type)
]
if line.op == 'creat':
line.hidden_disk_ops = link_disk_ops(line.parent, line.inode,
line.name, line.mode)
elif line.op == 'unlink':
line.hidden_disk_ops = unlink_disk_ops(line.parent, line.inode,
line.name, line.size,
line.hardlinks)
elif line.op == 'link':
line.hidden_disk_ops = link_disk_ops(line.dest_parent,
line.source_inode, line.dest)
elif line.op == 'rename':
line.hidden_disk_ops = []
# source: source_inode, dest: dest_inode
if line.dest_hardlinks >= 1:
line.hidden_disk_ops += unlink_disk_ops(
line.dest_parent,
line.dest_inode,
line.dest,
line.dest_size,
line.dest_hardlinks,
atomicity_prefix='destination unlinking partial ')
line.hidden_disk_ops[
-1].atomicity = 'destination unlinked fully, source untouched'
# source: source_inode, dest: None
line.hidden_disk_ops += unlink_disk_ops(
line.source_parent,
line.source_inode,
line.source,
line.source_size,
2,
atomicity_prefix=
'destination unlinked fully, source unlinking partial '
) # Setting hardlinks as 2 so that trunc does not happen
line.hidden_disk_ops[
-1].atomicity = 'destination and source unlinked fully'
# source: None, dest: None
line.hidden_disk_ops += link_disk_ops(line.dest_parent,
line.source_inode, line.dest)
# source: None, dest: source_inode
elif line.op == 'trunc':
line.hidden_disk_ops = trunc_disk_ops(line.inode,
line.initial_size,
line.final_size)
elif line.op == 'append':
line.hidden_disk_ops = trunc_disk_ops(line.inode, line.offset,
line.offset + line.count,
line)
elif line.op == 'write':
assert line.count > 0
line.hidden_disk_ops = []
offset = line.offset
remaining = line.count
if split_mode == 'count':
per_slice_size = int(math.ceil(float(line.count) / splits))
while remaining > 0:
if split_mode == 'aligned':
count = min(splits - (offset % splits), remaining)
else:
count = min(per_slice_size, remaining)
dump_offset = line.dump_offset + (offset - line.offset)
disk_op = Struct(op = 'write', inode = line.inode, offset = offset, dump_offset = dump_offset, \
count = count, dump_file = line.dump_file, override_data = None, special_write = None, \
atomicity = str(splits) + ' ' + split_mode + ' split')
line.hidden_disk_ops.append(disk_op)
remaining -= count
offset += count
elif line.op == 'mkdir':
line.hidden_disk_ops = link_disk_ops(line.parent, line.inode,
line.name, eval(line.mode),
Struct.TYPE_DIR)
elif line.op == 'rmdir':
line.hidden_disk_ops = unlink_disk_ops(line.parent, line.inode,
line.name, 0, 0,
Struct.TYPE_DIR)
elif line.op in ['fsync', 'fdatasync', 'file_sync_range']:
line.hidden_disk_ops = []
if line.op in ['fsync', 'fdatasync']:
offset = 0
count = line.size
else:
offset = line.offset
count = line.count
disk_op = Struct(op='sync',
inode=line.inode,
offset=offset,
count=count)
line.hidden_disk_ops.append(disk_op)
elif line.op in ['sync']:
line.hidden_disk_ops = []
for f in line.hidden_files:
disk_op = Struct(op='sync',
inode=f.inode,
offset=0,
count=f.size)
line.hidden_disk_ops.append(disk_op)
elif line.op == 'stdout':
line.hidden_disk_ops = [Struct(op=line.op, data=line.data)]
else:
assert False
def trunc_disk_ops(inode, initial_size, final_size, append_micro_op = None, atomicity_prefix = ''):
toret = []
if initial_size == final_size:
print 'Warning: trunc_disk_ops called for the same initial and final size, ' + str(initial_size)
return toret
# If we are making the file smaller, follow the same algorithm
# as making the file bigger. But, exchange the initial_size and
# final_size in the beginning, and then reverse the final
# output list.
invert = False
if initial_size > final_size:
t = initial_size
initial_size = final_size
final_size = t
invert = True
if append_micro_op:
assert not invert
assert append_micro_op.inode == inode
assert append_micro_op.offset == initial_size
assert append_micro_op.count == (final_size - initial_size)
splits = self.splits
split_mode = self.split_mode
start = initial_size
remaining = final_size - initial_size
if split_mode == 'count':
per_slice_size = int(math.ceil(float(remaining) / splits))
end = 0
while remaining > 0:
if split_mode == 'aligned':
count = min(splits - (start % splits), remaining)
else:
count = min(per_slice_size, remaining)
end = count + start
if invert:
# Actually truncate. Final operation (if there is no size-splitting) for size-decreasing truncates.
atomicity = atomicity_prefix
if remaining != count:
atomicity += ' semi-truncated (' + str(splits) + ' ' + split_mode + ' splits)'
disk_op = Struct(op = 'truncate', inode = inode, initial_size = start, final_size = end, atomicity = atomicity)
toret.append(disk_op)
if append_micro_op:
# Write zeros
atomicity = atomicity_prefix + 'zero written'
if remaining != count:
atomicity += ' semi-expanded (' + str(splits) + ' ' + split_mode + ' splits)'
disk_op = Struct(op = 'write', inode = inode, offset = start, dump_offset = 0, count = count, \
dump_file = None, override_data = None, special_write = 'ZEROS', atomicity = atomicity)
toret.append(disk_op)
if not invert:
# Write garbage
atomicity = atomicity_prefix + 'garbage written'
if remaining != count:
atomicity += ' semi-expanded (' + str(splits) + ' ' + split_mode + ' splits)'
disk_op = Struct(op = 'write', inode = inode, offset = start, dump_offset = 0, count = count, \
dump_file = None, override_data = None, special_write = 'GARBAGE', atomicity = atomicity)
toret.append(disk_op)
if (not invert) and not append_micro_op:
# Write zeros. Final operation (if there is no size-splitting) for size-increasing truncate.
atomicity = atomicity_prefix
if remaining != count:
atomicity += 'semi-expanded (' + str(splits) + ' ' + split_mode + ' splits)'
disk_op = Struct(op = 'write', inode = inode, offset = start, dump_offset = 0, count = count, \
dump_file = None, override_data = None, special_write = 'ZEROS', atomicity = atomicity)
toret.append(disk_op)
if append_micro_op:
# Write data. Final operation (if there is no size-splitting) for appends.
atomicity = atomicity_prefix
if remaining != count:
atomicity += 'semi-expanded (' + str(splits) + ' ' + split_mode + ' splits)'
dump_offset = append_micro_op.dump_offset + (start - append_micro_op.offset)
disk_op = Struct(op = 'write', inode = inode, offset = start, dump_offset = dump_offset, count = count, \
dump_file = append_micro_op.dump_file, special_write = None, atomicity = atomicity)
toret.append(disk_op)
remaining -= count
start = end
assert end == final_size
if invert == True:
toret.reverse()
for disk_op in toret:
t = disk_op.initial_size
disk_op.initial_size = disk_op.final_size
disk_op.final_size = t
return toret