def tidy(directory: str, collection: dict):
ignoring_file = '.rmdupignore'
wildcard = Wildcard()
for dirpath, _, filenames in os.walk(directory):
wildcard.append(dirpath, ignoring_file)
if wildcard.match(dirpath):
continue
path = PurePath(dirpath)
for filename in filenames:
if filename == ignoring_file or filename == 'Icon\r':
continue
filepath = str(path.joinpath(filename))
# ignore
if wildcard.match(filepath):
continue
f = FileInfo(filepath)
if isinstance(f, FileInfo):
logging.debug(f.uri)
collection.setdefault(f, []).append(str(f))
else:
logging.warning(f)
def _get_dir_info(self, rootdir):
for item in os.listdir(rootdir):
fullname = os.path.join(rootdir, item)
if not item.startswith('.') and not os.path.islink(fullname):
if os.path.isdir(fullname):
dir = DirInfo(fullname, self._recursive)
self._dirs.append(dir)
self._dircount += 1
if self._recursive:
self._filecount += dir.get_filecount()
self._dircount += dir.get_dircount()
self._totalsize += dir.get_totalsize()
self._codelines += dir.get_code_lines()
self._commentlines += dir.get_comment_lines()
self._whitespacelines += dir.get_whitespace_lines()
else:
file = FileInfo(rootdir, item)
self._files.append(file)
self._filecount += 1
self._totalsize += file.get_filesize()
self._codelines += file.get_code_lines()
self._commentlines += file.get_comment_lines()
self._whitespacelines += file.get_whitespace_lines()
def test_get_info():
filename = 'somefile.txt'
original_path = '../{}'.format(filename)
with patch('os.path.abspath') as abspath_mock:
test_abspath = 'some/abs/path'
abspath_mock.return_value = test_abspath
fi = FileInfo(original_path)
assert fi.get_info() == (filename, original_path, test_abspath)
def get_fileinfo_set(root, path, dict):
tmp_set = set()
for t_root, dirs, files in os.walk(path):
for info_file in files + dirs:
tmp_FI = FileInfo(t_root + '/' + info_file, root)
if is_win():
tmp_FI.set_info(dict.get(tmp_FI.rela_path, [0, 0, 644, '']))
tmp_FI.calc_sha1()
tmp_set.add(tmp_FI)
return tmp_set
def test_get_info(abspath_mock, getsize_mock):
filename = 'somefile.txt'
original_path = '../{}'.format(filename)
test_abspath = 'some/abs/path'
abspath_mock.return_value = test_abspath
test_size = 1234
getsize_mock.return_value = test_size
fi = FileInfo(original_path)
assert fi.get_info() == (filename, original_path, test_abspath, test_size)
def test_little_file(monkeypatch):
data = bytes('''
.git/
*/.git/
*.jpg
*.py
a/*/c/
*/mame0184-64bit/
''',
encoding='utf8')
tmp = NamedTemporaryFile()
tmp.write(data)
tmp.flush()
assert hash(FileInfo(tmp.name)) == adler32(data)
tmp.close()
def next_event(self):
# Note that this function will throw an exception if there are no more
# events to process.
event = self.event_list[self.current_event]
self.current_event += 1
# Extract the data about the event
time = event["timestamp"]
# Update the timestamp so that the first event happens at time zero
time -= self.first_event_timestamp
time *= units.ns
# Divide the time by a speedup factor to simulate higher load
time /= self.SPEEDUP_FACTOR
access_type_string = event["syscall"]
if access_type_string == "read":
access_type = EventType.READ
else:
access_type = EventType.WRITE
# The entire path is used as the file name and the path is set to an
# empty string
file_name = event["path"]
file_path = ""
file_size = event["size_bytes"]
# Convert mimetype to file type. We only use the first part of the
# mimetype (the part before the ";").
mimetype = event["mimetype"]
simple_type = mimetype.split(';')[0]
# If a mimetype hasn't been assigned to a file type yet, print a
# message instead of throwing an exception. This allows us to detect
# multiple missing mimetypes on a single run.
try:
file_type = self.types[simple_type]
except KeyError:
print "Unknown mimetype: " + simple_type
file_type = FileType.BINARY
self.error = True
file_info = FileInfo(file_name, file_path, file_type, file_size)
return Event(time, file_info, access_type)
def processor_test():
# Set up some objects for testing
model = ProcessorModel(
13, # idle power
27.1, # active power
3.0 * units.GHz) # speed
processor = Processor(model)
compression_alg = CompressionAlgorithm(
11 * units.MiB, # compression speed
[
0.1, # text compression ratio
1.2, # binary compression ratio
0.9, # image compression ratio
1.0, # video compression ratio
0.8
], # audio compression ratio
17 * units.MiB) # decompression speed
file1 = FileInfo("file1", "/", FileType.TEXT, 1 * units.GiB)
file2 = FileInfo("file2", "/", FileType.BINARY, 40 * units.MiB)
# Tests
passed = True
# Run the processor idle
current_time = 1000
processor.update_time(current_time)
energy = processor.get_energy_usage()
expected_energy = current_time * model.idle_power
if not floateq(energy, expected_energy):
passed = False
print "Failed idle power test"
# Compress a file
results = processor.compress(file1, compression_alg)
energy = processor.get_energy_usage()
expected_size = file1.size * compression_alg.compression_ratio[
file1.file_type]
expected_time = file1.size / compression_alg.compression_speed
expected_energy += expected_time * model.active_power
if not floateq(results.compressed_size, expected_size):
passed = False
print "Failed compression test for size"
if not floateq(results.execution_time, expected_time):
passed = False
print "Failed compression test for time"
if not floateq(energy, expected_energy):
passed = False
print "Failed compression test for energy"
# Decompress a file
time = processor.decompress(file2, compression_alg)
energy = processor.get_energy_usage()
expected_time = file2.size / compression_alg.decompression_speed
expected_energy += expected_time * model.active_power
if not floateq(time, expected_time):
passed = False
print "Failed decompression test for time"
if not floateq(energy, expected_energy):
passed = False
print "Failed decompression test for energy"
if passed:
print "All processor tests passed"
else:
print "Processor tests FAILED"
def disk_test():
timeout = 500
# Set up a disk model, disk, and some files for testing purposes
test_disk = DiskModel(
2.5, # spin up time
30, # spin up energy
10, # spin down energy
3, # idle power
7, # read power
8, # write power
300 * units.MiB, # speed
0.003, # seek time
500 * units.GiB) # capacity
disk = Disk(test_disk, timeout)
file1 = FileInfo("file1", "/", FileType.TEXT, 1 * units.GiB)
file1_compressed_size = 300 * units.MiB
file2 = FileInfo("file2", "/", FileType.BINARY, 40 * units.MiB)
file2_compressed_size = 35 * units.MiB
# Tests
passed = True
# Write when the disk is on
current_time = timeout / 2
disk.update_time(current_time)
time = disk.write(file1, file1_compressed_size)
energy = disk.get_energy_usage()
expected_time = test_disk.seek_time + (file1_compressed_size /
test_disk.speed)
expected_energy = (timeout / 2) * test_disk.idle_power + \
expected_time * test_disk.write_power
if not floateq(time, expected_time):
passed = False
print "Failed write test 1 for time"
if not floateq(energy, expected_energy):
passed = False
print "Failed write test 1 for energy"
# Write when the disk is off
current_time += timeout * 2
disk.update_time(current_time)
time = disk.write(file2, file2_compressed_size)
energy = disk.get_energy_usage()
expected_time = test_disk.spin_up_time + \
test_disk.seek_time + \
(file2_compressed_size / test_disk.speed)
expected_energy += timeout * test_disk.idle_power + \
test_disk.spin_down_energy + \
test_disk.spin_up_energy + \
(expected_time - test_disk.spin_up_time) * \
test_disk.write_power
if not floateq(time, expected_time):
passed = False
print "Failed write test 2 for time"
if not floateq(energy, expected_energy):
passed = False
print "Failed write test 2 for energy"
# Read when the disk is on
current_time += timeout / 2
disk.update_time(current_time)
time = disk.read(file1)
energy = disk.get_energy_usage()
expected_time = test_disk.seek_time + (file1_compressed_size /
test_disk.speed)
expected_energy += (timeout / 2) * test_disk.idle_power + \
expected_time * test_disk.read_power
if not floateq(time, expected_time):
passed = False
print "Failed read test 1 for time"
if not floateq(energy, expected_energy):
passed = False
print "Failed read test 1 for energy"
# Read when the disk is off
current_time += timeout * 5
disk.update_time(current_time)
time = disk.read(file2)
energy = disk.get_energy_usage()
expected_time = test_disk.spin_up_time + \
test_disk.seek_time + \
(file2_compressed_size / test_disk.speed)
expected_energy += timeout * test_disk.idle_power + \
test_disk.spin_down_energy + \
test_disk.spin_up_energy + \
(expected_time - test_disk.spin_up_time) * \
test_disk.read_power
if not floateq(time, expected_time):
passed = False
print "Failed read test 2 for time"
if not floateq(energy, expected_energy):
passed = False
print "Failed read test 2 for energy"
# Try to read something that doesn't exist
file3 = FileInfo("file1", "/newpath/", FileType.TEXT, 1 * units.GiB)
exception_occurred = False
try:
disk.read(file3)
except:
exception_occurred = True
if not exception_occurred:
passed = False
print "Failed read test for non-existant file"
if passed:
print "All disk tests passed"
else:
print "Disk tests FAILED"
def test_init():
filename = 'somefile.txt'
fi = FileInfo(filename)
assert fi.filename == filename
def test_init():
filename = 'somefile.txt'
relative_path = '../{}'.format(filename)
fi = FileInfo(relative_path)
assert fi.filename == filename
from fileinfo import FileInfo
f = FileInfo("test.txt")
f.getsize()
f.firstline()
f.lastline()
f.age()
f.linecount()
f.close()
def disk_array_test():
timeout = 500
disks = 10
# Set up a disk model, disk array, and some files for testing purposes
test_disk = DiskModel(
2.5, # spin up time
30, # spin up energy
10, # spin down energy
3, # idle power
7, # read power
8, # write power
300 * units.MiB, # speed
0.003, # seek time
500 * units.GiB) # capacity
disk_array = DiskArray(0, test_disk, disks, test_disk, timeout)
file1 = FileInfo("file1", "/", FileType.TEXT, 1 * units.GiB)
file1_compressed_size = 300 * units.MiB
file2 = FileInfo("file2", "/", FileType.BINARY, 40 * units.MiB)
file2_compressed_size = 35 * units.MiB
# Tests
passed = True
# Write before the disks turn off
current_time = timeout / 2
disk_array.update_time(current_time)
time = disk_array.write(file1, file1_compressed_size)
energy = disk_array.get_energy_usage()
expected_time = test_disk.seek_time + (file1_compressed_size /
test_disk.speed)
expected_energy = (timeout / 2) * test_disk.idle_power * disks + \
expected_time * test_disk.write_power
if not floateq(time, expected_time):
passed = False
print "Failed write test 1 for time"
if not floateq(energy, expected_energy):
passed = False
print "Failed write test 1 for energy"
# Update the time to when most of the disks turn off
current_time += timeout / 2
disk_array.update_time(current_time)
energy = disk_array.get_energy_usage()
expected_energy += disks * (timeout / 2) * test_disk.idle_power + \
(disks - 1) * test_disk.spin_down_energy
if not floateq(energy, expected_energy):
passed = False
print "Failed test 2"
# Update the time so that the last disk turns off
current_time += timeout / 2
disk_array.update_time(current_time)
energy = disk_array.get_energy_usage()
expected_energy += (timeout / 2) * test_disk.idle_power + \
test_disk.spin_down_energy
if not floateq(energy, expected_energy):
passed = False
print "Failed test 3"
# Turn a disk back on to read
current_time += timeout * 10
disk_array.update_time(current_time)
time = disk_array.read(file1)
energy = disk_array.get_energy_usage()
read_time = file1_compressed_size / test_disk.speed
expected_time = test_disk.spin_up_time + read_time
expected_energy += test_disk.spin_up_energy + \
read_time * test_disk.read_power
if not floateq(time, expected_time):
passed = False
print "Failed read test 4 for time"
if not floateq(energy, expected_energy):
passed = False
print "Failed read test 4 for energy"
# Try to read a file that's not there
exception_occurred = False
try:
disk_array.read(file2)
except:
exception_occurred = True
if not exception_occurred:
passed = False
print "Failed read test for non-existant file"
# Try to allocate some cache disks
exception_occurred = False
try:
disk_array = DiskArray(1, test_disk, disks, test_disk, timeout)
except:
exception_occurred = True
if not exception_occurred:
passed = False
print "Failed read test for non-existant file"
# TODO: add a test where multiple disks are involved (one disk is still on
# from a previous operation while others are turned on for new operations)
if passed:
print "All disk array tests passed"
else:
print "Disk array tests FAILED"
