def tidy(directory: str, collection: dict): ignoring_file = '.rmdupignore' wildcard = Wildcard() for dirpath, _, filenames in os.walk(directory): wildcard.append(dirpath, ignoring_file) if wildcard.match(dirpath): continue path = PurePath(dirpath) for filename in filenames: if filename == ignoring_file or filename == 'Icon\r': continue filepath = str(path.joinpath(filename)) # ignore if wildcard.match(filepath): continue f = FileInfo(filepath) if isinstance(f, FileInfo): logging.debug(f.uri) collection.setdefault(f, []).append(str(f)) else: logging.warning(f)
def _get_dir_info(self, rootdir): for item in os.listdir(rootdir): fullname = os.path.join(rootdir, item) if not item.startswith('.') and not os.path.islink(fullname): if os.path.isdir(fullname): dir = DirInfo(fullname, self._recursive) self._dirs.append(dir) self._dircount += 1 if self._recursive: self._filecount += dir.get_filecount() self._dircount += dir.get_dircount() self._totalsize += dir.get_totalsize() self._codelines += dir.get_code_lines() self._commentlines += dir.get_comment_lines() self._whitespacelines += dir.get_whitespace_lines() else: file = FileInfo(rootdir, item) self._files.append(file) self._filecount += 1 self._totalsize += file.get_filesize() self._codelines += file.get_code_lines() self._commentlines += file.get_comment_lines() self._whitespacelines += file.get_whitespace_lines()
def test_get_info(): filename = 'somefile.txt' original_path = '../{}'.format(filename) with patch('os.path.abspath') as abspath_mock: test_abspath = 'some/abs/path' abspath_mock.return_value = test_abspath fi = FileInfo(original_path) assert fi.get_info() == (filename, original_path, test_abspath)
def get_fileinfo_set(root, path, dict): tmp_set = set() for t_root, dirs, files in os.walk(path): for info_file in files + dirs: tmp_FI = FileInfo(t_root + '/' + info_file, root) if is_win(): tmp_FI.set_info(dict.get(tmp_FI.rela_path, [0, 0, 644, ''])) tmp_FI.calc_sha1() tmp_set.add(tmp_FI) return tmp_set
def test_get_info(abspath_mock, getsize_mock): filename = 'somefile.txt' original_path = '../{}'.format(filename) test_abspath = 'some/abs/path' abspath_mock.return_value = test_abspath test_size = 1234 getsize_mock.return_value = test_size fi = FileInfo(original_path) assert fi.get_info() == (filename, original_path, test_abspath, test_size)
def test_little_file(monkeypatch): data = bytes(''' .git/ */.git/ *.jpg *.py a/*/c/ */mame0184-64bit/ ''', encoding='utf8') tmp = NamedTemporaryFile() tmp.write(data) tmp.flush() assert hash(FileInfo(tmp.name)) == adler32(data) tmp.close()
def next_event(self): # Note that this function will throw an exception if there are no more # events to process. event = self.event_list[self.current_event] self.current_event += 1 # Extract the data about the event time = event["timestamp"] # Update the timestamp so that the first event happens at time zero time -= self.first_event_timestamp time *= units.ns # Divide the time by a speedup factor to simulate higher load time /= self.SPEEDUP_FACTOR access_type_string = event["syscall"] if access_type_string == "read": access_type = EventType.READ else: access_type = EventType.WRITE # The entire path is used as the file name and the path is set to an # empty string file_name = event["path"] file_path = "" file_size = event["size_bytes"] # Convert mimetype to file type. We only use the first part of the # mimetype (the part before the ";"). mimetype = event["mimetype"] simple_type = mimetype.split(';')[0] # If a mimetype hasn't been assigned to a file type yet, print a # message instead of throwing an exception. This allows us to detect # multiple missing mimetypes on a single run. try: file_type = self.types[simple_type] except KeyError: print "Unknown mimetype: " + simple_type file_type = FileType.BINARY self.error = True file_info = FileInfo(file_name, file_path, file_type, file_size) return Event(time, file_info, access_type)
def processor_test(): # Set up some objects for testing model = ProcessorModel( 13, # idle power 27.1, # active power 3.0 * units.GHz) # speed processor = Processor(model) compression_alg = CompressionAlgorithm( 11 * units.MiB, # compression speed [ 0.1, # text compression ratio 1.2, # binary compression ratio 0.9, # image compression ratio 1.0, # video compression ratio 0.8 ], # audio compression ratio 17 * units.MiB) # decompression speed file1 = FileInfo("file1", "/", FileType.TEXT, 1 * units.GiB) file2 = FileInfo("file2", "/", FileType.BINARY, 40 * units.MiB) # Tests passed = True # Run the processor idle current_time = 1000 processor.update_time(current_time) energy = processor.get_energy_usage() expected_energy = current_time * model.idle_power if not floateq(energy, expected_energy): passed = False print "Failed idle power test" # Compress a file results = processor.compress(file1, compression_alg) energy = processor.get_energy_usage() expected_size = file1.size * compression_alg.compression_ratio[ file1.file_type] expected_time = file1.size / compression_alg.compression_speed expected_energy += expected_time * model.active_power if not floateq(results.compressed_size, expected_size): passed = False print "Failed compression test for size" if not floateq(results.execution_time, expected_time): passed = False print "Failed compression test for time" if not floateq(energy, expected_energy): passed = False print "Failed compression test for energy" # Decompress a file time = processor.decompress(file2, compression_alg) energy = processor.get_energy_usage() expected_time = file2.size / compression_alg.decompression_speed expected_energy += expected_time * model.active_power if not floateq(time, expected_time): passed = False print "Failed decompression test for time" if not floateq(energy, expected_energy): passed = False print "Failed decompression test for energy" if passed: print "All processor tests passed" else: print "Processor tests FAILED"
def disk_test(): timeout = 500 # Set up a disk model, disk, and some files for testing purposes test_disk = DiskModel( 2.5, # spin up time 30, # spin up energy 10, # spin down energy 3, # idle power 7, # read power 8, # write power 300 * units.MiB, # speed 0.003, # seek time 500 * units.GiB) # capacity disk = Disk(test_disk, timeout) file1 = FileInfo("file1", "/", FileType.TEXT, 1 * units.GiB) file1_compressed_size = 300 * units.MiB file2 = FileInfo("file2", "/", FileType.BINARY, 40 * units.MiB) file2_compressed_size = 35 * units.MiB # Tests passed = True # Write when the disk is on current_time = timeout / 2 disk.update_time(current_time) time = disk.write(file1, file1_compressed_size) energy = disk.get_energy_usage() expected_time = test_disk.seek_time + (file1_compressed_size / test_disk.speed) expected_energy = (timeout / 2) * test_disk.idle_power + \ expected_time * test_disk.write_power if not floateq(time, expected_time): passed = False print "Failed write test 1 for time" if not floateq(energy, expected_energy): passed = False print "Failed write test 1 for energy" # Write when the disk is off current_time += timeout * 2 disk.update_time(current_time) time = disk.write(file2, file2_compressed_size) energy = disk.get_energy_usage() expected_time = test_disk.spin_up_time + \ test_disk.seek_time + \ (file2_compressed_size / test_disk.speed) expected_energy += timeout * test_disk.idle_power + \ test_disk.spin_down_energy + \ test_disk.spin_up_energy + \ (expected_time - test_disk.spin_up_time) * \ test_disk.write_power if not floateq(time, expected_time): passed = False print "Failed write test 2 for time" if not floateq(energy, expected_energy): passed = False print "Failed write test 2 for energy" # Read when the disk is on current_time += timeout / 2 disk.update_time(current_time) time = disk.read(file1) energy = disk.get_energy_usage() expected_time = test_disk.seek_time + (file1_compressed_size / test_disk.speed) expected_energy += (timeout / 2) * test_disk.idle_power + \ expected_time * test_disk.read_power if not floateq(time, expected_time): passed = False print "Failed read test 1 for time" if not floateq(energy, expected_energy): passed = False print "Failed read test 1 for energy" # Read when the disk is off current_time += timeout * 5 disk.update_time(current_time) time = disk.read(file2) energy = disk.get_energy_usage() expected_time = test_disk.spin_up_time + \ test_disk.seek_time + \ (file2_compressed_size / test_disk.speed) expected_energy += timeout * test_disk.idle_power + \ test_disk.spin_down_energy + \ test_disk.spin_up_energy + \ (expected_time - test_disk.spin_up_time) * \ test_disk.read_power if not floateq(time, expected_time): passed = False print "Failed read test 2 for time" if not floateq(energy, expected_energy): passed = False print "Failed read test 2 for energy" # Try to read something that doesn't exist file3 = FileInfo("file1", "/newpath/", FileType.TEXT, 1 * units.GiB) exception_occurred = False try: disk.read(file3) except: exception_occurred = True if not exception_occurred: passed = False print "Failed read test for non-existant file" if passed: print "All disk tests passed" else: print "Disk tests FAILED"
def test_init(): filename = 'somefile.txt' fi = FileInfo(filename) assert fi.filename == filename
def test_init(): filename = 'somefile.txt' relative_path = '../{}'.format(filename) fi = FileInfo(relative_path) assert fi.filename == filename
from fileinfo import FileInfo f = FileInfo("test.txt") f.getsize() f.firstline() f.lastline() f.age() f.linecount() f.close()
def disk_array_test(): timeout = 500 disks = 10 # Set up a disk model, disk array, and some files for testing purposes test_disk = DiskModel( 2.5, # spin up time 30, # spin up energy 10, # spin down energy 3, # idle power 7, # read power 8, # write power 300 * units.MiB, # speed 0.003, # seek time 500 * units.GiB) # capacity disk_array = DiskArray(0, test_disk, disks, test_disk, timeout) file1 = FileInfo("file1", "/", FileType.TEXT, 1 * units.GiB) file1_compressed_size = 300 * units.MiB file2 = FileInfo("file2", "/", FileType.BINARY, 40 * units.MiB) file2_compressed_size = 35 * units.MiB # Tests passed = True # Write before the disks turn off current_time = timeout / 2 disk_array.update_time(current_time) time = disk_array.write(file1, file1_compressed_size) energy = disk_array.get_energy_usage() expected_time = test_disk.seek_time + (file1_compressed_size / test_disk.speed) expected_energy = (timeout / 2) * test_disk.idle_power * disks + \ expected_time * test_disk.write_power if not floateq(time, expected_time): passed = False print "Failed write test 1 for time" if not floateq(energy, expected_energy): passed = False print "Failed write test 1 for energy" # Update the time to when most of the disks turn off current_time += timeout / 2 disk_array.update_time(current_time) energy = disk_array.get_energy_usage() expected_energy += disks * (timeout / 2) * test_disk.idle_power + \ (disks - 1) * test_disk.spin_down_energy if not floateq(energy, expected_energy): passed = False print "Failed test 2" # Update the time so that the last disk turns off current_time += timeout / 2 disk_array.update_time(current_time) energy = disk_array.get_energy_usage() expected_energy += (timeout / 2) * test_disk.idle_power + \ test_disk.spin_down_energy if not floateq(energy, expected_energy): passed = False print "Failed test 3" # Turn a disk back on to read current_time += timeout * 10 disk_array.update_time(current_time) time = disk_array.read(file1) energy = disk_array.get_energy_usage() read_time = file1_compressed_size / test_disk.speed expected_time = test_disk.spin_up_time + read_time expected_energy += test_disk.spin_up_energy + \ read_time * test_disk.read_power if not floateq(time, expected_time): passed = False print "Failed read test 4 for time" if not floateq(energy, expected_energy): passed = False print "Failed read test 4 for energy" # Try to read a file that's not there exception_occurred = False try: disk_array.read(file2) except: exception_occurred = True if not exception_occurred: passed = False print "Failed read test for non-existant file" # Try to allocate some cache disks exception_occurred = False try: disk_array = DiskArray(1, test_disk, disks, test_disk, timeout) except: exception_occurred = True if not exception_occurred: passed = False print "Failed read test for non-existant file" # TODO: add a test where multiple disks are involved (one disk is still on # from a previous operation while others are turned on for new operations) if passed: print "All disk array tests passed" else: print "Disk array tests FAILED"