def testIterate_restricted(self):
"""Test getting rorps restricted to certain index
In this case, get assume subdir (subdir3, subdir10) has 50
files in it.
"""
temprp = self.write_metadata_to_temp()
mf = MetadataFile(temprp, 'rb')
start_time = time.time()
i = 0
for rorp in mf.get_objects((b"subdir3", b"subdir10")):
i += 1
print("Reading %s metadata entries took %s seconds." %
(i, time.time() - start_time))
assert i == 51
def test_write(self):
"""Test writing to metadata file, then reading back contents"""
global tempdir
temprp = tempdir.append(
"mirror_metadata.2005-11-03T12:51:06-06:00.snapshot.gz")
if temprp.lstat():
temprp.delete()
self.make_temp()
rootrp = rpath.RPath(Globals.local_connection,
os.path.join(old_test_dir, b"various_file_types"))
# the following 3 lines make sure that we ignore incorrect files
sel = selection.Select(rootrp)
sel.parse_selection_args((), ())
rps = list(sel.set_iter())
assert not temprp.lstat()
write_mf = MetadataFile(temprp, 'w')
for rp in rps:
write_mf.write_object(rp)
write_mf.close()
assert temprp.lstat()
reread_rps = list(MetadataFile(temprp, 'r').get_objects())
assert len(reread_rps) == len(rps), (len(reread_rps), len(rps))
for i in range(len(reread_rps)):
assert reread_rps[i] == rps[i], i
def testSpeed(self):
"""Test testIterator on 10000 files"""
temprp = self.write_metadata_to_temp()
mf = MetadataFile(temprp, 'r')
start_time = time.time()
i = 0
for rorp in mf.get_objects():
i += 1
print("Reading %s metadata entries took %s seconds." %
(i, time.time() - start_time))
start_time = time.time()
blocksize = 32 * 1024
with temprp.open("rb", compress=1) as tempfp:
while 1:
buf = tempfp.read(blocksize)
if not buf:
break
print("Simply decompressing metadata file took %s seconds" %
(time.time() - start_time))
def write_metadata_to_temp(self):
"""If necessary, write metadata of bigdir to file metadata.gz"""
global tempdir
temprp = tempdir.append(
"mirror_metadata.2005-11-03T14:51:06-06:00.snapshot.gz")
if temprp.lstat():
return temprp
self.make_temp()
rootrp = rpath.RPath(Globals.local_connection,
os.path.join(old_test_dir, b"bigdir"))
rpath_iter = selection.Select(rootrp).set_iter()
start_time = time.time()
mf = MetadataFile(temprp, 'w')
for rp in rpath_iter:
mf.write_object(rp)
mf.close()
print("Writing metadata took %s seconds" % (time.time() - start_time))
return temprp
def write_rorp_iter_to_file(rorp_iter, file):
for rorp in rorp_iter:
file.write(MetadataFile._object_to_record(rorp))
def testRORP2Record(self):
"""Test turning RORPs into records and back again"""
for rp in self.get_rpaths():
record = MetadataFile._object_to_record(rp)
new_rorp = RorpExtractor._record_to_object(record)
assert new_rorp == rp, (new_rorp, rp, record)