def _put_pack(self, sos, commit_amount=1, marker='Default'): odata = create_commits(length=commit_amount, marker=marker) data = [(d.type_num, d.as_raw_string()) for d in odata] f = BytesIO() build_pack(f, data, store=sos) sos.add_thin_pack(f.read, None) return odata
def setUp(self): super(TestThinPack, self).setUp() self.store = MemoryObjectStore() self.blobs = {} for blob in ('foo', 'bar', 'foo1234', 'bar2468'): self.blobs[blob] = make_object(Blob, data=blob) self.store.add_object(self.blobs['foo']) self.store.add_object(self.blobs['bar']) # Build a thin pack. 'foo' is as an external reference, 'bar' an # internal reference. self.pack_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.pack_dir) self.pack_prefix = os.path.join(self.pack_dir, 'pack') f = open(self.pack_prefix + '.pack', 'wb') try: build_pack(f, [ (REF_DELTA, (self.blobs['foo'].id, 'foo1234')), (Blob.type_num, 'bar'), (REF_DELTA, (self.blobs['bar'].id, 'bar2468'))], store=self.store) finally: f.close() # Index the new pack. with self.make_pack(True) as pack: with PackData(pack._data_path) as data: data.pack = pack data.create_index(self.pack_prefix + '.idx') del self.store[self.blobs['bar'].id]
def setUp(self): super(TestThinPack, self).setUp() self.store = MemoryObjectStore() self.blobs = {} for blob in (b'foo', b'bar', b'foo1234', b'bar2468'): self.blobs[blob] = make_object(Blob, data=blob) self.store.add_object(self.blobs[b'foo']) self.store.add_object(self.blobs[b'bar']) # Build a thin pack. 'foo' is as an external reference, 'bar' an # internal reference. self.pack_dir = tempfile.mkdtemp() if not isinstance(self.pack_dir, bytes): self.pack_dir = self.pack_dir.encode(sys.getfilesystemencoding()) self.addCleanup(shutil.rmtree, self.pack_dir) self.pack_prefix = os.path.join(self.pack_dir, b'pack') with open(self.pack_prefix + b'.pack', 'wb') as f: build_pack(f, [ (REF_DELTA, (self.blobs[b'foo'].id, b'foo1234')), (Blob.type_num, b'bar'), (REF_DELTA, (self.blobs[b'bar'].id, b'bar2468'))], store=self.store) # Index the new pack. with self.make_pack(True) as pack: with PackData(pack._data_path) as data: data.pack = pack data.create_index(self.pack_prefix + b'.idx') del self.store[self.blobs[b'bar'].id]
def setUp(self): super(TestThinPack, self).setUp() self.store = MemoryObjectStore() self.blobs = {} for blob in ('foo', 'bar', 'foo1234', 'bar2468'): self.blobs[blob] = make_object(Blob, data=blob) self.store.add_object(self.blobs['foo']) self.store.add_object(self.blobs['bar']) # Build a thin pack. 'foo' is as an external reference, 'bar' an # internal reference. self.pack_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.pack_dir) self.pack_prefix = os.path.join(self.pack_dir, 'pack') f = open(self.pack_prefix + '.pack', 'wb') try: build_pack(f, [ (REF_DELTA, (self.blobs['foo'].id, 'foo1234')), (Blob.type_num, 'bar'), (REF_DELTA, (self.blobs['bar'].id, 'bar2468'))], store=self.store) finally: f.close() # Index the new pack. pack = self.make_pack(True) data = PackData(pack._data_path) data.pack = pack data.create_index(self.pack_prefix + '.idx') del self.store[self.blobs['bar'].id]
def setUp(self): super(TestThinPack, self).setUp() self.store = MemoryObjectStore() self.blobs = {} for blob in (b"foo", b"bar", b"foo1234", b"bar2468"): self.blobs[blob] = make_object(Blob, data=blob) self.store.add_object(self.blobs[b"foo"]) self.store.add_object(self.blobs[b"bar"]) # Build a thin pack. 'foo' is as an external reference, 'bar' an # internal reference. self.pack_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.pack_dir) self.pack_prefix = os.path.join(self.pack_dir, "pack") with open(self.pack_prefix + ".pack", "wb") as f: build_pack( f, [ (REF_DELTA, (self.blobs[b"foo"].id, b"foo1234")), (Blob.type_num, b"bar"), (REF_DELTA, (self.blobs[b"bar"].id, b"bar2468")), ], store=self.store, ) # Index the new pack. with self.make_pack(True) as pack: with PackData(pack._data_path) as data: data.pack = pack data.create_index(self.pack_prefix + ".idx") del self.store[self.blobs[b"bar"].id]
def test_read_objects_buffered(self): f = BytesIO() build_pack(f, [ (Blob.type_num, 'blob'), (OFS_DELTA, (0, 'blob1')), ]) reader = PackStreamReader(f.read, zlib_bufsize=4) self.assertEqual(2, len(list(reader.read_objects())))
def test_bad_ext_ref_non_thin_pack(self): blob, = self.store_blobs([b'blob']) f = BytesIO() build_pack(f, [(REF_DELTA, (blob.id, b'blob1'))], store=self.store) pack_iter = self.make_pack_iter(f, thin=False) try: list(pack_iter._walk_all_chains()) self.fail() except KeyError as e: self.assertEqual(([blob.id],), e.args)
def test_read_objects(self): f = BytesIO() entries = build_pack(f, [ (Blob.type_num, 'blob'), (OFS_DELTA, (0, 'blob1')), ]) reader = PackStreamReader(f.read) objects = list(reader.read_objects(compute_crc32=True)) self.assertEqual(2, len(objects)) unpacked_blob, unpacked_delta = objects self.assertEqual(entries[0][0], unpacked_blob.offset) self.assertEqual(Blob.type_num, unpacked_blob.pack_type_num) self.assertEqual(Blob.type_num, unpacked_blob.obj_type_num) self.assertEqual(None, unpacked_blob.delta_base) self.assertEqual('blob', ''.join(unpacked_blob.decomp_chunks)) self.assertEqual(entries[0][4], unpacked_blob.crc32) self.assertEqual(entries[1][0], unpacked_delta.offset) self.assertEqual(OFS_DELTA, unpacked_delta.pack_type_num) self.assertEqual(None, unpacked_delta.obj_type_num) self.assertEqual(unpacked_delta.offset - unpacked_blob.offset, unpacked_delta.delta_base) delta = create_delta('blob', 'blob1') self.assertEqual(delta, ''.join(unpacked_delta.decomp_chunks)) self.assertEqual(entries[1][4], unpacked_delta.crc32)
def test_add_thin_pack(self): o = DiskObjectStore(self.store_dir) blob = make_object(Blob, data='yummy data') o.add_object(blob) f = StringIO() entries = build_pack(f, [ (REF_DELTA, (blob.id, 'more yummy data')), ], store=o) pack = o.add_thin_pack(f.read, None) try: packed_blob_sha = sha_to_hex(entries[0][3]) pack.check_length_and_checksum() self.assertEqual(sorted([blob.id, packed_blob_sha]), list(pack)) self.assertTrue(o.contains_packed(packed_blob_sha)) self.assertTrue(o.contains_packed(blob.id)) self.assertEqual((Blob.type_num, 'more yummy data'), o.get_raw(packed_blob_sha)) finally: # FIXME: DiskObjectStore should have close() which do the following: for p in o._pack_cache or []: p.close() pack.close()
def test_add_thin_pack(self): o = DiskObjectStore(self.store_dir) try: blob = make_object(Blob, data=b"yummy data") o.add_object(blob) f = BytesIO() entries = build_pack( f, [ (REF_DELTA, (blob.id, b"more yummy data")), ], store=o, ) with o.add_thin_pack(f.read, None) as pack: packed_blob_sha = sha_to_hex(entries[0][3]) pack.check_length_and_checksum() self.assertEqual(sorted([blob.id, packed_blob_sha]), list(pack)) self.assertTrue(o.contains_packed(packed_blob_sha)) self.assertTrue(o.contains_packed(blob.id)) self.assertEqual( (Blob.type_num, b"more yummy data"), o.get_raw(packed_blob_sha), ) finally: o.close()
def test_add_thin_pack_empty(self): o = MemoryObjectStore() f = BytesIO() entries = build_pack(f, [], store=o) self.assertEqual([], entries) o.add_thin_pack(f.read, None)
def test_add_thin_pack_empty(self): o = DiskObjectStore(self.store_dir) f = BytesIO() entries = build_pack(f, [], store=o) self.assertEquals([], entries) o.add_thin_pack(f.read, None)
def test_bad_ext_ref_thin_pack(self): b1, b2, b3 = self.store_blobs(['foo', 'bar', 'baz']) f = BytesIO() build_pack(f, [ (REF_DELTA, (1, 'foo99')), (REF_DELTA, (b1.id, 'foo1')), (REF_DELTA, (b2.id, 'bar2')), (REF_DELTA, (b3.id, 'baz3')), ], store=self.store) del self.store[b2.id] del self.store[b3.id] pack_iter = self.make_pack_iter(f) try: list(pack_iter._walk_all_chains()) self.fail() except KeyError as e: self.assertEqual((sorted([b2.id, b3.id]),), (sorted(e.args[0]),))
def test_add_thin_pack_ext_ref(self): sos = swift.SwiftObjectStore(self.fsc) odata = self._put_pack(sos, 1, 'Default1') ref_blob_content = odata[0].as_raw_string() ref_blob_id = odata[0].id new_blob = Blob.from_string(ref_blob_content.replace('blob', 'yummy blob')) blob, tree, tag, cmt = \ create_commit([], marker='Default2', blob=new_blob) data = [(REF_DELTA, (ref_blob_id, blob.as_raw_string())), (tree.type_num, tree.as_raw_string()), (cmt.type_num, cmt.as_raw_string()), (tag.type_num, tag.as_raw_string())] f = BytesIO() build_pack(f, data, store=sos) sos.add_thin_pack(f.read, None) self.assertEqual(len(self.fsc.store), 6)
def test_long_chain(self): n = 100 objects_spec = [(Blob.type_num, b'blob')] for i in range(n): objects_spec.append((OFS_DELTA, (i, b'blob' + str(i).encode('ascii')))) f = BytesIO() entries = build_pack(f, objects_spec) self.assertEntriesMatch(range(n + 1), entries, self.make_pack_iter(f))
def test_ref_deltas_chain(self): f = BytesIO() entries = build_pack(f, [ (REF_DELTA, (2, 'blob1')), (Blob.type_num, ('blob')), (REF_DELTA, (1, 'blob2')), ]) self.assertEntriesMatch([1, 2, 0], entries, self.make_pack_iter(f))
def test_branchy_chain(self): n = 100 objects_spec = [(Blob.type_num, 'blob')] for i in range(n): objects_spec.append((OFS_DELTA, (0, 'blob%i' % i))) f = BytesIO() entries = build_pack(f, objects_spec) self.assertEntriesMatch(range(n + 1), entries, self.make_pack_iter(f))
def test_ext_ref(self): blob, = self.store_blobs(['blob']) f = BytesIO() entries = build_pack(f, [(REF_DELTA, (blob.id, 'blob1'))], store=self.store) pack_iter = self.make_pack_iter(f) self.assertEntriesMatch([0], entries, pack_iter) self.assertEqual([hex_to_sha(blob.id)], pack_iter.ext_refs())
def test_add_thin_pack_ext_ref(self): sos = swift.SwiftObjectStore(self.fsc) odata = self._put_pack(sos, 1, 'Default1') ref_blob_content = odata[0].as_raw_string() ref_blob_id = odata[0].id new_blob = Blob.from_string( ref_blob_content.replace('blob', 'yummy blob')) blob, tree, tag, cmt = \ create_commit([], marker='Default2', blob=new_blob) data = [(REF_DELTA, (ref_blob_id, blob.as_raw_string())), (tree.type_num, tree.as_raw_string()), (cmt.type_num, cmt.as_raw_string()), (tag.type_num, tag.as_raw_string())] f = BytesIO() build_pack(f, data, store=sos) sos.add_thin_pack(f.read, None) self.assertEqual(len(self.fsc.store), 6)
def test_ofs_deltas_chain(self): f = BytesIO() entries = build_pack(f, [ (Blob.type_num, 'blob'), (OFS_DELTA, (0, 'blob1')), (OFS_DELTA, (1, 'blob2')), ]) self.assertEntriesMatch([0, 1, 2], entries, self.make_pack_iter(f))
def test_no_deltas(self): f = BytesIO() entries = build_pack(f, [ (Commit.type_num, 'commit'), (Blob.type_num, 'blob'), (Tree.type_num, 'tree'), ]) self.assertEntriesMatch([0, 1, 2], entries, self.make_pack_iter(f))
def test_branchy_chain(self): n = 100 objects_spec = [(Blob.type_num, b"blob")] for i in range(n): objects_spec.append( (OFS_DELTA, (0, b"blob" + str(i).encode("ascii")))) f = BytesIO() entries = build_pack(f, objects_spec) self.assertEntriesMatch(range(n + 1), entries, self.make_pack_iter(f))
def test_ofs_and_ref_deltas(self): # Deltas pending on this offset are popped before deltas depending on # this ref. f = BytesIO() entries = build_pack(f, [ (REF_DELTA, (1, 'blob1')), (Blob.type_num, ('blob')), (OFS_DELTA, (1, 'blob2')), ]) self.assertEntriesMatch([1, 2, 0], entries, self.make_pack_iter(f))
def test_multiple_ext_refs(self): b1, b2 = self.store_blobs(['foo', 'bar']) f = BytesIO() entries = build_pack(f, [ (REF_DELTA, (b1.id, 'foo1')), (REF_DELTA, (b2.id, 'bar2')), ], store=self.store) pack_iter = self.make_pack_iter(f) self.assertEntriesMatch([0, 1], entries, pack_iter) self.assertEqual([hex_to_sha(b1.id), hex_to_sha(b2.id)], pack_iter.ext_refs())
def test_no_deltas(self): f = BytesIO() entries = build_pack( f, [ (Commit.type_num, b"commit"), (Blob.type_num, b"blob"), (Tree.type_num, b"tree"), ], ) self.assertEntriesMatch([0, 1, 2], entries, self.make_pack_iter(f))
def test_ref_deltas(self): f = BytesIO() entries = build_pack( f, [ (REF_DELTA, (1, b"blob1")), (Blob.type_num, (b"blob")), (REF_DELTA, (1, b"blob2")), ], ) self.assertEntriesMatch([1, 0, 2], entries, self.make_pack_iter(f))
def test_ofs_deltas(self): f = BytesIO() entries = build_pack( f, [ (Blob.type_num, b"blob"), (OFS_DELTA, (0, b"blob1")), (OFS_DELTA, (0, b"blob2")), ], ) self.assertEntriesMatch([0, 1, 2], entries, self.make_pack_iter(f))
def test_branchy_chain(self): n = 100 objects_spec = [(Blob.type_num, b"blob")] for i in range(n): objects_spec.append( (OFS_DELTA, (0, b"blob" + str(i).encode("ascii")))) f = BytesIO() entries = build_pack(f, objects_spec) # Delta resolution changed to DFS indices = [0] + list(range(100, 0, -1)) self.assertEntriesMatch(indices, entries, self.make_pack_iter(f))
def test_mixed_chain(self): f = BytesIO() entries = build_pack(f, [ (Blob.type_num, 'blob'), (REF_DELTA, (2, 'blob2')), (OFS_DELTA, (0, 'blob1')), (OFS_DELTA, (1, 'blob3')), (OFS_DELTA, (0, 'bob')), ]) self.assertEntriesMatch([0, 2, 1, 3, 4], entries, self.make_pack_iter(f))
def test_ref_deltas(self): f = BytesIO() entries = build_pack( f, [ (REF_DELTA, (1, b"blob1")), (Blob.type_num, (b"blob")), (REF_DELTA, (1, b"blob2")), ], ) # Delta resolution changed to DFS self.assertEntriesMatch([1, 2, 0], entries, self.make_pack_iter(f))
def test_ofs_deltas(self): f = BytesIO() entries = build_pack( f, [ (Blob.type_num, b"blob"), (OFS_DELTA, (0, b"blob1")), (OFS_DELTA, (0, b"blob2")), ], ) # Delta resolution changed to DFS self.assertEntriesMatch([0, 2, 1], entries, self.make_pack_iter(f))
def test_mixed_chain(self): f = BytesIO() entries = build_pack(f, [ (Blob.type_num, b"blob"), (REF_DELTA, (2, b"blob2")), (OFS_DELTA, (0, b"blob1")), (OFS_DELTA, (1, b"blob3")), (OFS_DELTA, (0, b"bob")), ]) # Delta resolution changed to DFS self.assertEntriesMatch([0, 4, 2, 1, 3], entries, self.make_pack_iter(f))
def test_add_thin_pack(self): o = MemoryObjectStore() blob = make_object(Blob, data=b'yummy data') o.add_object(blob) f = BytesIO() entries = build_pack(f, [ (REF_DELTA, (blob.id, b'more yummy data')), ], store=o) o.add_thin_pack(f.read, None) packed_blob_sha = sha_to_hex(entries[0][3]) self.assertEqual((Blob.type_num, b'more yummy data'), o.get_raw(packed_blob_sha))
def setUp(self): conf = swift.load_conf(file=StringIO(config_file % def_config_file)) sos = swift.SwiftObjectStore(FakeSwiftConnector("fakerepo", conf=conf)) commit_amount = 10 self.commits = create_commits(length=commit_amount, marker="m") data = [(d.type_num, d.as_raw_string()) for d in self.commits] f = StringIO() fi = StringIO() expected = build_pack(f, data, store=sos) entries = [(sha, ofs, checksum) for ofs, _, _, sha, checksum in expected] self.pack_data = PackData.from_file(file=f, size=None) write_pack_index_v2(fi, entries, self.pack_data.calculate_checksum()) fi.seek(0) self.pack_index = load_pack_index_file("", fi)
def test_ext_ref_chain_degenerate(self): # Test a degenerate case where the sender is sending a REF_DELTA # object that expands to an object already in the repository. blob, = self.store_blobs(['blob']) blob2, = self.store_blobs(['blob2']) assert blob.id < blob2.id f = BytesIO() entries = build_pack(f, [ (REF_DELTA, (blob.id, 'blob2')), (REF_DELTA, (0, 'blob3')), ], store=self.store) pack_iter = self.make_pack_iter(f) self.assertEntriesMatch([0, 1], entries, pack_iter) self.assertEqual([hex_to_sha(blob.id)], pack_iter.ext_refs())
def test_add_thin_pack(self): o = DiskObjectStore(self.store_dir) blob = make_object(Blob, data='yummy data') o.add_object(blob) f = StringIO() entries = build_pack(f, [ (REF_DELTA, (blob.id, 'more yummy data')), ], store=o) pack = o.add_thin_pack(f.read, None) packed_blob_sha = sha_to_hex(entries[0][3]) pack.check_length_and_checksum() self.assertEqual(sorted([blob.id, packed_blob_sha]), list(pack)) self.assertTrue(o.contains_packed(packed_blob_sha)) self.assertTrue(o.contains_packed(blob.id)) self.assertEqual((Blob.type_num, 'more yummy data'), o.get_raw(packed_blob_sha))
def test_add_thin_pack(self): blob = make_object(Blob, data=b'yummy data') with DiskObjectStore(self.store_dir) as o: o.add_object(blob) with BytesIO() as f: # BUG IS ON THE NEXT LINE entries = build_pack(f, [ (REF_DELTA, (blob.id, b'more yummy data')), ], store=o) with o.add_thin_pack(f.read, None) as pack: packed_blob_sha = entries[0][3] pack.check_length_and_checksum() self.assertEqual(sorted([blob.id, packed_blob_sha]), list(pack)) self.assertTrue(o.contains_packed(packed_blob_sha)) self.assertTrue(o.contains_packed(blob.id)) self.assertEqual((Blob.type_num, b'more yummy data'), o.get_raw(packed_blob_sha))
def test_add_thin_pack_empty(self): with closing(DiskObjectStore(self.store_dir)) as o: f = BytesIO() entries = build_pack(f, [], store=o) self.assertEqual([], entries) o.add_thin_pack(f.read, None)
def test_read_objects_emtpy(self): f = BytesIO() build_pack(f, []) reader = PackStreamReader(f.read) self.assertEqual(0, len(list(reader.read_objects())))