def __init__(self, key):
self.buffer = BytesIO()
self.packer = msgpack.Packer(unicode_errors='surrogateescape')
self.chunks = []
self.key = key
self.chunker = Chunker(WINDOW_SIZE, CHUNK_MASK, CHUNK_MIN,
self.key.chunk_seed)
def __init__(self,
repository,
key,
manifest,
name,
cache=None,
create=False,
checkpoint_interval=300,
numeric_owner=False):
self.cwd = os.getcwd()
self.key = key
self.repository = repository
self.cache = cache
self.manifest = manifest
self.hard_links = {}
self.stats = Statistics()
self.name = name
self.checkpoint_interval = checkpoint_interval
self.numeric_owner = numeric_owner
self.pipeline = DownloadPipeline(self.repository, self.key)
if create:
self.items_buffer = CacheChunkBuffer(self.cache, self.key,
self.stats)
self.chunker = Chunker(WINDOW_SIZE, CHUNK_MASK, CHUNK_MIN,
self.key.chunk_seed)
if name in manifest.archives:
raise self.AlreadyExists(name)
self.last_checkpoint = time.time()
i = 0
while True:
self.checkpoint_name = '%s.checkpoint%s' % (name, i and
('.%d' % i) or '')
if not self.checkpoint_name in manifest.archives:
break
i += 1
else:
if name not in self.manifest.archives:
raise self.DoesNotExist(name)
info = self.manifest.archives[name]
self.load(info[b'id'])
def test_chunkify(self):
data = b'0' * 1024 * 1024 * 15 + b'Y'
parts = [
bytes(c) for c in Chunker(2, 0x3, 2, 0).chunkify(BytesIO(data))
]
self.assert_equal(len(parts), 2)
self.assert_equal(b''.join(parts), data)
self.assert_equal(
[bytes(c) for c in Chunker(2, 0x3, 2, 0).chunkify(BytesIO(b''))],
[])
self.assert_equal([
bytes(c) for c in Chunker(2, 0x3, 2, 0).chunkify(
BytesIO(b'foobarboobaz' * 3))
], [b'fooba', b'rboobaz', b'fooba', b'rboobaz', b'fooba', b'rboobaz'])
self.assert_equal([
bytes(c) for c in Chunker(2, 0x3, 2, 1).chunkify(
BytesIO(b'foobarboobaz' * 3))
], [
b'fo', b'obarb', b'oob', b'azf', b'oobarb', b'oob', b'azf',
b'oobarb', b'oobaz'
])
self.assert_equal([
bytes(c) for c in Chunker(2, 0x3, 2, 2).chunkify(
BytesIO(b'foobarboobaz' * 3))
], [
b'foob', b'ar', b'boobazfoob', b'ar', b'boobazfoob', b'ar',
b'boobaz'
])
self.assert_equal([
bytes(c) for c in Chunker(3, 0x3, 3, 0).chunkify(
BytesIO(b'foobarboobaz' * 3))
], [b'foobarboobaz' * 3])
self.assert_equal([
bytes(c) for c in Chunker(3, 0x3, 3, 1).chunkify(
BytesIO(b'foobarboobaz' * 3))
], [
b'foobar', b'boo', b'bazfo', b'obar', b'boo', b'bazfo', b'obar',
b'boobaz'
])
self.assert_equal([
bytes(c) for c in Chunker(3, 0x3, 3, 2).chunkify(
BytesIO(b'foobarboobaz' * 3))
], [b'foo', b'barboobaz', b'foo', b'barboobaz', b'foo', b'barboobaz'])
self.assert_equal([
bytes(c) for c in Chunker(3, 0x3, 4, 0).chunkify(
BytesIO(b'foobarboobaz' * 3))
], [b'foobarboobaz' * 3])
self.assert_equal([
bytes(c) for c in Chunker(3, 0x3, 4, 1).chunkify(
BytesIO(b'foobarboobaz' * 3))
], [b'foobar', b'boobazfo', b'obar', b'boobazfo', b'obar', b'boobaz'])
self.assert_equal([
bytes(c) for c in Chunker(3, 0x3, 4, 2).chunkify(
BytesIO(b'foobarboobaz' * 3))
], [b'foob', b'arboobaz', b'foob', b'arboobaz', b'foob', b'arboobaz'])
