def write_record_v3(archive,fh,compression_method=COMPR_NONE,encrypted=False,compression_block_size=0):
if compression_method != COMPR_NONE and compression_method != COMPR_ZLIB:
raise NotImplementedError("compression is not implemented")
record_offset = archive.tell()
if compression_block_size == 0 and compression_method == COMPR_ZLIB:
compression_block_size = 65536
st = os.fstat(fh.fileno())
size = st.st_size
record = st_pack('<16xQI20x',size,compression_method)
archive.write(record)
if compression_method == COMPR_ZLIB:
compressed_size, sha1,block_count, blocks = write_data_zlib(archive,fh,size,compression_method,encrypted,compression_block_size)
else:
record = st_pack('<BI',int(encrypted),compression_block_size)
archive.write(record)
compressed_size, sha1 = write_data(archive,fh,size,compression_method,encrypted,compression_block_size)
data_end = archive.tell()
archive.seek(record_offset+8, 0)
archive.write(st_pack('<Q',compressed_size))
archive.seek(record_offset+28, 0)
archive.write(sha1)
archive.seek(data_end, 0)
if compression_method == COMPR_ZLIB:
return st_pack('<QQQI20s',record_offset,compressed_size,size,compression_method,sha1) + st_pack('<I%dQ' % (block_count * 2), block_count, *blocks) + st_pack('<BI',int(encrypted),compression_block_size)
else:
return st_pack('<QQQI20sBI',record_offset,compressed_size,size,compression_method,sha1,int(encrypted),compression_block_size)
def write_index(stream,version,mount_point,records):
hasher = hashlib.sha1()
index_offset = stream.tell()
index_header = pack_path(mount_point) + st_pack('<I',len(records))
index_size = len(index_header)
hasher.update(index_header)
stream.write(index_header)
for filename, record in records:
filename = pack_path(filename)
hasher.update(filename)
stream.write(filename)
index_size += len(filename)
hasher.update(record)
stream.write(record)
index_size += len(record)
index_sha1 = hasher.digest()
stream.write(st_pack('<IIQQ20s', 0x5A6F12E1, version, index_offset, index_size, index_sha1))
def write_index(stream,version,mount_point,records):
hasher = hashlib.sha1()
index_offset = stream.tell()
index_header = pack_path(mount_point) + st_pack('<I',len(records))
index_size = len(index_header)
hasher.update(index_header)
stream.write(index_header)
for filename, record in records:
filename = pack_path(filename)
hasher.update(filename)
stream.write(filename)
index_size += len(filename)
hasher.update(record)
stream.write(record)
index_size += len(record)
index_sha1 = hasher.digest()
stream.write(st_pack('<IIQQ20s', 0x5A6F12E1, version, index_offset, index_size, index_sha1))
def write_record_v2(archive,fh,compression_method=COMPR_NONE,encrypted=False,compression_block_size=0):
if encrypted:
raise ValueError('version 2 does not support encryption')
record_offset = archive.tell()
st = os.fstat(fh.fileno())
size = st.st_size
record = st_pack('<16xQI20x',size,compression_method)
archive.write(record)
compressed_size, sha1 = write_data(archive,fh,size,compression_method,encrypted,compression_block_size)
data_end = archive.tell()
archive.seek(record_offset+8, 0)
archive.write(st_pack('<Q',compressed_size))
archive.seek(record_offset+28, 0)
archive.write(sha1)
archive.seek(data_end, 0)
return st_pack('<QQQI20s',record_offset,compressed_size,size,compression_method,sha1)
def write_record_v3(archive,fh,compression_method=COMPR_NONE,encrypted=False,compression_block_size=0):
if compression_method != COMPR_NONE:
raise NotImplementedError("compression is not implemented")
record_offset = archive.tell()
st = os.fstat(fh.fileno())
size = st.st_size
record = st_pack('<16xQI20xBI',size,compression_method,int(encrypted),compression_block_size)
archive.write(record)
compressed_size, sha1 = write_data(archive,fh,size,compression_method,encrypted,compression_block_size)
data_end = archive.tell()
archive.seek(record_offset+8, 0)
archive.write(st_pack('<Q',compressed_size))
archive.seek(record_offset+28, 0)
archive.write(sha1)
archive.seek(data_end, 0)
return st_pack('<QQQI20sBI',record_offset,compressed_size,size,compression_method,sha1,int(encrypted),compression_block_size)
def write_record_v2(archive,fh,compression_method=COMPR_NONE,encrypted=False,compression_block_size=0):
if encrypted:
raise ValueError('version 2 does not support encryption')
record_offset = archive.tell()
st = os.fstat(fh.fileno())
size = st.st_size
record = st_pack('<16xQI20x',size,compression_method)
archive.write(record)
compressed_size, sha1 = write_data(archive,fh,size,compression_method,encrypted,compression_block_size)
data_end = archive.tell()
archive.seek(record_offset+8, 0)
archive.write(st_pack('<Q',compressed_size))
archive.seek(record_offset+28, 0)
archive.write(sha1)
archive.seek(data_end, 0)
return st_pack('<QQQI20s',record_offset,compressed_size,size,compression_method,sha1)
def write_record_v1(archive,fh,compression_method=COMPR_NONE,encrypted=False,compression_block_size=0):
if encrypted:
raise ValueError('version 1 does not support encryption')
record_offset = archive.tell()
st = os.fstat(fh.fileno())
size = st.st_size
# XXX: timestamp probably needs multiplication with some factor?
record = st_pack('<16xQIQ20x',size,compression_method,int(st.st_mtime))
archive.write(record)
compressed_size, sha1 = write_data(archive,fh,size,compression_method,encrypted,compression_block_size)
data_end = archive.tell()
archive.seek(record_offset+8, 0)
archive.write(st_pack('<Q',compressed_size))
archive.seek(record_offset+36, 0)
archive.write(sha1)
archive.seek(data_end, 0)
return st_pack('<QQQIQ20s',record_offset,compressed_size,size,compression_method,int(st.st_mtime),sha1)
def write_record_v1(archive,fh,compression_method=COMPR_NONE,encrypted=False,compression_block_size=0):
if encrypted:
raise ValueError('version 1 does not support encryption')
record_offset = archive.tell()
st = os.fstat(fh.fileno())
size = st.st_size
# XXX: timestamp probably needs multiplication with some factor?
record = st_pack('<16xQIQ20x',size,compression_method,int(st.st_mtime))
archive.write(record)
compressed_size, sha1 = write_data(archive,fh,size,compression_method,encrypted,compression_block_size)
data_end = archive.tell()
archive.seek(record_offset+8, 0)
archive.write(st_pack('<Q',compressed_size))
archive.seek(record_offset+36, 0)
archive.write(sha1)
archive.seek(data_end, 0)
return st_pack('<QQQIQ20s',record_offset,compressed_size,size,compression_method,int(st.st_mtime),sha1)
def write_data_zlib(archive,fh,size,compression_method=COMPR_NONE,encrypted=False,compression_block_size=65536):
if encrypted:
raise NotImplementedError("encryption is not implemented")
buf_size = compression_block_size
block_count = int(math.ceil(size / compression_block_size))
base_offset = archive.tell()
archive.write(st_pack('<I',block_count))
# Seek Skip Offset
archive.seek(block_count * 8 * 2, 1)
record = st_pack('<BI',int(encrypted),compression_block_size)
archive.write(record)
cur_offset = base_offset + 4 + block_count * 8 * 2 + 5
compress_blocks = [0] * block_count * 2
compressed_size = 0
compress_block_no = 0
buf = bytearray(buf_size)
bytes_left = size
hasher = hashlib.sha1()
while bytes_left > 0:
if bytes_left >= buf_size:
n = fh.readinto(buf)
data = zlib.compress(buffer(buf))
compressed_size += len(data)
compress_blocks[compress_block_no * 2] = cur_offset
cur_offset += len(data)
compress_blocks[compress_block_no * 2 + 1] = cur_offset
compress_block_no += 1
if n < buf_size:
raise IOError('unexpected end of file')
else:
data = fh.read(bytes_left)
n = len(data)
data = zlib.compress(data)
compressed_size += len(data)
compress_blocks[compress_block_no * 2] = cur_offset
cur_offset += len(data)
compress_blocks[compress_block_no * 2 + 1] = cur_offset
compress_block_no += 1
if n < bytes_left:
raise IOError('unexpected end of file')
bytes_left -= n
hasher.update(data)
archive.write(data)
cur_offset = archive.tell()
archive.seek(base_offset + 4, 0)
archive.write(st_pack('<%dQ' % (block_count * 2), *compress_blocks))
archive.seek(cur_offset, 0)
return compressed_size, hasher.digest(), block_count, compress_blocks
def pack_path(path):
path = path.replace(os.path.sep,'/').encode('utf-8') + b'\0'
return st_pack('<I', len(path)) + path
def pack_path(path):
path = path.replace(os.path.sep,'/').encode('utf-8') + b'\0'
return st_pack('<I', len(path)) + path