def _collect_infos(dirname):
""" Utility function used by ExplodedZipFile to generate ZipInfo
entries for all of the files and directories under dirname """
for r, _ds, fs in walk(dirname):
if not islink(r) and r != dirname:
i = ZipInfo()
i.filename = join(relpath(r, dirname), "")
i.file_size = 0
i.compress_size = 0
i.CRC = 0
yield i.filename, i
for f in fs:
df = join(r, f)
relfn = relpath(join(r, f), dirname)
if islink(df):
pass
elif isfile(df):
i = ZipInfo()
i.filename = relfn
i.file_size = getsize(df)
i.compress_size = i.file_size
i.CRC = file_crc32(df)
yield i.filename, i
else:
# TODO: is there any more special treatment?
pass
def _collect_infos(dirname):
""" Utility function used by ExplodedZipFile to generate ZipInfo
entries for all of the files and directories under dirname """
for r, _ds, fs in walk(dirname):
if not islink(r) and r != dirname:
i = ZipInfo()
i.filename = join(relpath(r, dirname), "")
i.file_size = 0
i.compress_size = 0
i.CRC = 0
yield i.filename, i
for f in fs:
df = join(r, f)
relfn = relpath(join(r, f), dirname)
if islink(df):
pass
elif isfile(df):
i = ZipInfo()
i.filename = relfn
i.file_size = getsize(df)
i.compress_size = i.file_size
i.CRC = file_crc32(df)
yield i.filename, i
else:
# TODO: is there any more special treatment?
pass
def create_zipinfo(filename, mtime=None, dir=False, executable=False, symlink=False,
comment=None):
"""Create a instance of `ZipInfo`.
:param filename: file name of the entry
:param mtime: modified time of the entry
:param dir: if `True`, the entry is a directory
:param executable: if `True`, the entry is a executable file
:param symlink: if `True`, the entry is a symbolic link
:param comment: comment of the entry
"""
from zipfile import ZipInfo, ZIP_DEFLATED, ZIP_STORED
zipinfo = ZipInfo()
# The general purpose bit flag 11 is used to denote
# UTF-8 encoding for path and comment. Only set it for
# non-ascii files for increased portability.
# See http://www.pkware.com/documents/casestudies/APPNOTE.TXT
if any(ord(c) >= 128 for c in filename):
zipinfo.flag_bits |= 0x0800
zipinfo.filename = filename.encode('utf-8')
if mtime is not None:
mtime = to_datetime(mtime, utc)
zipinfo.date_time = mtime.utctimetuple()[:6]
# The "extended-timestamp" extra field is used for the
# modified time of the entry in unix time. It avoids
# extracting wrong modified time if non-GMT timezone.
# See http://www.opensource.apple.com/source/zip/zip-6/unzip/unzip
# /proginfo/extra.fld
zipinfo.extra += struct.pack(
'<hhBl',
0x5455, # extended-timestamp extra block type
1 + 4, # size of this block
1, # modification time is present
to_timestamp(mtime)) # time of last modification
# external_attr is 4 bytes in size. The high order two
# bytes represent UNIX permission and file type bits,
# while the low order two contain MS-DOS FAT file
# attributes, most notably bit 4 marking directories.
if dir:
if not zipinfo.filename.endswith('/'):
zipinfo.filename += '/'
zipinfo.compress_type = ZIP_STORED
zipinfo.external_attr = 040755 << 16L # permissions drwxr-xr-x
zipinfo.external_attr |= 0x10 # MS-DOS directory flag
else:
zipinfo.compress_type = ZIP_DEFLATED
zipinfo.external_attr = 0644 << 16L # permissions -r-wr--r--
if executable:
zipinfo.external_attr |= 0755 << 16L # -rwxr-xr-x
if symlink:
zipinfo.compress_type = ZIP_STORED
zipinfo.external_attr |= 0120000 << 16L # symlink file type
if comment:
zipinfo.comment = comment.encode('utf-8')
return zipinfo
def create_zipinfo(filename, mtime=None, dir=False, executable=False, symlink=False,
comment=None):
"""Create a instance of `ZipInfo`.
:param filename: file name of the entry
:param mtime: modified time of the entry
:param dir: if `True`, the entry is a directory
:param executable: if `True`, the entry is a executable file
:param symlink: if `True`, the entry is a symbolic link
:param comment: comment of the entry
"""
from zipfile import ZipInfo, ZIP_DEFLATED, ZIP_STORED
zipinfo = ZipInfo()
# The general purpose bit flag 11 is used to denote
# UTF-8 encoding for path and comment. Only set it for
# non-ascii files for increased portability.
# See http://www.pkware.com/documents/casestudies/APPNOTE.TXT
if any(ord(c) >= 128 for c in filename):
zipinfo.flag_bits |= 0x0800
zipinfo.filename = filename.encode('utf-8')
if mtime is not None:
mtime = to_datetime(mtime, utc)
zipinfo.date_time = mtime.utctimetuple()[:6]
# The "extended-timestamp" extra field is used for the
# modified time of the entry in unix time. It avoids
# extracting wrong modified time if non-GMT timezone.
# See http://www.opensource.apple.com/source/zip/zip-6/unzip/unzip
# /proginfo/extra.fld
zipinfo.extra += struct.pack(
'<hhBl',
0x5455, # extended-timestamp extra block type
1 + 4, # size of this block
1, # modification time is present
to_timestamp(mtime)) # time of last modification
# external_attr is 4 bytes in size. The high order two
# bytes represent UNIX permission and file type bits,
# while the low order two contain MS-DOS FAT file
# attributes, most notably bit 4 marking directories.
if dir:
if not zipinfo.filename.endswith('/'):
zipinfo.filename += '/'
zipinfo.compress_type = ZIP_STORED
zipinfo.external_attr = 040755 << 16L # permissions drwxr-xr-x
zipinfo.external_attr |= 0x10 # MS-DOS directory flag
else:
zipinfo.compress_type = ZIP_DEFLATED
zipinfo.external_attr = 0644 << 16L # permissions -r-wr--r--
if executable:
zipinfo.external_attr |= 0755 << 16L # -rwxr-xr-x
if symlink:
zipinfo.compress_type = ZIP_STORED
zipinfo.external_attr |= 0120000 << 16L # symlink file type
if comment:
zipinfo.comment = comment.encode('utf-8')
return zipinfo
def _render_zip(self, req, filename, repos, data):
"""ZIP archive with all the added and/or modified files."""
new_rev = data['new_rev']
req.send_response(200)
req.send_header('Content-Type', 'application/zip')
req.send_header('Content-Disposition',
content_disposition('inline', filename + '.zip'))
from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED
buf = StringIO()
zipfile = ZipFile(buf, 'w', ZIP_DEFLATED)
for old_node, new_node, kind, change in repos.get_changes(
new_path=data['new_path'], new_rev=data['new_rev'],
old_path=data['old_path'], old_rev=data['old_rev']):
if kind == Node.FILE and change != Changeset.DELETE:
assert new_node
zipinfo = ZipInfo()
zipinfo.filename = new_node.path.strip('/').encode('utf-8')
# Note: unicode filenames are not supported by zipfile.
# UTF-8 is not supported by all Zip tools either,
# but as some do, I think UTF-8 is the best option here.
zipinfo.date_time = new_node.last_modified.utctimetuple()[:6]
zipinfo.external_attr = 0644 << 16L # needed since Python 2.5
zipinfo.compress_type = ZIP_DEFLATED
zipfile.writestr(zipinfo, new_node.get_content().read())
zipfile.close()
zip_str = buf.getvalue()
req.send_header("Content-Length", len(zip_str))
req.end_headers()
req.write(zip_str)
raise RequestDone
def _add_path(self, path, version, myzip):
mtime = os.path.getmtime(path)
info = ZipInfo("versions/%d/%s" % (version, path),
Archive.unixtime_to_utcziptime(mtime))
info.create_system = 3
info.extra += struct.pack('<HHBl', 0x5455, 5, 1, int(mtime))
# http://unix.stackexchange.com/questions/14705/the-zip-formats-external-file-attribute
# make mode without file type, which may be system-specific
clean_mode = os.stat(path).st_mode & 0o007777
if (os.path.islink(path)):
# set zip file type to link
info.external_attr = (Archive.ZIP_EXT_ATTR_LINK | clean_mode) << 16
myzip.writestr(info, os.readlink(path))
elif (os.path.isdir(path)):
# set zip file type to dir
info.external_attr = (Archive.ZIP_EXT_ATTR_DIR | clean_mode) << 16
# dos directory flag
info.external_attr |= 0x10
# it seems we should have a trailing slash for dirs
if not (info.filename.endswith('/')):
info.filename = "%s/" % (info.filename)
myzip.writestr(info, '')
for name in os.listdir(path):
self._add_path(os.path.join(path, name), version, myzip)
elif (os.path.isfile(path)):
info.external_attr = (Archive.ZIP_EXT_ATTR_FILE | clean_mode) << 16
myzip.writestr(info, open(path, 'rb').read())
else:
raise Exception()
def _render_zip(self, req, repos, chgset):
"""ZIP archive with all the added and/or modified files."""
req.send_response(200)
req.send_header('Content-Type', 'application/zip')
req.send_header('Content-Disposition', 'attachment;'
'filename=Changeset%s.zip' % chgset.rev)
req.end_headers()
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED
buf = StringIO()
zipfile = ZipFile(buf, 'w', ZIP_DEFLATED)
for path, kind, change, base_path, base_rev in chgset.get_changes():
if kind == Node.FILE and change != Changeset.DELETE:
node = repos.get_node(path, chgset.rev)
zipinfo = ZipInfo()
zipinfo.filename = node.path
zipinfo.date_time = time.gmtime(node.last_modified)[:6]
zipinfo.compress_type = ZIP_DEFLATED
zipfile.writestr(zipinfo, node.get_content().read())
zipfile.close()
req.write(buf.getvalue())
def _add_path(self, path, version, myzip):
mtime = os.path.getmtime(path)
info = ZipInfo("versions/%d/%s"%(version, path), Archive.unixtime_to_utcziptime(mtime))
info.create_system = 3
info.extra += struct.pack('<HHBl', 0x5455, 5, 1, mtime)
# http://unix.stackexchange.com/questions/14705/the-zip-formats-external-file-attribute
# make mode without file type, which may be system-specific
clean_mode = os.stat(path).st_mode & 0007777
if (os.path.islink(path)):
# set zip file type to link
info.external_attr = (Archive.ZIP_EXT_ATTR_LINK | clean_mode) << 16L
myzip.writestr(info, os.readlink(path))
elif (os.path.isdir(path)):
# set zip file type to dir
info.external_attr = (Archive.ZIP_EXT_ATTR_DIR | clean_mode) << 16L
# dos directory flag
info.external_attr |= 0x10
# it seems we should have a trailing slash for dirs
if not(info.filename.endswith('/')): info.filename = "%s/"%(info.filename)
myzip.writestr(info, '')
for name in os.listdir(path):
self._add_path(os.path.join(path, name), version, myzip)
elif (os.path.isfile(path)):
info.external_attr = (Archive.ZIP_EXT_ATTR_FILE | clean_mode) << 16L
myzip.writestr(info, open(path).read())
else:
raise Exception()
def _render_zip(self, req, repos, chgset):
"""ZIP archive with all the added and/or modified files."""
req.send_response(200)
req.send_header('Content-Type', 'application/zip')
req.send_header('Content-Disposition',
'filename=Changeset%s.zip' % chgset.rev)
req.end_headers()
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED
buf = StringIO()
zipfile = ZipFile(buf, 'w', ZIP_DEFLATED)
for path, kind, change, base_path, base_rev in chgset.get_changes():
if kind == Node.FILE and change != Changeset.DELETE:
node = repos.get_node(path, chgset.rev)
zipinfo = ZipInfo()
zipinfo.filename = node.path
zipinfo.date_time = time.gmtime(node.last_modified)[:6]
zipinfo.compress_type = ZIP_DEFLATED
zipfile.writestr(zipinfo, node.get_content().read())
zipfile.close()
req.write(buf.getvalue())
def _render_zip(self, req, filename, repos, diff):
"""ZIP archive with all the added and/or modified files."""
new_rev = diff.new_rev
req.send_response(200)
req.send_header('Content-Type', 'application/zip')
req.send_header('Content-Disposition', 'attachment;'
'filename=%s.zip' % filename)
from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED
buf = StringIO()
zipfile = ZipFile(buf, 'w', ZIP_DEFLATED)
for old_node, new_node, kind, change in repos.get_changes(**diff):
if kind == Node.FILE and change != Changeset.DELETE:
assert new_node
zipinfo = ZipInfo()
zipinfo.filename = new_node.path.encode('utf-8')
# Note: unicode filenames are not supported by zipfile.
# UTF-8 is not supported by all Zip tools either,
# but as some does, I think UTF-8 is the best option here.
zipinfo.date_time = time.gmtime(new_node.last_modified)[:6]
zipinfo.compress_type = ZIP_DEFLATED
zipfile.writestr(zipinfo, new_node.get_content().read())
zipfile.close()
buf.seek(0, 2) # be sure to be at the end
req.send_header("Content-Length", buf.tell())
req.end_headers()
req.write(buf.getvalue())
def _render_zip(self, req, filename, repos, diff):
"""ZIP archive with all the added and/or modified files."""
new_rev = diff.new_rev
req.send_response(200)
req.send_header('Content-Type', 'application/zip')
req.send_header('Content-Disposition', 'attachment;'
'filename=%s.zip' % filename)
from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED
buf = StringIO()
zipfile = ZipFile(buf, 'w', ZIP_DEFLATED)
for old_node, new_node, kind, change in repos.get_changes(**diff):
if kind == Node.FILE and change != Changeset.DELETE:
assert new_node
zipinfo = ZipInfo()
zipinfo.filename = new_node.path.encode('utf-8')
# Note: unicode filenames are not supported by zipfile.
# UTF-8 is not supported by all Zip tools either,
# but as some does, I think UTF-8 is the best option here.
zipinfo.date_time = time.gmtime(new_node.last_modified)[:6]
zipinfo.compress_type = ZIP_DEFLATED
zipfile.writestr(zipinfo, new_node.get_content().read())
zipfile.close()
buf.seek(0, 2) # be sure to be at the end
req.send_header("Content-Length", buf.tell())
req.end_headers()
req.write(buf.getvalue())
def zipadd(zipfile, data, fname):
zinfo = ZipInfo()
zinfo.filename = fname
tlocal = time.localtime()
zinfo.date_time = (tlocal[0], tlocal[1] + 1, tlocal[2] + 1, tlocal[3],
tlocal[4], tlocal[5])
zinfo.compress_type = ZIP_DEFLATED
zinfo.external_attr = 0o664 << 16
zipfile.writestr(zinfo, data)
def _extract_file(self, member: ZipInfo, dst_path: Path, zf: ZipFile):
full_dst_path = self.dir / dst_path
if full_dst_path.is_dir():
raise ValueError(
f"Cannot extract {dst_path}: a directory by that name exists"
)
if self.overwrite or not full_dst_path.is_file():
member.filename = _zip_name(dst_path)
zf.extract(member, self.dir)
else:
with zf.open(member, "r") as f:
self._write_file(dst_path, f)
def _add_file_to_zip(zipfile, path, archive_dest=None):
with open(path, 'r') as f:
file_bytes = f.read()
info = ZipInfo(path)
info.date_time = time.localtime()
# Set permissions to be executable
info.external_attr = 0o100755 << 16
# If archive dest was provided, use that as path
if archive_dest:
info.filename = archive_dest
zipfile.writestr(info, file_bytes, ZIP_DEFLATED)
def make_dir_entry(name=None, date_time=None, mode=MODE_DIRECTORY):
tt = date_time.timetuple()
dir = ZipInfo()
dir.filename = name+('/' if name[-1] != '/' else '')
dir.orig_filename = dir.filename
dir.date_time = date_time.isocalendar() + (tt.tm_hour,
tt.tm_min, tt.tm_sec)
dir.compress_type = 0
dir.create_system = 0
dir.create_version = 20
dir.extract_version = 10
dir.external_attr = mode
return dir
def make_file_entry(name=None, date_time=None, mode=MODE_FILE | MODE_ARCHIVE):
tt = date_time.timetuple()
file = ZipInfo()
file.filename = name
file.orig_filename = file.filename
file.date_time = date_time.isocalendar() + (tt.tm_hour,
tt.tm_min, tt.tm_sec)
file.compress_type = 8
file.create_system = 0
file.create_version = 20
file.extract_version = 20
file.flag_bits = 2
file.external_attr = mode
return file
def zipinfo_fixup_filename(inf: zipfile.ZipInfo):
# Support UTF-8 filenames using extra fields
# Code from https://github.com/python/cpython/pull/23736
extra = inf.extra
unpack = struct.unpack
while len(extra) >= 4:
type_, length = struct.unpack("<HH", extra[:4])
if length + 4 > len(extra):
raise zipfile.BadZipFile(
f"Corrupt extra field {type_:04x} (size={length})")
if type_ == 0x7075:
data = extra[4:length + 4]
# Unicode Path Extra Field
up_version, _up_name_crc = unpack("<BL", data[:5])
up_unicode_name = data[5:].decode("utf-8")
if up_version == 1:
inf.filename = up_unicode_name
extra = extra[length + 4:]
def add_str(self, str_to_add, name, dt=datetime.now()):
# type: (str,str,datetime) -> None
"""
Add a string to the archive as zip entry named 'name'
:param str_to_add: string to add
:param name: name of the zip.entry
:param dt: datetime, optional if not specified, current date time is assumed
:return: None
"""
# always use forward slash regardless of platform, this allows the calling
# code to use os.path.join for names
if os.pathsep in name:
name = name.replace(os.pathsep, "/")
info = ZipInfo()
info.filename = self.uuid + "/" + name
info.external_attr = 0o644 << 16
info.compress_type = ZIP_DEFLATED
info.date_time = (dt.year, dt.month, dt.day, dt.hour, dt.minute,
dt.second)
self.zip.writestr(info, str_to_add)
def zip_writer(dirpath, zippath):
basedir = os.path.dirname(dirpath) + os.sep
entry = ZipInfo()
entry.compress_type = compression
if os.path.isdir(dirpath):
for root, dirs, files in os.walk(dirpath):
if os.path.basename(root).startswith('.'):
# skip hidden directories
continue
dirname = root.replace(basedir, '')
for f in files:
if f[-1] == '~' or f.startswith('.'):
# skip backup files and all hidden files
continue
src = root + '/' + f
entry = ZipInfo()
entry.compress_type = compression
entry.filename = dirname + '/' + f
entry.date_time = localtime(os.path.getmtime(src))[:6]
# hacky
if dirname.startswith("html"):
if self.source == True:
entry.filename = dirname.replace('html', 'doc', 1) + "/" + f
else:
entry.filename = dirname.replace('html/', '', 1) + "/" + f
entry.filename = entry.filename.replace('html/', '', 1)
if entry.filename.startswith("examples"):
entry.filename = "tutorials/" + entry.filename
file_data = open( src, 'rb').read()
self.package.writestr(entry, file_data)
else:
# top files
entry.date_time = localtime(os.path.getmtime(dirpath))[:6]
entry.filename = os.path.basename(zippath)
file_data = open( dirpath, 'rb').read()
self.package.writestr(entry, file_data)
def _RealGetContents(self):
"""Read in the table of contents for the ZIP file."""
try:
endrec = _EndRecData(self.url)
except IOError:
raise BadZipfile("File is not a zip file")
if not endrec:
raise BadZipfile, "File is not a zip file"
if self.debug > 1:
print endrec
size_cd = endrec[_ECD_SIZE] # bytes in central directory
offset_cd = endrec[_ECD_OFFSET] # offset of central directory
self.comment = endrec[_ECD_COMMENT] # archive comment
# "concat" is zero, unless zip was concatenated to another file
concat = endrec[_ECD_LOCATION] - size_cd - offset_cd
# if endrec[_ECD_SIGNATURE] == stringEndArchive64:
# # If Zip64 extension structures are present, account for them
# concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator)
if self.debug > 2:
inferred = concat + offset_cd
print "given, inferred, offset", offset_cd, inferred, concat
# self.start_dir: Position of start of central directory
self.start_dir = offset_cd + concat
ECD = _http_get_partial_data(self.url, self.start_dir, self.start_dir + size_cd - 1)
data = ECD.read()
ECD.close()
fp = cStringIO.StringIO(data)
total = 0
while total < size_cd:
centdir = fp.read(sizeCentralDir)
if centdir[0:4] != stringCentralDir:
raise BadZipfile, "Bad magic number for central directory"
centdir = struct.unpack(structCentralDir, centdir)
if self.debug > 2:
print centdir
filename = fp.read(centdir[_CD_FILENAME_LENGTH])
# Create ZipInfo instance to store file information
x = ZipInfo(filename)
x.extra = fp.read(centdir[_CD_EXTRA_FIELD_LENGTH])
x.comment = fp.read(centdir[_CD_COMMENT_LENGTH])
x.header_offset = centdir[_CD_LOCAL_HEADER_OFFSET]
(
x.create_version,
x.create_system,
x.extract_version,
x.reserved,
x.flag_bits,
x.compress_type,
t,
d,
x.CRC,
x.compress_size,
x.file_size,
) = centdir[1:12]
x.volume, x.internal_attr, x.external_attr = centdir[15:18]
# Convert date/time code to (year, month, day, hour, min, sec)
x._raw_time = t
x.date_time = ((d >> 9) + 1980, (d >> 5) & 0xF, d & 0x1F, t >> 11, (t >> 5) & 0x3F, (t & 0x1F) * 2)
x._decodeExtra()
x.header_offset = x.header_offset + concat
x.filename = x._decodeFilename()
self.filelist.append(x)
self.NameToInfo[x.filename] = x
# update total bytes read from central directory
total = (
total
+ sizeCentralDir
+ centdir[_CD_FILENAME_LENGTH]
+ centdir[_CD_EXTRA_FIELD_LENGTH]
+ centdir[_CD_COMMENT_LENGTH]
)
if self.debug > 2:
print "total", total
def _rename(info: zipfile.ZipInfo) -> None:
"""ヘルパー: `ZipInfo` のファイル名を SJIS でデコードし直す"""
LANG_ENC_FLAG = 0x800
encoding = 'utf-8' if info.flag_bits & LANG_ENC_FLAG else 'cp437'
info.filename = info.filename.encode(encoding).decode('cp932')
