def create_zipinfo(filename, mtime=None, dir=False, executable=False, symlink=False,
comment=None):
"""Create a instance of `ZipInfo`.
:param filename: file name of the entry
:param mtime: modified time of the entry
:param dir: if `True`, the entry is a directory
:param executable: if `True`, the entry is a executable file
:param symlink: if `True`, the entry is a symbolic link
:param comment: comment of the entry
"""
from zipfile import ZipInfo, ZIP_DEFLATED, ZIP_STORED
zipinfo = ZipInfo()
# The general purpose bit flag 11 is used to denote
# UTF-8 encoding for path and comment. Only set it for
# non-ascii files for increased portability.
# See http://www.pkware.com/documents/casestudies/APPNOTE.TXT
if any(ord(c) >= 128 for c in filename):
zipinfo.flag_bits |= 0x0800
zipinfo.filename = filename.encode('utf-8')
if mtime is not None:
mtime = to_datetime(mtime, utc)
zipinfo.date_time = mtime.utctimetuple()[:6]
# The "extended-timestamp" extra field is used for the
# modified time of the entry in unix time. It avoids
# extracting wrong modified time if non-GMT timezone.
# See http://www.opensource.apple.com/source/zip/zip-6/unzip/unzip
# /proginfo/extra.fld
zipinfo.extra += struct.pack(
'<hhBl',
0x5455, # extended-timestamp extra block type
1 + 4, # size of this block
1, # modification time is present
to_timestamp(mtime)) # time of last modification
# external_attr is 4 bytes in size. The high order two
# bytes represent UNIX permission and file type bits,
# while the low order two contain MS-DOS FAT file
# attributes, most notably bit 4 marking directories.
if dir:
if not zipinfo.filename.endswith('/'):
zipinfo.filename += '/'
zipinfo.compress_type = ZIP_STORED
zipinfo.external_attr = 040755 << 16L # permissions drwxr-xr-x
zipinfo.external_attr |= 0x10 # MS-DOS directory flag
else:
zipinfo.compress_type = ZIP_DEFLATED
zipinfo.external_attr = 0644 << 16L # permissions -r-wr--r--
if executable:
zipinfo.external_attr |= 0755 << 16L # -rwxr-xr-x
if symlink:
zipinfo.compress_type = ZIP_STORED
zipinfo.external_attr |= 0120000 << 16L # symlink file type
if comment:
zipinfo.comment = comment.encode('utf-8')
return zipinfo
def _add_path(self, path, version, myzip):
mtime = os.path.getmtime(path)
info = ZipInfo("versions/%d/%s"%(version, path), Archive.unixtime_to_utcziptime(mtime))
info.create_system = 3
info.extra += struct.pack('<HHBl', 0x5455, 5, 1, mtime)
# http://unix.stackexchange.com/questions/14705/the-zip-formats-external-file-attribute
# make mode without file type, which may be system-specific
clean_mode = os.stat(path).st_mode & 0007777
if (os.path.islink(path)):
# set zip file type to link
info.external_attr = (Archive.ZIP_EXT_ATTR_LINK | clean_mode) << 16L
myzip.writestr(info, os.readlink(path))
elif (os.path.isdir(path)):
# set zip file type to dir
info.external_attr = (Archive.ZIP_EXT_ATTR_DIR | clean_mode) << 16L
# dos directory flag
info.external_attr |= 0x10
# it seems we should have a trailing slash for dirs
if not(info.filename.endswith('/')): info.filename = "%s/"%(info.filename)
myzip.writestr(info, '')
for name in os.listdir(path):
self._add_path(os.path.join(path, name), version, myzip)
elif (os.path.isfile(path)):
info.external_attr = (Archive.ZIP_EXT_ATTR_FILE | clean_mode) << 16L
myzip.writestr(info, open(path).read())
else:
raise Exception()
def write_blob(self, path, blob, compression=ZIP_DEFLATED, mode=0644):
"""Add something to the zip without adding to manifest"""
zinfo = ZipInfo(path)
zinfo.external_attr = mode << 16L # set permissions
zinfo.compress_type = compression
zinfo.date_time = self.now
self.zipfile.writestr(zinfo, blob)
def _render_zip(self, req, filename, repos, data):
"""ZIP archive with all the added and/or modified files."""
new_rev = data['new_rev']
req.send_response(200)
req.send_header('Content-Type', 'application/zip')
req.send_header('Content-Disposition',
content_disposition('inline', filename + '.zip'))
from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED
buf = StringIO()
zipfile = ZipFile(buf, 'w', ZIP_DEFLATED)
for old_node, new_node, kind, change in repos.get_changes(
new_path=data['new_path'], new_rev=data['new_rev'],
old_path=data['old_path'], old_rev=data['old_rev']):
if kind == Node.FILE and change != Changeset.DELETE:
assert new_node
zipinfo = ZipInfo()
zipinfo.filename = new_node.path.strip('/').encode('utf-8')
# Note: unicode filenames are not supported by zipfile.
# UTF-8 is not supported by all Zip tools either,
# but as some do, I think UTF-8 is the best option here.
zipinfo.date_time = new_node.last_modified.utctimetuple()[:6]
zipinfo.external_attr = 0644 << 16L # needed since Python 2.5
zipinfo.compress_type = ZIP_DEFLATED
zipfile.writestr(zipinfo, new_node.get_content().read())
zipfile.close()
zip_str = buf.getvalue()
req.send_header("Content-Length", len(zip_str))
req.end_headers()
req.write(zip_str)
raise RequestDone
def add(self, member):
if (member.isdir):
return # FIXME Should be able to add empty directories
info = ZipInfo(member.name)
info.date_time = member.mtime
info.external_attr = member.perm << 16L
self.archive.writestr(info, member.data)
def add_file(self, zf, path, name):
with open(path, 'rb') as f:
raw = f.read()
self.h.update(raw)
zi = ZipInfo(name)
zi.external_attr = 0o444 << 16
zf.writestr(zi, raw)
def handle(self, *args, **options):
options['platform'] = options['platform'].lower() # normalize
if options['platform'] not in ["all", "linux", "macos", "darwin", "windows"]:
raise CommandError("Unrecognized platform: %s; will include ALL files." % options['platform'])
# Step 1: recursively add all static files
kalite_base = os.path.realpath(settings.PROJECT_PATH + "/../")
files_dict = recursively_add_files(dirpath=kalite_base, **options)
# Step 2: Add a local_settings.py file.
# For distributed servers, this is a copy of the local local_settings.py,
# with a few properties (specified as command-line options) overridden
ls_file = create_local_settings_file(location=os.path.realpath(kalite_base+"/kalite/local_settings.py"), server_type=options['server_type'], locale=options['locale'])
files_dict[ls_file] = { "dest_path": "kalite/local_settings.py" }
# Step 3: select output file.
if options['file']=="__default__":
options['file'] = create_default_archive_filename(options)
# Step 4: package into a zip file
with ZipFile(options['file'], "w", ZIP_DEFLATED if options['compress'] else ZIP_STORED) as zfile:
for srcpath,fdict in files_dict.items():
if options['verbosity'] >= 1:
print "Adding to zip: %s" % srcpath
# Add without setting exec perms
if os.path.splitext(fdict["dest_path"])[1] != ".sh":
zfile.write(srcpath, arcname=fdict["dest_path"])
# Add with exec perms
else:
info = ZipInfo(fdict["dest_path"])
info.external_attr = 0755 << 16L # give full access to included file
with open(srcpath, "r") as fh:
zfile.writestr(info, fh.read())
def createZipArchive(self, source, destination, exclude=None):
"""
Create a zip file at `destination` based on files from `source`.
"""
"""
Create a zip file at `destination` based on files from `source`.
"""
if exclude is None:
exclude = []
source_path = self.fs.join(source)
parent_path = os.path.dirname(source_path)
archivename = self.fs.join(destination)
with closing(ZipFile(archivename, 'w', ZIP_DEFLATED)) as z:
for root, dirs, files in os.walk(source_path):
# Write all files.
for fn in files:
if fn in exclude:
continue
absolute_filename = os.path.join(root, fn)
zip_filename = absolute_filename[len(parent_path):]
# See http://bugs.python.org/issue1734346
# for adding unicode support.
z.write(str(absolute_filename), str(zip_filename))
# For empty folders, we need to create a special ZipInfo
# entry.
# 16 works, but some places suggest using 48.
if not files and not dirs:
foldername = root[len(parent_path):] + '/'
zip_info = ZipInfo(foldername)
zip_info.external_attr = 16
z.writestr(zip_info, "")
def compileToZip(self):
"""
Compile the exam as a .zip file
"""
def cleanpath(path):
if path=='':
return ''
dirname, basename = os.path.split(path)
dirname=cleanpath(dirname)
if basename!='.':
dirname = os.path.join(dirname,basename)
return dirname
f = ZipFile(self.options.output,'w')
for (dst,src) in self.files.items():
dst = ZipInfo(cleanpath(dst))
dst.external_attr = 0o644<<16
dst.date_time = datetime.datetime.today().timetuple()
if isinstance(src,basestring):
f.writestr(dst,open(src,'rb').read())
else:
f.writestr(dst,src.read())
print("Exam created in %s" % os.path.relpath(self.options.output))
f.close()
def dump(self, fp):
"""Dump the plugin as package into the filepointer or file."""
from zipfile import ZipFile, ZipInfo
f = ZipFile(fp, 'w')
# write all files into a "pdata/" folder
offset = len(self.path) + 1
for dirpath, dirnames, filenames in walk(self.path):
# don't recurse into hidden dirs
for i in range(len(dirnames)-1, -1, -1):
if dirnames[i].startswith('.'):
del dirnames[i]
for filename in filenames:
if filename.endswith('.pyc') or \
filename.endswith('.pyo'):
continue
f.write(path.join(dirpath, filename),
path.join('pdata', dirpath[offset:], filename))
# add the package information files
for name, data in [('ZINE_PLUGIN', self.name),
('ZINE_PACKAGE', PACKAGE_VERSION)]:
zinfo = ZipInfo(name, localtime(time()))
zinfo.compress_type = f.compression
zinfo.external_attr = (33188 & 0xFFFF) << 16L
f.writestr(zinfo, str(data))
f.close()
def system_specific_zipping(files_dict, zip_file=None, compression=ZIP_DEFLATED, callback=_default_callback_zip):
"""
Zip up files, adding permissions when appropriate.
"""
if not zip_file:
zip_file = tempfile.mkstemp()[1]
zfile = None
try:
zfile = ZipFile(zip_file, 'w', compression)
for fi, (dest_path, src_path) in enumerate(files_dict.iteritems()):
if callback:
callback(src_path, fi, len(files_dict))
# All platforms besides windows need permissions set.
ext = os.path.splitext(dest_path)[1]
if ext not in not_system_specific_scripts(system="windows"):
zfile.write(src_path, arcname=dest_path)
# Add with exec perms
else:
info = ZipInfo(dest_path)
info.external_attr = 0775 << ((1 - is_osx()) * 16L) # give full access to included file
with open(src_path, "r") as fh:
zfile.writestr(info, fh.read())
zfile.close()
finally:
if zfile:
zfile.close()
def add_file(file_path, arcname):
if not path.islink(file_path):
archive.write(file_path, arcname, ZIP_DEFLATED)
else:
i = ZipInfo(arcname)
i.create_system = 3
i.external_attr = 0xA1ED0000
archive.writestr(i, readlink(file_path))
def write(self, filename, arcname=None, compress_type=None):
with open(filename, 'rb') as f:
st = os.fstat(f.fileno())
data = f.read()
zinfo = ZipInfo(arcname or filename, date_time=get_zipinfo_datetime(st.st_mtime))
zinfo.external_attr = st.st_mode << 16
zinfo.compress_type = ZIP_DEFLATED
self.writestr(zinfo, data, compress_type)
def enviar_zip(self, entregas, nombre, extras = None, ignoreFileNames = []):
buffer = StringIO()
zip = ZipFile(buffer, 'w', zipfile.ZIP_DEFLATED)
for e in entregas:
szip = ZipFile(StringIO(e.archivos), 'r')
for file in szip.namelist():
if file not in ignoreFileNames:
zipinfo = ZipInfo('%s_%u/%s' % (e.entregador.alumno.padron.encode('ascii'), e.instancia.numero, file))
zipinfo.external_attr = 0664 << 16L
zip.writestr(zipinfo, szip.read(file))
szip.close()
if extras is not None:
for exk in extras.keys():
zipinfo = ZipInfo('%s' % exk)
zipinfo.external_attr = 0774 << 16L
zip.writestr(zipinfo, extras[exk])
zip.close()
download = Downloader(cherrypy.response)
return download.download_zip(buffer.getvalue(), nombre)
def zip(cls):
now = time.localtime(time.time())
zipio = StringIO()
zipfile = ZipFile(zipio, 'w', ZIP_DEFLATED)
for file in cls.files:
zipinfo = ZipInfo(file.filename, date_time=now)
zipinfo.external_attr = 0644 << 16L
zipfile.writestr(zipinfo, file.getvalue())
zipfile.close()
return zipio
def write_file(filename, data, mode):
if zip:
info = ZipInfo(filename)
info.external_attr = 0o755 << 16
zip_file.writestr(info, data)
else:
f = open(os.path.join(target, filename), mode)
try:
f.write(data)
finally:
f.close()
def close(self):
# Write RECORD
if self.fp is not None and self.mode == 'w' and self._file_hashes:
content = '\n'.join('{},{}={},{}'.format(fname, algorithm, hash_,
self._file_sizes[fname])
for fname, (algorithm, hash_) in self._file_hashes.items())
content += '\n{},,\n'.format(self.record_path)
zinfo = ZipInfo(native(self.record_path), date_time=get_zipinfo_datetime())
zinfo.compress_type = ZIP_DEFLATED
zinfo.external_attr = 0o664 << 16
self.writestr(zinfo, as_bytes(content))
super(WheelFile, self).close()
def write_file (self, data, filename, description = "") : """Write a file into the archive :Parameters: - `data` (str) - data to write - `filename` (str) - name of the file in which to store data - `description` (str) - textual description of the data """ info = ZipInfo(filename) info.comment = description info.date_time = localtime()[:6] info.external_attr = 0644 << 16L info.compress_type = ZIP_DEFLATED self._elms[filename] = (info,data)
def make_dir_entry(name=None, date_time=None, mode=MODE_DIRECTORY):
tt = date_time.timetuple()
dir = ZipInfo()
dir.filename = name+('/' if name[-1] != '/' else '')
dir.orig_filename = dir.filename
dir.date_time = date_time.isocalendar() + (tt.tm_hour,
tt.tm_min, tt.tm_sec)
dir.compress_type = 0
dir.create_system = 0
dir.create_version = 20
dir.extract_version = 10
dir.external_attr = mode
return dir
def make_file_entry(name=None, date_time=None, mode=MODE_FILE | MODE_ARCHIVE):
tt = date_time.timetuple()
file = ZipInfo()
file.filename = name
file.orig_filename = file.filename
file.date_time = date_time.isocalendar() + (tt.tm_hour,
tt.tm_min, tt.tm_sec)
file.compress_type = 8
file.create_system = 0
file.create_version = 20
file.extract_version = 20
file.flag_bits = 2
file.external_attr = mode
return file
def _zip_info(environment, name):
"""
@type environment : C{str}
@param environment : The environment name
@type name : C{str}
@param name : The name of the file
@rtype: C{ZipInfo}
@return: The Zip Info
"""
filename = "%s-%s" % (environment, name)
info = ZipInfo(filename)
info.date_time = time.localtime(time.time())[:6] #now
info.external_attr = 0666 << 16L # read-write access to everyone
info.compress_type = ZIP_DEFLATED
return info
def _inject_settings_python(zip_file, settings):
first = True
content = 'settings = {'
for k, v in settings.iteritems():
if not first:
content += ','
content += '\n \'{}\': \'{}\''.format(k, v)
first = False
content += '\n}'
print 'inserting settings', content
info = ZipInfo('CloudCanvas/settings.py')
info.external_attr = 0777 << 16L # give full access to included file
zip_file.writestr(info, content)
def _inject_settings_nodejs(zip_file, settings):
first = True
content = 'module.exports = {'
for k, v in settings.iteritems():
if not first:
content += ','
content += '\n "{}": "{}"'.format(k, v)
first = False
content += '\n};'
print 'inserting settings', content
info = ZipInfo('CloudCanvas/settings.js')
info.external_attr = 0777 << 16L # give full access to included file
zip_file.writestr(info, content)
def write_strings(path, items, pwd=None, compression=ZIP_DEFLATED):
"""
Write a dictionary of strings to a zip file.
If it already exsits, only append the new
strings or over write what is already there.
"""
old_items = {}
if exists(path):
old_items = read_strings(path)
unlink(path)
old_items.update(items)
with ZipFile(path, mode='w', compression=compression) as zf:
if pwd:
zf.setpassword(pwd)
for name, bytes in items.items():
zi = ZipInfo(name)
zi.external_attr = 0777 << 16L
zf.write(zi, bytes)
def prepare_zip():
from pkg_resources import resource_filename as resource
from config import config
from json import dumps
logger.info('creating/updating gimel.zip')
with ZipFile('gimel.zip', 'w', ZIP_DEFLATED) as zipf:
info = ZipInfo('config.json')
info.external_attr = 0o664 << 16
zipf.writestr(info, dumps(config))
zipf.write(resource('gimel', 'config.py'), 'config.py')
zipf.write(resource('gimel', 'gimel.py'), 'gimel.py')
zipf.write(resource('gimel', 'logger.py'), 'logger.py')
for root, dirs, files in os.walk(resource('gimel', 'vendor')):
for file in files:
real_file = os.path.join(root, file)
relative_file = os.path.relpath(real_file,
resource('gimel', ''))
zipf.write(real_file, relative_file)
def prepare_zip():
from pkg_resources import resource_filename as resource
from config import config
from json import dumps
logger.info("creating/updating gimel.zip")
with ZipFile("gimel.zip", "w", ZIP_DEFLATED) as zipf:
info = ZipInfo("config.json")
info.external_attr = 0o664 << 16
zipf.writestr(info, dumps(config))
zipf.write(resource("gimel", "config.py"), "config.py")
zipf.write(resource("gimel", "gimel.py"), "gimel.py")
zipf.write(resource("gimel", "logger.py"), "logger.py")
for root, dirs, files in os.walk(resource("gimel", "vendor")):
for file in files:
real_file = os.path.join(root, file)
relative_file = os.path.relpath(real_file, resource("gimel", ""))
zipf.write(real_file, relative_file)
def add_object_to_zip(self, obj, parent_path):
target = self.sm.queryAdapter(obj, IZipExportObject)
if target is None:
return
if target.skip:
return
target_path = parent_path + target.filename
if isinstance(target_path, unicode):
# we could convert target_path to utf-8, but that would just mask
# a bug elsewhere; better to raise an exception so we fix the
# original cause.
raise ValueError("All paths must be byte strings!")
data = target.data
if isinstance(data, unicode):
data = data.encode('utf-8')
t = target.timestamp
if t is None:
zipinfo = ZipInfo(target_path)
else:
date_time = (t.year(), t.month(), t.day(),
t.hour(), t.minute(), int(t.second()))
zipinfo = ZipInfo(target_path, date_time)
if target.export_as_folder:
# Set external_attr to 16, otherwise empty folders will not be
# preserved
zipinfo.external_attr = 16
self.zip_file.writestr(zipinfo, data)
index_row = (target.title.encode('utf-8'),
target.meta_label, target_path)
self.index_txt.write('\t'.join(index_row) + '\n')
if target.export_as_folder:
self.recurse(obj, target_path)
def create_zip_archive( zipfilename, path, inarchpath ):
with ZipFile( zipfilename, "w", ZIP_DEFLATED ) as zf:
for root, dirs, files in os.walk(path):
archpath = os.path.join( inarchpath, os.path.relpath( root, path ) )
zf.write( root, archpath )
for f in files:
filename = os.path.join( root, f )
if not os.path.isfile(filename):
continue
archname = os.path.join( archpath, f )
# this hack is needed to avoid leading ./ in the archive
while archname.startswith ('./'):
archname = archname[2:]
zi = ZipInfo( archname)
stat = os.stat( path + '/' + archname )
zi.external_attr = stat.st_mode << 16L
# this hack is needed to use the external attributes
# there is no way to set a zipinfo object directly to an archive
with open (filename, 'rb') as f:
zf.writestr( zi, f.read () )
def pack(args):
with ZipFile(str(args.output), 'w', ZIP_DEFLATED) as zf:
for path in sorted(args.directory.glob('**/*')):
if path.name == 'meson.build':
continue
relpath = path.relative_to(args.directory)
if path.is_dir():
zi = ZipInfo(str(relpath) + "/", datetime.fromtimestamp(path.stat().st_mtime).timetuple())
zi.compress_type = ZIP_STORED
zi.external_attr = 0o40755 << 16 # drwxr-xr-x
zf.writestr(zi, '')
else:
zf.write(str(path), str(relpath))
if args.depfile is not None:
write_depfile(args.depfile, args.output,
[args.directory.resolve() / x for x in zf.namelist()] + [str(Path(__file__).resolve())]
)
def zip_them(str, data_content):
dimap_file = str + 'dim'
dimap_data_dir = str + 'data'
zip_file_name = 'zipped/' + str + 'zip'
if not exists(zip_file_name):
dest_zip_file = ZipFile(zip_file_name, 'w')
print(("\nZip file created: ", zip_file_name, "."))
data_content = glob(dimap_data_dir + '/*') # find content inside the data dir
zip_info = ZipInfo(dimap_data_dir + '/') # create the directory inside
zip_info.external_attr = 0o775 << 16 # the zipfile, set permissions for the directory
zip_info.create_system = 3 # tell the zip file that we are on UNIX
dest_zip_file.writestr(zip_info, '') # write the info to it
print(("Writing to zip: ", dimap_file, "..."))
dest_zip_file.write(dimap_file, dimap_file, ZIP_DEFLATED) # write the dimap file into
for item in data_content:
print(("Writing to zip: ", item, "..."))
dest_zip_file.write(item, item, ZIP_DEFLATED) # write all dimap data to the zip file
dest_zip_file.close()
else:
print(("Zip file ", zip_file_name, " exists already; skipping."))
def zip_add_dir(zipfile, path, prefix=''):
'''
Add a directory recursively to the zip file with an optional prefix.
'''
if prefix:
zi = ZipInfo(prefix+'/')
zi.external_attr = 16
zipfile.writestr(zi, '')
cwd = os.path.abspath(os.getcwd())
try:
os.chdir(path)
fp = (prefix + ('/' if prefix else '')).replace('//', '/')
for f in os.listdir('.'):
arcname = fp + f
if os.path.isdir(f):
zipfile.add_dir(f, prefix=arcname)
else:
zipfile.write(f, arcname)
finally:
os.chdir(cwd)
def create_zip_archive(zipfilename, path, inarchpath):
with ZipFile(zipfilename, "w", ZIP_DEFLATED) as zf:
for root, dirs, files in os.walk(path):
archpath = os.path.join(inarchpath, os.path.relpath(root, path))
zf.write(root, archpath)
for f in files:
filename = os.path.join(root, f)
if not os.path.isfile(filename):
continue
archname = os.path.join(archpath, f)
# this hack is needed to avoid leading ./ in the archive
while archname.startswith('./'):
archname = archname[2:]
zi = ZipInfo(archname)
stat = os.stat(path + '/' + archname)
zi.external_attr = stat.st_mode << 16L
# this hack is needed to use the external attributes
# there is no way to set a zipinfo object directly to an archive
with open(filename, 'rb') as f:
zf.writestr(zi, f.read())
def close(self):
# Write RECORD
if self.fp is not None and self.mode == 'w' and self._file_hashes:
data = StringIO()
writer = csv.writer(data, delimiter=',', quotechar='"', lineterminator='\n')
writer.writerows((
(
fname,
algorithm + "=" + hash_,
self._file_sizes[fname]
)
for fname, (algorithm, hash_) in self._file_hashes.items()
))
writer.writerow((format(self.record_path), "", ""))
zinfo = ZipInfo(native(self.record_path), date_time=get_zipinfo_datetime())
zinfo.compress_type = self.compression
zinfo.external_attr = 0o664 << 16
self.writestr(zinfo, as_bytes(data.getvalue()))
ZipFile.close(self)
def compress_string(self, arcname, data):
if isinstance(data, str):
data = data.encode("utf-8")
zinfo = ZipInfo(filename=arcname,
date_time=time.localtime(time.time())[:6])
zinfo.compress_type = ZIP_DEFLATED
zinfo.external_attr = 0o600 << 16
zinfo.file_size = len(data)
zinfo.header_offset = self._data_p
zinfo.CRC = zlib.crc32(data) & 0xffffffff
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15)
if cmpr:
data = cmpr.compress(data) + cmpr.flush()
zinfo.compress_size = len(data)
else:
zinfo.compress_size = zinfo.file_size
zip64 = zinfo.file_size > ZIP64_LIMIT or zinfo.compress_size > ZIP64_LIMIT
yield self.__get_data(zinfo.FileHeader(zip64))
yield self.__get_data(data)
self._filelist.append(zinfo)
def compileToZip(self):
"""
Compile the exam as a .zip file
"""
Path(self.options.output).parent.mkdir(exist_ok=True, parents=True)
f = ZipFile(self.options.output, 'w')
for (dst, src) in self.files.items():
dst = ZipInfo(str(Path(dst).relative_to('.')))
dst.compress_type = zipfile.ZIP_DEFLATED
dst.external_attr = 0o644<<16
dst.date_time = datetime.datetime.today().timetuple()
if isinstance(src, Path):
f.writestr(dst, src.read_bytes())
else:
f.writestr(dst, src.read())
print("Exam created in %s" % os.path.relpath(self.options.output))
f.close()
def add_str(self, str_to_add, name, dt=datetime.now()):
# type: (str,str,datetime) -> None
"""
Add a string to the archive as zip entry named 'name'
:param str_to_add: string to add
:param name: name of the zip.entry
:param dt: datetime, optional if not specified, current date time is assumed
:return: None
"""
# always use forward slash regardless of platform, this allows the calling
# code to use os.path.join for names
if os.pathsep in name:
name = name.replace(os.pathsep, "/")
info = ZipInfo()
info.filename = self.uuid + "/" + name
info.external_attr = 0o644 << 16
info.compress_type = ZIP_DEFLATED
info.date_time = (dt.year, dt.month, dt.day, dt.hour, dt.minute,
dt.second)
self.zip.writestr(info, str_to_add)
def compress_file(self, filename, arcname):
st = os.stat(filename)
zinfo = ZipInfo(arcname, time.localtime(time.time())[:6])
zinfo.external_attr = (st[0] & 0xFFFF) << 16
zinfo.compress_type = ZIP_DEFLATED
zinfo.flag_bits = 0x08
zinfo.header_offset = self._data_p
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15)
with open(filename, "rb") as fp:
zinfo.CRC = crc = 0
zinfo.compress_size = 0
zinfo.file_size = 0
yield self.__get_data(zinfo.FileHeader())
while 1:
buf = fp.read(CHUNK_SIZE)
if not buf:
break
zinfo.file_size += len(buf)
crc = zlib.crc32(buf, crc) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
zinfo.compress_size += len(buf)
yield self.__get_data(buf)
if cmpr:
buf = cmpr.flush()
zinfo.compress_size += len(buf)
yield self.__get_data(buf)
else:
zinfo.compress_size = zinfo.file_size
zinfo.CRC = crc
zip64 = zinfo.file_size > ZIP64_LIMIT or zinfo.compress_size > ZIP64_LIMIT
fmt = '<4sLQQ' if zip64 else '<4sLLL'
data_descriptor = struct.pack(fmt, stringDataDescriptor, zinfo.CRC,
zinfo.compress_size, zinfo.file_size)
yield self.__get_data(data_descriptor)
self._filelist.append(zinfo)
def create_archive(tiles, root, max_zoom, meta, ext):
# expand bounds
roots = generate_tiles(root, root.z, meta.get("metatile", 1))
min_x, min_y, max_x, max_y = mercantile.bounds(root)
for r in roots:
l_min_x, l_min_y, l_max_x, l_max_y = mercantile.bounds(r)
min_x = min(min_x, l_min_x)
max_x = max(max_x, l_max_x)
min_y = min(min_y, l_min_y)
max_y = max(max_y, l_max_y)
meta["minzoom"] = root.z
meta["maxzoom"] = max_zoom
meta["bounds"] = [min_x, min_y, max_x, max_y]
meta["root"] = "{}/{}/{}".format(root.z, root.x, root.y)
date_time = gmtime()[0:6]
out = BytesIO()
with ZipFile(out, "w", ZIP_DEFLATED, allowZip64=True) as archive:
archive.comment = json.dumps(meta).encode("utf-8")
for tile, (_, data) in tiles:
if data is None:
logger.warn("Not writing tile %d/%d/%d because it has no data",
tile.z, tile.x, tile.y)
continue
logger.debug("Writing tile %d/%d/%d", tile.z, tile.x, tile.y)
info = ZipInfo(
"{}/{}/{}@2x.{}".format(tile.z, tile.x, tile.y, ext),
date_time)
info.external_attr = 0o755 << 16
archive.writestr(info, data, ZIP_DEFLATED)
return out.getvalue()
def compress_stream(self, arcname, datagen):
zinfo = ZipInfo(arcname, time.localtime(time.time())[:6])
zinfo.external_attr = 0o600 << 16
zinfo.compress_type = ZIP_DEFLATED
zinfo.flag_bits = 0x08
zinfo.header_offset = self._data_p
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15)
zinfo.CRC = crc = 0
zinfo.compress_size = 0
zinfo.file_size = 0
yield self.__get_data(zinfo.FileHeader())
for buf in datagen:
if not buf:
continue
zinfo.file_size += len(buf)
crc = zlib.crc32(buf, crc) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
zinfo.compress_size += len(buf)
yield self.__get_data(buf)
if cmpr:
buf = cmpr.flush()
zinfo.compress_size += len(buf)
yield self.__get_data(buf)
else:
zinfo.compress_size = zinfo.file_size
zinfo.CRC = crc
zip64 = zinfo.file_size > ZIP64_LIMIT
fmt = '<4sLQQ' if zip64 else '<4sLLL'
data_descriptor = struct.pack(fmt, stringDataDescriptor, zinfo.CRC,
zinfo.compress_size, zinfo.file_size)
yield self.__get_data(data_descriptor)
self._filelist.append(zinfo)
def zip_them(str, data_content):
dimap_file = str + 'dim'
dimap_data_dir = str + 'data'
zip_file_name = 'zipped/' + str + 'zip'
if not exists(zip_file_name):
dest_zip_file = ZipFile(zip_file_name, 'w')
print(("\nZip file created: ", zip_file_name, "."))
data_content = glob(dimap_data_dir +
'/*') # find content inside the data dir
zip_info = ZipInfo(dimap_data_dir + '/') # create the directory inside
zip_info.external_attr = 0o775 << 16 # the zipfile, set permissions for the directory
zip_info.create_system = 3 # tell the zip file that we are on UNIX
dest_zip_file.writestr(zip_info, '') # write the info to it
print(("Writing to zip: ", dimap_file, "..."))
dest_zip_file.write(dimap_file, dimap_file,
ZIP_DEFLATED) # write the dimap file into
for item in data_content:
print(("Writing to zip: ", item, "..."))
dest_zip_file.write(
item, item,
ZIP_DEFLATED) # write all dimap data to the zip file
dest_zip_file.close()
else:
print(("Zip file ", zip_file_name, " exists already; skipping."))
def compileToZip(exam, files, options):
def cleanpath(path):
if path == '':
return ''
dirname, basename = os.path.split(path)
dirname = cleanpath(dirname)
if basename != '.':
dirname = os.path.join(dirname, basename)
return dirname
f = ZipFile(options.output, 'w')
for (dst, src) in files.items():
dst = ZipInfo(cleanpath(dst))
dst.external_attr = 0o644 << 16
dst.date_time = datetime.datetime.today().timetuple()
if isinstance(src, basestring):
f.writestr(dst, open(src, 'rb').read())
else:
f.writestr(dst, src.read())
print("Exam created in %s" % os.path.relpath(options.output))
f.close()
def upload(self, name, expansions=None):
# Load and customize script:
script = self._load(name)
if expansions:
for key, value in expansions.items():
script = script.replace(key, value)
encoded_script = script.encode('utf-8')
# Hash script
script_hash = self._hash(encoded_script)
# Package into zipfile:
zip_buffer = BytesIO()
with ZipFile(zip_buffer, 'w') as zip_file:
zip_info = ZipInfo('index.js')
zip_info.compress_type = ZIP_DEFLATED
zip_info.external_attr = 0o0755 << 16
zip_file.writestr(zip_info, encoded_script)
zip_buffer.seek(0)
# Upload to S3:
zip_path = '%s/%s.zip' % (name, script_hash)
self._upload(zip_path, zip_buffer)
return self._bucket, zip_path
def write(self, filename, arcname=None, compress_type=None):
"""Put the bytes from filename into the archive under the name
arcname."""
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16L # Unix attributes
if self.compression == ZIP_AUTO:
ext = os.path.splitext(filename)[1].lower()
compression = ZIP_STORED if ext and ext[1:] in STORED_FORMATS \
else ZIP_DEFLATED
else:
compression = self.compression
if compress_type is None:
zinfo.compress_type = compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits |= 0x08
zinfo.header_offset = self.tell # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
header = zinfo.FileHeader()
yield header
self.tell += len(header)
return
fp = open(filename, "rb")
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
zinfo.file_size = file_size = 0
header = zinfo.FileHeader()
yield header
self.tell += len(header)
if zinfo.compress_type == ZIP_DEFLATED:
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
else:
cmpr = None
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
yield buf
fp.close()
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
yield buf
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
self.tell += zinfo.compress_size
zinfo.CRC = CRC
zinfo.file_size = file_size
# write the data descriptor
data_descriptor = struct.pack("<LLL", zinfo.CRC, zinfo.compress_size,
zinfo.file_size)
yield data_descriptor
self.tell += len(data_descriptor)
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def write_iter(self, filename, arcname=None, compress_type=None):
"""Put the bytes from filename into the archive under the name
arcname."""
from zipfile import stat, time, ZipInfo, ZIP_DEFLATED, crc32, struct
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16L # Unix attributes
if compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x00
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
yield zinfo.FileHeader()
#self.fp.write(zinfo.FileHeader())
return
with open(filename, "rb") as fp:
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
zinfo.file_size = file_size = 0
cmpr = None
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
zinfo.compress_size = file_size
zinfo.file_size = file_size
zinfo.CRC = CRC
#self.fp.write(zinfo.FileHeader())
yield zinfo.FileHeader()
fp.seek(0)
while 1:
buf = fp.read(1024 * 8)
if not buf:
break
#self.fp.write(buf)
yield buf
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def pack(args):
nocompress_file = args.directory / '.nocompress'
if 1:
comp_type = ZIP_ZSTANDARD
comp_level = 20
else:
comp_type = ZIP_DEFLATED
comp_level = 9
try:
nocompress = list(
map(re.compile,
filter(None,
nocompress_file.read_text().strip().split('\n'))))
except FileNotFoundError:
nocompress = []
nocompress_file = None
zkwargs = {}
if (sys.version_info.major, sys.version_info.minor) >= (3, 7):
zkwargs['compresslevel'] = comp_level
dependencies = []
with ZipFile(str(args.output), 'w', comp_type, **zkwargs) as zf:
for path in sorted(args.directory.glob('**/*')):
if path.name[0] == '.' or any(path.match(x) for x in args.exclude):
continue
relpath = path.relative_to(args.directory)
if path.is_dir():
zi = ZipInfo(
str(relpath) + "/",
datetime.fromtimestamp(path.stat().st_mtime).timetuple())
zi.compress_type = ZIP_STORED
zi.external_attr = 0o40755 << 16 # drwxr-xr-x
log_file(path, zi.filename)
zf.writestr(zi, '')
else:
dependencies.append(path)
if path.suffix == '.zst':
write_zst_file(zf, path, str(relpath.with_suffix('')))
else:
ctype = comp_type
for pattern in nocompress:
if pattern.match(str(relpath)):
ctype = ZIP_STORED
break
log_file(path, relpath, ctype)
zf.write(str(path), str(relpath), compress_type=ctype)
if args.depfile is not None:
if nocompress_file is not None:
dependencies.append(nocompress_file)
dependencies.append(Path(__file__).resolve())
write_depfile(args.depfile, args.output, dependencies)
def test_subnet_connectivity(region,
stackid,
logical_resource_id,
physical_resource_id,
endpoints=[['www.amazon.com', "80"]]):
logger.debug({"test_subnet_connectivity": "starting"})
error_msg = []
if region not in clients.get_available_regions('lambda'):
msg = "Test for %s %s skipped, %s not supprted by lambda" % (
stackid, logical_resource_id, region)
logger.warning(msg)
return {
"success": True,
"logical_resource_id": logical_resource_id,
"physical_resource_id": physical_resource_id,
"warning":
"Test skipped, region %s not supprted by lambda" % region,
"region": region,
"stackid": stackid
}
try:
function_name = 'test_subnet_%s_%s' % (physical_resource_id,
random_string(8))
iam_name = function_name.replace('_', '-')
lambda_client = clients.get("lambda", region=region)
ec2_client = clients.get("ec2", region=region)
iam_role_arn = get_iam_role()
response = ec2_client.describe_subnets(
SubnetIds=[physical_resource_id])
logger.debug({"test_subnet_connectivity:describe_subnets": response})
vpc_id = response['Subnets'][0]['VpcId']
logger.debug({"test_subnet_connectivity:vpc_id": vpc_id})
security_group_id = ec2_client.create_security_group(
GroupName=iam_name, Description=iam_name, VpcId=vpc_id)['GroupId']
logger.debug(
{"test_subnet_connectivity:security_group_id": security_group_id})
now = datetime.now()
zi_timestamp = (now.year, now.month, now.day, now.hour, now.minute,
now.second)
zinfo = ZipInfo('lambda_function.py', zi_timestamp)
zinfo.external_attr = 0x0744 << 16
f = StringIO()
z = ZipFile(f, 'w', ZIP_DEFLATED)
z.writestr(zinfo, function_code)
z.close()
zip_bytes = f.getvalue()
logger.debug({
"test_subnet_connectivity:create_function_input": {
"FunctionName": function_name,
"Role": iam_role_arn,
"Code": {
'ZipFile': zip_bytes
},
"VpcConfig": {
'SubnetIds': [physical_resource_id],
'SecurityGroupIds': [security_group_id]
}
}
})
lambda_function = False
retries = 0
max_retries = 4
while not lambda_function:
try:
lambda_function = lambda_client.create_function(
FunctionName=function_name,
Runtime='python2.7',
Role=iam_role_arn,
Handler='lambda_function.lambda_handler',
Code={'ZipFile': zip_bytes},
Timeout=120,
MemorySize=128,
VpcConfig={
'SubnetIds': [physical_resource_id],
'SecurityGroupIds': [security_group_id]
})
except botocore.exceptions.ClientError as e:
codes = [
'InvalidParameterValueException', 'AccessDeniedException'
]
logger.debug("boto exception: ", exc_info=1)
logger.debug(e.response)
if "The provided subnets contain availability zone Lambda doesn't support." in e.response[
'Error']['Message']:
raise
if e.response['Error'][
'Code'] in codes and retries < max_retries:
logger.debug(
{"test_subnet_connectivity:create_function": str(e)},
exc_info=1)
msg = "role not propagated yet, sleeping a bit and then retrying"
logger.debug({
"test_subnet_connectivity:create_function_retry":
msg
})
retries += 1
sleep(10 * (retries**2))
else:
raise
for endpoint in endpoints:
f = StringIO()
f.write(json.dumps({"address": endpoint[0], "port": endpoint[1]}))
payload = f.getvalue()
f.close()
response = lambda_client.invoke(FunctionName=function_name,
InvocationType='RequestResponse',
Payload=payload)
response['Payload'] = response['Payload'].read()
try:
response['Payload'] = json.loads(response['Payload'])
except Exception:
pass
logger.debug({"test_subnet_connectivity:response": response})
if response[
'StatusCode'] != 200 or 'FunctionError' in response.keys():
results = {
"success": False,
"logical_resource_id": logical_resource_id,
"physical_resource_id": physical_resource_id,
"region": region,
"stackid": stackid
}
error_msg.append({
"endpoint": endpoint,
"response": response['Payload']
})
elif response['StatusCode'] == 200 and len(error_msg) == 0:
results = {
"success": True,
"logical_resource_id": logical_resource_id,
"physical_resource_id": physical_resource_id,
"region": region,
"stackid": stackid
}
except Exception as e:
logger.error({"test_subnet_connectivity": str(e)}, exc_info=1)
if "subnets contain availability zone Lambda doesn't support" in str(
e):
results = {
"success": True,
"logical_resource_id": logical_resource_id,
"physical_resource_id": physical_resource_id,
"region": region,
"stackid": stackid
}
logger.warning(
"test skipped as lambda is not supported in the subnet's az. %s"
% str(results))
else:
results = {
"success": False,
"logical_resource_id": logical_resource_id,
"physical_resource_id": physical_resource_id,
"region": region,
"stackid": stackid
}
error_msg.append({"exception": str(e)})
finally:
try:
lambda_client.delete_function(FunctionName=function_name)
except Exception:
logger.warning("Failed to cleanup lambda function", exc_info=1)
try:
logger.debug({
"test_subnet_connectivity:security_group_id":
security_group_id
})
enis = ec2_client.describe_network_interfaces(
Filters=[{
'Name': 'group-id',
'Values': [security_group_id]
}])
for eni in enis['NetworkInterfaces']:
if 'Attachment' in eni.keys():
logger.debug("Detaching ENI...")
ec2_client.detach_network_interface(
AttachmentId=eni['Attachment']['AttachmentId'])
while 'Attachment' in ec2_client.describe_network_interfaces(
NetworkInterfaceIds=[eni['NetworkInterfaceId']]
)['NetworkInterfaces'][0].keys():
logger.debug(
"eni still attached, waiting 5 seconds...")
sleep(5)
logger.debug("Deleting ENI %s" % eni['NetworkInterfaceId'])
ec2_client.delete_network_interface(
NetworkInterfaceId=eni['NetworkInterfaceId'])
sg = False
retries = 0
max_retries = 3
while not sg:
try:
sg = ec2_client.delete_security_group(
GroupId=security_group_id)
except botocore.exceptions.ClientError as e:
msg = "has a dependent object"
dep_violation = e.response['Error'][
'Code'] == 'DependencyViolation'
logger.debug("boto exception: ", exc_info=1)
if dep_violation and msg in str(
e) and retries < max_retries:
msg = "eni deletion not propagated yet, sleeping a bit and then retrying"
logger.debug({
"test_subnet_connectivity:delete_sg_retry":
security_group_id
})
retries += 1
sleep(5 * (retries**2))
else:
raise
logger.debug({
"test_subnet_connectivity:security_group_id_response":
response
})
except Exception:
logger.warning("Failed to cleanup security group", exc_info=1)
if len(error_msg) > 0:
results["error_msg"] = error_msg
return results
def decryptBook(userkey, inpath, outpath):
if AES is None:
raise ADEPTError(u"PyCrypto or OpenSSL must be installed.")
rsa = RSA(userkey)
with closing(ZipFile(open(inpath, 'rb'))) as inf:
namelist = set(inf.namelist())
if 'META-INF/rights.xml' not in namelist or \
'META-INF/encryption.xml' not in namelist:
print u"{0:s} is DRM-free.".format(os.path.basename(inpath))
return 1
for name in META_NAMES:
namelist.remove(name)
try:
rights = etree.fromstring(inf.read('META-INF/rights.xml'))
adept = lambda tag: '{%s}%s' % (NSMAP['adept'], tag)
expr = './/%s' % (adept('encryptedKey'),)
bookkey = ''.join(rights.findtext(expr))
if len(bookkey) != 172:
print u"{0:s} is not a secure Adobe Adept ePub.".format(os.path.basename(inpath))
return 1
bookkey = rsa.decrypt(bookkey.decode('base64'))
# Padded as per RSAES-PKCS1-v1_5
if bookkey[-17] != '\x00':
print u"Could not decrypt {0:s}. Wrong key".format(os.path.basename(inpath))
return 2
encryption = inf.read('META-INF/encryption.xml')
decryptor = Decryptor(bookkey[-16:], encryption)
kwds = dict(compression=ZIP_DEFLATED, allowZip64=False)
with closing(ZipFile(open(outpath, 'wb'), 'w', **kwds)) as outf:
zi = ZipInfo('mimetype')
zi.compress_type=ZIP_STORED
try:
# if the mimetype is present, get its info, including time-stamp
oldzi = inf.getinfo('mimetype')
# copy across fields to be preserved
zi.date_time = oldzi.date_time
zi.comment = oldzi.comment
zi.extra = oldzi.extra
zi.internal_attr = oldzi.internal_attr
# external attributes are dependent on the create system, so copy both.
zi.external_attr = oldzi.external_attr
zi.create_system = oldzi.create_system
except:
pass
outf.writestr(zi, inf.read('mimetype'))
for path in namelist:
data = inf.read(path)
zi = ZipInfo(path)
zi.compress_type=ZIP_DEFLATED
try:
# get the file info, including time-stamp
oldzi = inf.getinfo(path)
# copy across useful fields
zi.date_time = oldzi.date_time
zi.comment = oldzi.comment
zi.extra = oldzi.extra
zi.internal_attr = oldzi.internal_attr
# external attributes are dependent on the create system, so copy both.
zi.external_attr = oldzi.external_attr
zi.create_system = oldzi.create_system
except:
pass
outf.writestr(zi, decryptor.decrypt(path, data))
except:
print u"Could not decrypt {0:s} because of an exception:\n{1:s}".format(os.path.basename(inpath), traceback.format_exc())
return 2
return 0
def write(self, filename, arcname=None, compress_type=None):
"""
Fixed version of write supporting bitflag 0x08 to write crc and size
at end of file.
"""
if not self.fp:
raise RuntimeError(
"Attempt to write to ZIP archive that was already closed")
st = os.stat(filename)
isdir = stat.S_ISDIR(st.st_mode)
mtime = time.localtime(st.st_mtime)
date_time = mtime[0:6]
# Create ZipInfo instance to store file information
if arcname is None:
arcname = filename
arcname = os.path.normpath(os.path.splitdrive(arcname)[1])
while arcname[0] in (os.sep, os.altsep):
arcname = arcname[1:]
if isdir:
arcname += '/'
zinfo = ZipInfo(arcname, date_time)
zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes
if isdir:
zinfo.compress_type = ZIP_STORED
elif compress_type is None:
zinfo.compress_type = self.compression
else:
zinfo.compress_type = compress_type
zinfo.file_size = st.st_size
zinfo.flag_bits = 0x00
zinfo.header_offset = self.fp.tell() # Start of header bytes
self._writecheck(zinfo)
self._didModify = True
if isdir:
zinfo.file_size = 0
zinfo.compress_size = 0
zinfo.CRC = 0
zinfo.external_attr |= 0x10 # MS-DOS directory flag
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
self.fp.write(zinfo.FileHeader())
self.start_dir = self.fp.tell()
return
zinfo.flag_bits |= 0x08
with open(filename, "rb") as fp:
# Must overwrite CRC and sizes with correct data later
zinfo.CRC = CRC = 0
zinfo.compress_size = compress_size = 0
try:
# Python > 2.7.3
# Compressed size can be larger than uncompressed size
zip64 = self._allowZip64 and \
zinfo.file_size * 1.05 > ZIP64_LIMIT
self.fp.write(zinfo.FileHeader(zip64))
except TypeError:
# Python <= 2.7.3
zip64 = zinfo.file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT
self.fp.write(zinfo.FileHeader())
if zinfo.compress_type == ZIP_DEFLATED:
cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
zlib.DEFLATED, -15)
else:
cmpr = None
file_size = 0
while 1:
buf = fp.read(CHUNK_SIZE)
if not buf:
break
file_size = file_size + len(buf)
CRC = crc32(buf, CRC) & 0xffffffff
if cmpr:
buf = cmpr.compress(buf)
compress_size = compress_size + len(buf)
self.fp.write(buf)
if cmpr:
buf = cmpr.flush()
compress_size = compress_size + len(buf)
self.fp.write(buf)
zinfo.compress_size = compress_size
else:
zinfo.compress_size = file_size
zinfo.CRC = CRC
zinfo.file_size = file_size
if not zip64 and self._allowZip64:
if file_size > ZIP64_LIMIT:
raise RuntimeError('File size has increased during compressing')
if compress_size > ZIP64_LIMIT:
raise RuntimeError('Compressed size larger than uncompressed size')
# Write CRC and file sizes after the file data
fmt = b'<LQQ' if zip64 else b'<LLL'
self.fp.write(struct.pack(fmt, zinfo.CRC, zinfo.compress_size,
zinfo.file_size))
self.start_dir = self.fp.tell()
self.filelist.append(zinfo)
self.NameToInfo[zinfo.filename] = zinfo
def write_blob(self, filename, blob, compression=ZIP_DEFLATED, mode=0644):
"""Add something to the zip without adding to manifest"""
zinfo = ZipInfo(filename)
zinfo.external_attr = mode << 16L # set permissions
zinfo.compress_type = compression
self.zipfile.writestr(zinfo, blob)
def write_folder(z, path):
info = ZipInfo(path)
info.external_attr = 16
z.writestr(info, b'')
def decryptFontsBook(inpath, outpath):
with closing(ZipFile(open(inpath, 'rb'))) as inf:
namelist = inf.namelist()
if 'META-INF/encryption.xml' not in namelist:
return 1
# Font key handling:
font_master_key = None
adobe_master_encryption_key = None
contNS = lambda tag: '{%s}%s' % (
'urn:oasis:names:tc:opendocument:xmlns:container', tag)
path = None
try:
container = etree.fromstring(inf.read("META-INF/container.xml"))
rootfiles = container.find(contNS("rootfiles")).findall(
contNS("rootfile"))
for rootfile in rootfiles:
path = rootfile.get("full-path", None)
if (path is not None):
break
except:
pass
# If path is None, we didn't find an OPF, so we probably don't have a font key.
# If path is set, it's the path to the main content OPF file.
if (path is None):
print("FontDecrypt: No OPF for font obfuscation found")
return 1
else:
packageNS = lambda tag: '{%s}%s' % ('http://www.idpf.org/2007/opf',
tag)
metadataDCNS = lambda tag: '{%s}%s' % (
'http://purl.org/dc/elements/1.1/', tag)
try:
container = etree.fromstring(inf.read(path))
except:
container = []
## IETF font key algorithm:
print(
"FontDecrypt: Checking {0} for IETF font obfuscation keys ... "
.format(path),
end='')
secret_key_name = None
try:
secret_key_name = container.get("unique-identifier")
except:
pass
try:
identify_element = container.find(packageNS("metadata")).find(
metadataDCNS("identifier"))
if (secret_key_name is None
or secret_key_name == identify_element.get("id")):
font_master_key = identify_element.text
except:
pass
if (font_master_key is not None):
if (secret_key_name is None):
print("found '%s'" % (font_master_key))
else:
print("found '%s' (%s)" %
(font_master_key, secret_key_name))
# Trim / remove forbidden characters from the key, then hash it:
font_master_key = font_master_key.replace(' ', '')
font_master_key = font_master_key.replace('\t', '')
font_master_key = font_master_key.replace('\r', '')
font_master_key = font_master_key.replace('\n', '')
font_master_key = font_master_key.encode('utf-8')
font_master_key = hashlib.sha1(font_master_key).digest()
else:
print("not found")
## Adobe font key algorithm
print(
"FontDecrypt: Checking {0} for Adobe font obfuscation keys ... "
.format(path),
end='')
try:
metadata = container.find(packageNS("metadata"))
identifiers = metadata.findall(metadataDCNS("identifier"))
uid = None
uidMalformed = False
for identifier in identifiers:
if identifier.get(packageNS("scheme")) == "UUID":
if identifier.text[:9] == "urn:uuid:":
uid = identifier.text[9:]
else:
uid = identifier.text
break
if identifier.text[:9] == "urn:uuid:":
uid = identifier.text[9:]
break
if uid is not None:
uid = uid.replace(chr(0x20), '').replace(chr(0x09), '')
uid = uid.replace(chr(0x0D),
'').replace(chr(0x0A),
'').replace('-', '')
if len(uid) < 16:
uidMalformed = True
if not all(c in "0123456789abcdefABCDEF" for c in uid):
uidMalformed = True
if not uidMalformed:
print("found '{0}'".format(uid))
uid = uid + uid
adobe_master_encryption_key = binascii.unhexlify(
uid[:32])
if adobe_master_encryption_key is None:
print("not found")
except:
print("exception")
pass
# Begin decrypting.
try:
encryption = inf.read('META-INF/encryption.xml')
decryptor = Decryptor(font_master_key, adobe_master_encryption_key,
encryption)
kwds = dict(compression=ZIP_DEFLATED, allowZip64=False)
with closing(ZipFile(open(outpath, 'wb'), 'w', **kwds)) as outf:
# Mimetype needs to be the first entry, so remove it from the list
# whereever it is, then add it at the beginning.
namelist.remove("mimetype")
for path in (["mimetype"] + namelist):
data = inf.read(path)
zi = ZipInfo(path)
zi.compress_type = ZIP_DEFLATED
if path == "mimetype":
# mimetype must not be compressed
zi.compress_type = ZIP_STORED
elif path == "META-INF/encryption.xml":
# Check if there's still other entries not related to fonts
if (decryptor.check_if_remaining()):
data = decryptor.get_xml()
print(
"FontDecrypt: There's remaining entries in encryption.xml, adding file ..."
)
else:
# No remaining entries, no need for that file.
continue
try:
# get the file info, including time-stamp
oldzi = inf.getinfo(path)
# copy across useful fields
zi.date_time = oldzi.date_time
zi.comment = oldzi.comment
zi.extra = oldzi.extra
zi.internal_attr = oldzi.internal_attr
# external attributes are dependent on the create system, so copy both.
zi.external_attr = oldzi.external_attr
zi.create_system = oldzi.create_system
if any(ord(c) >= 128 for c in path) or any(
ord(c) >= 128 for c in zi.comment):
# If the file name or the comment contains any non-ASCII char, set the UTF8-flag
zi.flag_bits |= 0x800
except:
pass
if path == "mimetype":
outf.writestr(zi, inf.read('mimetype'))
elif path == "META-INF/encryption.xml":
outf.writestr(zi, data)
else:
outf.writestr(zi, decryptor.decrypt(path, data))
except:
print(
"FontDecrypt: Could not decrypt fonts in {0:s} because of an exception:\n{1:s}"
.format(os.path.basename(inpath), traceback.format_exc()))
traceback.print_exc()
return 2
return 0
def add_to_zip(file_obj, z_file):
"""Add the file to the zip and sets the attributes."""
zinfo = ZipInfo(file_obj['name'], file_obj['time'])
zinfo.compress_type = z_file.compression
zinfo.external_attr = 0o644 << 16
z_file.writestr(zinfo, file_obj['data'])
def create_zipinfo(filename,
mtime=None,
dir=False,
executable=False,
symlink=False,
comment=None):
"""Create a instance of `ZipInfo`.
:param filename: file name of the entry
:param mtime: modified time of the entry
:param dir: if `True`, the entry is a directory
:param executable: if `True`, the entry is a executable file
:param symlink: if `True`, the entry is a symbolic link
:param comment: comment of the entry
"""
from zipfile import ZipInfo, ZIP_DEFLATED, ZIP_STORED
zipinfo = ZipInfo()
# The general purpose bit flag 11 is used to denote
# UTF-8 encoding for path and comment. Only set it for
# non-ascii files for increased portability.
# See http://www.pkware.com/documents/casestudies/APPNOTE.TXT
if any(ord(c) >= 128 for c in filename):
zipinfo.flag_bits |= 0x0800
zipinfo.filename = filename.encode('utf-8')
if mtime is not None:
mtime = to_datetime(mtime, utc)
zipinfo.date_time = mtime.utctimetuple()[:6]
# The "extended-timestamp" extra field is used for the
# modified time of the entry in unix time. It avoids
# extracting wrong modified time if non-GMT timezone.
# See http://www.opensource.apple.com/source/zip/zip-6/unzip/unzip
# /proginfo/extra.fld
zipinfo.extra += struct.pack(
'<hhBl',
0x5455, # extended-timestamp extra block type
1 + 4, # size of this block
1, # modification time is present
to_timestamp(mtime)) # time of last modification
# external_attr is 4 bytes in size. The high order two
# bytes represent UNIX permission and file type bits,
# while the low order two contain MS-DOS FAT file
# attributes, most notably bit 4 marking directories.
if dir:
if not zipinfo.filename.endswith('/'):
zipinfo.filename += '/'
zipinfo.compress_type = ZIP_STORED
zipinfo.external_attr = 040755 << 16L # permissions drwxr-xr-x
zipinfo.external_attr |= 0x10 # MS-DOS directory flag
else:
zipinfo.compress_type = ZIP_DEFLATED
zipinfo.external_attr = 0644 << 16L # permissions -r-wr--r--
if executable:
zipinfo.external_attr |= 0755 << 16L # -rwxr-xr-x
if symlink:
zipinfo.compress_type = ZIP_STORED
zipinfo.external_attr |= 0120000 << 16L # symlink file type
if comment:
zipinfo.comment = comment.encode('utf-8')
return zipinfo
def mkdir(self, filename, arcname):
st = os.stat(filename)
zinfo = ZipInfo(arcname + '/', date_time=get_zipinfo_datetime(st.st_mtime))
zinfo.external_attr = st.st_mode << 16
zinfo.compress_type = ZIP_DEFLATED
self.writestr(zinfo, b'')
def _add_entry(self, name: str, external_attr: int, mtime: int,
data: Union[bytes, str]) -> None:
info = ZipInfo(name, self._prepare_zipinfo_date_time(mtime))
info.external_attr = external_attr
self._set_zipinfo_compression(info)
self._inner.writestr(info, data)
encoding='UTF-8'
) as outfile: # Storing the initial config in the 'Input' folder.
outfile.write(python_code)
shutil.copy(
path_prefix + '/Backup/PROD/Lambda/Code/' + df['Lambda'][index] +
".py", config['Python_file_name']
) # Moves and renames the python code file as 'lambda_function.py'
file = open(config['Python_file_name'], 'rb')
zip_code = file.read()
zip_file_name = path_prefix + config['Zip_Code_prefix'] + df['Lambda'][
index] + '.zip' # Creates the zip file and add the permissions.
zip_file = ZipFile(zip_file_name, 'w', compression=ZIP_DEFLATED)
zip_info = ZipInfo(config['Python_file_name'])
zip_info.compress_type = ZIP_DEFLATED
zip_info.create_system = 3
zip_info.external_attr = 0o777 << 16
zip_file.writestr(zip_info, zip_code) # Adds the code to the zip file.
os.remove(config['Python_file_name'])
if 'Environment' in func_config:
env = func_config['Environment']
variables = env['Variables']
if 'LAMBDA_ENV' in variables:
variables['LAMBDA_ENV'] = 'NPROD'
if 'envprefix' in variables:
variables['envprefix'] = 'us-nprod-odp'
if 'BUCKET_NAME' in variables:
variables['BUCKET_NAME'] = 'odp-us-nprod-servicesuite'
env['Variables'] = variables
func_config['Environment'] = env
with open(path_prefix + config['Config_prefix'] + df['Lambda'][index] +
'.json',
def setUpClass(cls):
""" get_some_resource() is slow, to avoid calling it for each test use setUpClass()
and store the result as class variable
"""
# Suppress ResourceWarning messages about unclosed connections.
warnings.simplefilter('ignore')
cls.setUpParams(cls)
lambda_client = boto3.client('lambda', region_name=get_region())
cls.test_lambda = 'IntTest-{}'.format(cls.domain).replace('.', '-')
try:
lambda_client.delete_function(FunctionName=cls.test_lambda)
except botocore.exceptions.ClientError:
pass
resp = lambda_client.get_function(
FunctionName=cls.object_store_config['page_out_lambda_function'])
lambda_cfg = resp['Configuration']
vpc_cfg = lambda_cfg['VpcConfig']
# VpcId is not a valid field when creating a lambda fcn.
del vpc_cfg['VpcId']
temp_file = tempfile.NamedTemporaryFile()
temp_name = temp_file.name + '.zip'
temp_file.close()
with ZipFile(temp_name, mode='w') as zip:
t = time.localtime()
lambda_file = ZipInfo('lambda_function.py',
date_time=(t.tm_year, t.tm_mon, t.tm_mday,
t.tm_hour, t.tm_min, t.tm_sec))
# Set file permissions.
lambda_file.external_attr = 0o777 << 16
code = 'def handler(event, context):\n return\n'
zip.writestr(lambda_file, code)
with open(temp_name, 'rb') as zip2:
lambda_bytes = zip2.read()
lambda_client.create_function(FunctionName=cls.test_lambda,
VpcConfig=vpc_cfg,
Role=lambda_cfg['Role'],
Runtime=lambda_cfg['Runtime'],
Handler='lambda_function.handler',
MemorySize=128,
Code={'ZipFile': lambda_bytes})
# Set page out function to the test lambda.
cls.object_store_config['page_out_lambda_function'] = cls.test_lambda
print('standby for queue creation (slow ~30s)')
try:
cls.object_store_config[
"s3_flush_queue"] = cls.setup_helper.create_flush_queue(
cls.s3_flush_queue_name)
except ClientError:
try:
cls.setup_helper.delete_flush_queue(
cls.object_store_config["s3_flush_queue"])
except:
pass
time.sleep(61)
cls.object_store_config[
"s3_flush_queue"] = cls.setup_helper.create_flush_queue(
cls.s3_flush_queue_name)
print('done')
def decryptBook(userkey, inpath, outpath):
with closing(ZipFile(open(inpath, 'rb'))) as inf:
namelist = inf.namelist()
if 'META-INF/rights.xml' not in namelist or \
'META-INF/encryption.xml' not in namelist:
print("{0:s} is DRM-free.".format(os.path.basename(inpath)))
return 1
for name in META_NAMES:
namelist.remove(name)
try:
rights = etree.fromstring(inf.read('META-INF/rights.xml'))
adept = lambda tag: '{%s}%s' % (NSMAP['adept'], tag)
expr = './/%s' % (adept('encryptedKey'), )
bookkeyelem = rights.find(expr)
bookkey = bookkeyelem.text
keytype = bookkeyelem.attrib.get('keyType', '0')
if len(bookkey) >= 172 and int(keytype, 10) > 2:
print("{0:s} is a secure Adobe Adept ePub with hardening.".
format(os.path.basename(inpath)))
elif len(bookkey) == 172:
print("{0:s} is a secure Adobe Adept ePub.".format(
os.path.basename(inpath)))
elif len(bookkey) == 64:
print("{0:s} is a secure Adobe PassHash (B&N) ePub.".format(
os.path.basename(inpath)))
else:
print("{0:s} is not an Adobe-protected ePub!".format(
os.path.basename(inpath)))
return 1
if len(bookkey) != 64:
# Normal or "hardened" Adobe ADEPT
rsakey = RSA.import_key(userkey) # parses the ASN1 structure
bookkey = base64.b64decode(bookkey)
if int(keytype, 10) > 2:
bookkey = removeHardening(rights, keytype, bookkey)
try:
bookkey = PKCS1_v1_5.new(rsakey).decrypt(
bookkey, None) # automatically unpads
except ValueError:
bookkey = None
if bookkey is None:
print("Could not decrypt {0:s}. Wrong key".format(
os.path.basename(inpath)))
return 2
else:
# Adobe PassHash / B&N
key = base64.b64decode(userkey)[:16]
bookkey = base64.b64decode(bookkey)
bookkey = unpad(
AES.new(key, AES.MODE_CBC, b'\x00' * 16).decrypt(bookkey),
16) # PKCS#7
if len(bookkey) > 16:
bookkey = bookkey[-16:]
encryption = inf.read('META-INF/encryption.xml')
decryptor = Decryptor(bookkey, encryption)
kwds = dict(compression=ZIP_DEFLATED, allowZip64=False)
with closing(ZipFile(open(outpath, 'wb'), 'w', **kwds)) as outf:
for path in (["mimetype"] + namelist):
data = inf.read(path)
zi = ZipInfo(path)
zi.compress_type = ZIP_DEFLATED
if path == "mimetype":
zi.compress_type = ZIP_STORED
elif path == "META-INF/encryption.xml":
# Check if there's still something in there
if (decryptor.check_if_remaining()):
data = decryptor.get_xml()
print(
"Adding encryption.xml for the remaining embedded files."
)
# We removed DRM, but there's still stuff like obfuscated fonts.
else:
continue
try:
# get the file info, including time-stamp
oldzi = inf.getinfo(path)
# copy across useful fields
zi.date_time = oldzi.date_time
zi.comment = oldzi.comment
zi.extra = oldzi.extra
zi.internal_attr = oldzi.internal_attr
# external attributes are dependent on the create system, so copy both.
zi.external_attr = oldzi.external_attr
zi.create_system = oldzi.create_system
if any(ord(c) >= 128 for c in path) or any(
ord(c) >= 128 for c in zi.comment):
# If the file name or the comment contains any non-ASCII char, set the UTF8-flag
zi.flag_bits |= 0x800
except:
pass
if path == "META-INF/encryption.xml":
outf.writestr(zi, data)
else:
outf.writestr(zi, decryptor.decrypt(path, data))
except:
print("Could not decrypt {0:s} because of an exception:\n{1:s}".
format(os.path.basename(inpath), traceback.format_exc()))
return 2
return 0
from io import BytesIO, StringIO
from re import compile as re_compile
from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED
BUCKET_NAME = "dist-gov"
CFN_PREFIX = "cfn-templates/"
ZIP_PREFIX = ""
# Create a ZIP archive for Lambda.
archive_bytes_io = BytesIO()
with ZipFile(archive_bytes_io, mode="w", compression=ZIP_DEFLATED) as zf:
zi = ZipInfo("ebs_snapshot_manager.py")
zi.compress_type = ZIP_DEFLATED
zi.create_system = 3 # Unix
zi.external_attr = 0o775 << 16 # rwxrwxr-x
with open("ebs_snapshot_manager.py", mode="rb") as src_file:
zf.writestr(zi, src_file.read())
# Compute the SHA 256 value of the file we'll use.
archive_bytes = archive_bytes_io.getvalue()
digest = sha256(archive_bytes).hexdigest()
assert isinstance(digest, str)
zip_obj_name = ZIP_PREFIX + "ebs_snapshot_manager.zip.%s" % digest
# Upload the archie to our CFN endpoint.
s3 = boto3.resource("s3")
s3_obj = s3.Object(BUCKET_NAME, zip_obj_name)
s3_obj.put(ACL="public-read",
Body=archive_bytes,
ContentType="application/zip")
#!/usr/bin/env python2
import os, sys, mmap
from zipfile import ZipFile, ZipInfo
with ZipFile(sys.argv[1], "w") as z:
for filename in sys.argv[2:]:
with open(filename, "r+b") as f:
mm = mmap.mmap(f.fileno(), 0)
try:
info = ZipInfo(os.path.basename(filename), (1980, 1, 1, 0, 0, 0))
info.create_system = 3 # Unix
info.external_attr = 0644 << 16L # rw-r--r--
z.writestr(info, mm)
finally:
mm.close()
