def test_packages():
for name, package in lambda_packages.items():
for runtime, details in package.items():
package_file_name = os.path.split(details['path'])[1]
assert_true(package_file_name.startswith(runtime + "-"))
assert_true(os.path.exists(details['path']))
def test_packages():
for name, package in lambda_packages.items():
for runtime, details in package.items():
package_file_name = os.path.split(details['path'])[1]
assert_true(package_file_name.startswith(runtime + "-"))
assert_true(os.path.exists(details['path']))
def find_lambda_pkg(item):
found = False
for name, detail in lambda_packages.items():
if item.startswith(name.lower()):
lambda_pkg_to_install[name.lower()] = detail
return False
else:
continue
return True
def create_lambda_zip(self, prefix='lambda_package', handler_file=None,
minify=True, exclude=None, use_precompiled_packages=True, include=None, venv=None):
"""
Creates a Lambda-ready zip file of the current virtualenvironment and working directory.
Returns path to that file.
"""
print("Packaging project as zip...")
if not venv:
try:
venv = os.environ['VIRTUAL_ENV']
except KeyError as e: # pragma: no cover
print("Zappa requires an active virtual environment.")
quit()
cwd = os.getcwd()
zip_fname = prefix + '-' + str(int(time.time())) + '.zip'
zip_path = os.path.join(cwd, zip_fname)
# Files that should be excluded from the zip
if exclude is None:
exclude = list()
# Exclude the zip itself
exclude.append(zip_path)
def splitpath(path):
parts = []
(path, tail) = os.path.split(path)
while path and tail:
parts.append(tail)
(path, tail) = os.path.split(path)
parts.append(os.path.join(path, tail))
return map(os.path.normpath, parts)[::-1]
split_venv = splitpath(venv)
split_cwd = splitpath(cwd)
# Ideally this should be avoided automatically,
# but this serves as an okay stop-gap measure.
if split_venv[-1] == split_cwd[-1]: # pragma: no cover
print("Warning! Your project and virtualenv have the same name! You may want to re-create your venv with a new name, or explicitly define a 'project_name', as this may cause errors.")
# First, do the project..
temp_project_path = os.path.join(tempfile.gettempdir(), str(int(time.time())))
if minify:
excludes = ZIP_EXCLUDES + exclude + [split_venv[-1]]
shutil.copytree(cwd, temp_project_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
else:
shutil.copytree(cwd, temp_project_path, symlinks=False)
# Then, do the site-packages..
# TODO Windows: %VIRTUAL_ENV%\Lib\site-packages
temp_package_path = os.path.join(tempfile.gettempdir(), str(int(time.time() + 1)))
site_packages = os.path.join(venv, 'lib', 'python2.7', 'site-packages')
if minify:
excludes = ZIP_EXCLUDES + exclude
shutil.copytree(site_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
else:
shutil.copytree(site_packages, temp_package_path, symlinks=False)
# We may have 64-bin specific packages too.
site_packages_64 = os.path.join(venv, 'lib64', 'python2.7', 'site-packages')
if os.path.exists(site_packages_64):
if minify:
excludes = ZIP_EXCLUDES + exclude
shutil.copytree(site_packages_64, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
else:
shutil.copytree(site_packages_64, temp_package_path, symlinks=False)
copy_tree(temp_package_path, temp_project_path, update=True)
# Then the pre-compiled packages..
if use_precompiled_packages:
installed_packages_name_set = {package.project_name.lower() for package in
pip.get_installed_distributions()}
for name, details in lambda_packages.items():
if name.lower() in installed_packages_name_set:
tar = tarfile.open(details['path'], mode="r:gz")
for member in tar.getmembers():
# If we can, trash the local version.
if member.isdir():
shutil.rmtree(os.path.join(temp_project_path, member.name), ignore_errors=True)
continue
tar.extract(member, temp_project_path)
# If a handler_file is supplied, copy that to the root of the package,
# because that's where AWS Lambda looks for it. It can't be inside a package.
if handler_file:
filename = handler_file.split(os.sep)[-1]
shutil.copy(handler_file, os.path.join(temp_project_path, filename))
# Then zip it all up..
try:
import zlib
compression_method = zipfile.ZIP_DEFLATED
except ImportError as e: # pragma: no cover
compression_method = zipfile.ZIP_STORED
zipf = zipfile.ZipFile(zip_path, 'w', compression_method)
for root, dirs, files in os.walk(temp_project_path):
for filename in files:
# If there is a .pyc file in this package,
# we can skip the python source code as we'll just
# use the compiled bytecode anyway..
if filename[-3:] == '.py':
abs_filname = os.path.join(root, filename)
abs_pyc_filename = abs_filname + 'c'
if os.path.isfile(abs_pyc_filename):
# but only if the pyc is older than the py,
# otherwise we'll deploy outdated code!
py_time = os.stat(abs_filname).st_mtime
pyc_time = os.stat(abs_pyc_filename).st_mtime
if pyc_time > py_time:
continue
zipf.write(os.path.join(root, filename), os.path.join(root.replace(temp_project_path, ''), filename))
# And, we're done!
zipf.close()
# Trash the temp directory
shutil.rmtree(temp_project_path)
shutil.rmtree(temp_package_path)
# Warn if this is too large for Lambda.
file_stats = os.stat(zip_path)
if file_stats.st_size > 52428800: # pragma: no cover
print("\n\nWarning: Application zip package is likely to be too large for AWS Lambda.\n\n")
return zip_fname
def create_lambda_zip(self,
prefix='lambda_package',
handler_file=None,
minify=True,
exclude=None,
use_precompiled_packages=True,
include=None,
venv=None):
"""
Creates a Lambda-ready zip file of the current virtualenvironment and working directory.
Returns path to that file.
"""
print("Packaging project as zip...")
if not venv:
try:
venv = os.environ['VIRTUAL_ENV']
except KeyError as e: # pragma: no cover
print("Zappa requires an active virtual environment.")
quit()
cwd = os.getcwd()
zip_fname = prefix + '-' + str(int(time.time())) + '.zip'
zip_path = os.path.join(cwd, zip_fname)
# Files that should be excluded from the zip
if exclude is None:
exclude = list()
# Exclude the zip itself
exclude.append(zip_path)
def splitpath(path):
parts = []
(path, tail) = os.path.split(path)
while path and tail:
parts.append(tail)
(path, tail) = os.path.split(path)
parts.append(os.path.join(path, tail))
return map(os.path.normpath, parts)[::-1]
split_venv = splitpath(venv)
split_cwd = splitpath(cwd)
# Ideally this should be avoided automatically,
# but this serves as an okay stop-gap measure.
if split_venv[-1] == split_cwd[-1]:
print(
"Warning! Your project and virtualenv have the same name! You may want to re-create your venv with a new name, or explicitly define a 'project_name', as this may cause errors."
)
# First, do the project..
temp_project_path = os.path.join(tempfile.gettempdir(),
str(int(time.time())))
if minify:
excludes = ZIP_EXCLUDES + exclude + [split_venv[-1]]
shutil.copytree(cwd,
temp_project_path,
symlinks=False,
ignore=shutil.ignore_patterns(*excludes))
else:
shutil.copytree(cwd, temp_project_path, symlinks=False)
# Then, do the site-packages..
# TODO Windows: %VIRTUAL_ENV%\Lib\site-packages
temp_package_path = os.path.join(tempfile.gettempdir(),
str(int(time.time() + 1)))
site_packages = os.path.join(venv, 'lib', 'python2.7', 'site-packages')
if minify:
excludes = ZIP_EXCLUDES + exclude
shutil.copytree(site_packages,
temp_package_path,
symlinks=False,
ignore=shutil.ignore_patterns(*excludes))
else:
shutil.copytree(site_packages, temp_package_path, symlinks=False)
copy_tree(temp_package_path, temp_project_path, update=True)
# Then the pre-compiled packages..
if use_precompiled_packages:
installed_packages_name_set = {
package.project_name.lower()
for package in pip.get_installed_distributions()
}
for name, details in lambda_packages.items():
if name.lower() in installed_packages_name_set:
tar = tarfile.open(details['path'], mode="r:gz")
for member in tar.getmembers():
# If we can, trash the local version.
if member.isdir():
shutil.rmtree(os.path.join(temp_project_path,
member.name),
ignore_errors=True)
continue
tar.extract(member, temp_project_path)
# If a handler_file is supplied, copy that to the root of the package,
# because that's where AWS Lambda looks for it. It can't be inside a package.
if handler_file:
filename = handler_file.split(os.sep)[-1]
shutil.copy(handler_file, os.path.join(temp_project_path,
filename))
# Then zip it all up..
try:
import zlib
compression_method = zipfile.ZIP_DEFLATED
except Exception as e: # pragma: no cover
compression_method = zipfile.ZIP_STORED
zipf = zipfile.ZipFile(zip_path, 'w', compression_method)
for root, dirs, files in os.walk(temp_project_path):
for filename in files:
# If there is a .pyc file in this package,
# we can skip the python source code as we'll just
# use the compiled bytecode anyway.
if filename[-3:] == '.py':
if os.path.isfile(os.path.join(root, filename) + 'c'):
continue
zipf.write(
os.path.join(root, filename),
os.path.join(root.replace(temp_project_path, ''),
filename))
# And, we're done!
zipf.close()
# Trash the temp directory
shutil.rmtree(temp_project_path)
shutil.rmtree(temp_package_path)
# Warn if this is too large for Lambda.
file_stats = os.stat(zip_path)
if file_stats.st_size > 52428800: # pragma: no cover
print(
"\n\nWarning: Application zip package is likely to be too large for AWS Lambda.\n\n"
)
return zip_fname
def test_packages():
for name, details in lambda_packages.items():
assert_true(os.path.exists(details['path']))
def create_lambda_zip(self, prefix='lambda_package', handler_file=None,
minify=True, exclude=None, use_precompiled_packages=True):
"""
Creates a Lambda-ready zip file of the current virtualenvironment and working directory.
Returns path to that file.
"""
print("Packaging project as zip...")
venv = os.environ['VIRTUAL_ENV']
cwd = os.getcwd()
zip_fname = prefix + '-' + str(int(time.time())) + '.zip'
zip_path = os.path.join(cwd, zip_fname)
# Files that should be excluded from the zip
if exclude is None:
exclude = list()
# Exclude the zip itself
exclude.append(zip_path)
def splitpath(path):
parts = []
(path, tail) = os.path.split(path)
while path and tail:
parts.append(tail)
(path, tail) = os.path.split(path)
parts.append(os.path.join(path, tail))
return map(os.path.normpath, parts)[::-1]
split_venv = splitpath(venv)
# First, do the project..
temp_project_path = os.path.join(tempfile.gettempdir(), str(int(time.time())))
if minify:
excludes = ZIP_EXCLUDES + exclude + [split_venv[-1]]
shutil.copytree(cwd, temp_project_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
else:
shutil.copytree(cwd, temp_project_path, symlinks=False)
# Then, do the site-packages..
# TODO Windows: %VIRTUAL_ENV%\Lib\site-packages
temp_package_path = os.path.join(tempfile.gettempdir(), str(int(time.time() + 1)))
site_packages = os.path.join(venv, 'lib', 'python2.7', 'site-packages')
if minify:
excludes = ZIP_EXCLUDES + exclude
shutil.copytree(site_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
else:
shutil.copytree(site_packages, temp_package_path, symlinks=False)
copy_tree(temp_package_path, temp_project_path, update=True)
# Then the pre-compiled packages..
if use_precompiled_packages:
installed_packages = pip.get_installed_distributions()
for package in installed_packages:
package_name = package.project_name.lower()
for name, details in lambda_packages.items():
if name.lower() == package_name:
tar = tarfile.open(details['path'], mode="r:gz")
for member in tar.getmembers():
# If we can, trash the local version.
if member.isdir():
shutil.rmtree(os.path.join(temp_project_path, member.name), ignore_errors=True)
continue
tar.extract(member, temp_project_path)
# If a handler_file is supplied, copy that to the root of the package,
# because that's where AWS Lambda looks for it. It can't be inside a package.
if handler_file:
filename = handler_file.split(os.sep)[-1]
shutil.copy(handler_file, os.path.join(temp_project_path, filename))
# Then zip it all up..
try:
import zlib
compression_method = zipfile.ZIP_DEFLATED
except Exception as e: # pragma: no cover
compression_method = zipfile.ZIP_STORED
zipf = zipfile.ZipFile(zip_path, 'w', compression_method)
for root, dirs, files in os.walk(temp_project_path):
for filename in files:
# If there is a .pyc file in this package,
# we can skip the python source code as we'll just
# use the compiled bytecode anyway.
if filename[-3:] == '.py':
if os.path.isfile(os.path.join(root, filename) + 'c'):
continue
zipf.write(os.path.join(root, filename), os.path.join(root.replace(temp_project_path, ''), filename))
# And, we're done!
zipf.close()
# Trash the temp directory
shutil.rmtree(temp_project_path)
# Warn if this is too large for Lambda.
file_stats = os.stat(zip_path)
if file_stats.st_size > 52428800: # pragma: no cover
print("\n\nWarning: Application zip package is likely to be too large for AWS Lambda.\n\n")
return zip_fname
def create_lambda_zip(self,
prefix='lambda_package',
handler_file=None,
minify=True,
exclude=None,
use_precompiled_packages=True,
include=None,
venv=None):
"""
Creates a Lambda-ready zip file of the current virtualenvironment and working directory.
Returns path to that file.
"""
print("Packaging project as zip...")
if not venv:
if 'VIRTUAL_ENV' in os.environ:
venv = os.environ['VIRTUAL_ENV']
elif os.path.exists('.python-version'): # pragma: no cover
logger.debug("Pyenv's local virtualenv detected.")
try:
subprocess.check_output('pyenv', stderr=subprocess.STDOUT)
except OSError as e:
print("This directory seems to have pyenv's local venv"
"but pyenv executable was not found.")
with open('.python-version', 'r') as f:
env_name = f.read()[:-1]
logger.debug('env name = {}'.format(env_name))
bin_path = subprocess.check_output(
['pyenv', 'which', 'python']).decode('utf-8')
venv = bin_path[:bin_path.rfind(env_name)] + env_name
logger.debug('env path = {}'.format(venv))
else: # pragma: no cover
print("Zappa requires an active virtual environment.")
quit()
cwd = os.getcwd()
zip_fname = prefix + '-' + str(int(time.time())) + '.zip'
zip_path = os.path.join(cwd, zip_fname)
# Files that should be excluded from the zip
if exclude is None:
exclude = list()
# Exclude the zip itself
exclude.append(zip_path)
def splitpath(path):
parts = []
(path, tail) = os.path.split(path)
while path and tail:
parts.append(tail)
(path, tail) = os.path.split(path)
parts.append(os.path.join(path, tail))
return map(os.path.normpath, parts)[::-1]
split_venv = splitpath(venv)
split_cwd = splitpath(cwd)
# Ideally this should be avoided automatically,
# but this serves as an okay stop-gap measure.
if split_venv[-1] == split_cwd[-1]: # pragma: no cover
print(
"Warning! Your project and virtualenv have the same name! You may want to re-create your venv with a new name, or explicitly define a 'project_name', as this may cause errors."
)
# First, do the project..
temp_project_path = os.path.join(tempfile.gettempdir(),
str(int(time.time())))
if minify:
excludes = ZIP_EXCLUDES + exclude + [split_venv[-1]]
copytree(cwd,
temp_project_path,
symlinks=False,
ignore=shutil.ignore_patterns(*excludes))
else:
copytree(cwd, temp_project_path, symlinks=False)
# Then, do the site-packages..
temp_package_path = os.path.join(tempfile.gettempdir(),
str(int(time.time() + 1)))
if os.sys.platform == 'win32':
site_packages = os.path.join(venv, 'Lib', 'site-packages')
else:
site_packages = os.path.join(venv, 'lib', 'python2.7',
'site-packages')
if minify:
excludes = ZIP_EXCLUDES + exclude
copytree(site_packages,
temp_package_path,
symlinks=False,
ignore=shutil.ignore_patterns(*excludes))
else:
copytree(site_packages, temp_package_path, symlinks=False)
# We may have 64-bin specific packages too.
site_packages_64 = os.path.join(venv, 'lib64', 'python2.7',
'site-packages')
if os.path.exists(site_packages_64):
if minify:
excludes = ZIP_EXCLUDES + exclude
copytree(site_packages_64,
temp_package_path,
symlinks=False,
ignore=shutil.ignore_patterns(*excludes))
else:
copytree(site_packages_64, temp_package_path, symlinks=False)
copy_tree(temp_package_path, temp_project_path, update=True)
# Then the pre-compiled packages..
if use_precompiled_packages:
installed_packages_name_set = {
package.project_name.lower()
for package in pip.get_installed_distributions()
}
for name, details in lambda_packages.items():
if name.lower() in installed_packages_name_set:
tar = tarfile.open(details['path'], mode="r:gz")
for member in tar.getmembers():
# If we can, trash the local version.
if member.isdir():
shutil.rmtree(os.path.join(temp_project_path,
member.name),
ignore_errors=True)
continue
tar.extract(member, temp_project_path)
# If a handler_file is supplied, copy that to the root of the package,
# because that's where AWS Lambda looks for it. It can't be inside a package.
if handler_file:
filename = handler_file.split(os.sep)[-1]
shutil.copy(handler_file, os.path.join(temp_project_path,
filename))
# Then zip it all up..
try:
import zlib
compression_method = zipfile.ZIP_DEFLATED
except ImportError as e: # pragma: no cover
compression_method = zipfile.ZIP_STORED
zipf = zipfile.ZipFile(zip_path, 'w', compression_method)
for root, dirs, files in os.walk(temp_project_path):
for filename in files:
# If there is a .pyc file in this package,
# we can skip the python source code as we'll just
# use the compiled bytecode anyway..
if filename[-3:] == '.py':
abs_filname = os.path.join(root, filename)
abs_pyc_filename = abs_filname + 'c'
if os.path.isfile(abs_pyc_filename):
# but only if the pyc is older than the py,
# otherwise we'll deploy outdated code!
py_time = os.stat(abs_filname).st_mtime
pyc_time = os.stat(abs_pyc_filename).st_mtime
if pyc_time > py_time:
continue
zipf.write(
os.path.join(root, filename),
os.path.join(root.replace(temp_project_path, ''),
filename))
# And, we're done!
zipf.close()
# Trash the temp directory
shutil.rmtree(temp_project_path)
shutil.rmtree(temp_package_path)
# Warn if this is too large for Lambda.
file_stats = os.stat(zip_path)
if file_stats.st_size > 52428800: # pragma: no cover
print(
"\n\nWarning: Application zip package is likely to be too large for AWS Lambda.\n\n"
)
return zip_fname
def test_packages():
for name, details in lambda_packages.items():
assert_true(os.path.exists(details['path']))