def copy_(src, dst):
if os.path.isdir(src):
util.copytree(src, dst)
elif os.path.isfile(src):
util.print_D(src + " => " + dst)
if not os.path.isdir(os.path.dirname(dst)):
os.makedirs(os.path.dirname(dst))
shutil.copyfile(src, dst)
else:
util.print_E("ERROR: " + src + " is not either a file or dir")
sys.exit(-1)
def copy_(src, dst):
if (os.path.isdir(src)):
util.copytree(src, dst)
elif (os.path.isfile(src)):
util.print_D(src + " => " + dst)
if (not os.path.isdir(os.path.dirname(dst))):
os.makedirs(os.path.dirname(dst))
shutil.copyfile(src, dst)
else:
util.print_E("ERROR: " + src + " is not either a file or dir")
sys.exit(-1)
def build_package(package, package_dir, packages, plugins, nosource):
"""Support function for building package files into a given directory."""
for p in package.extends_list:
try:
build_package(packages[p], package_dir, packages, plugins, nosource)
except KeyError:
err_msg = 'Package {} extends non-existent package {}'
util.error(err_msg.format(package.name, p))
cfg_dir = os.path.join(package_dir, 'cfg')
sm_dir = os.path.join(package_dir, 'addons', 'sourcemod')
plugin_dir = os.path.join(sm_dir, 'plugins')
config_dir = os.path.join(sm_dir, 'configs')
translation_dir = os.path.join(sm_dir, 'translations')
data_dir = os.path.join(sm_dir, 'data')
gamedata_dir = os.path.join(sm_dir, 'gamedata')
output_source_dir = os.path.join(plugin_dir, '..', 'scripting')
for p in package.plugins:
if p not in plugins:
err_msg = 'Package {} used non-existent plugin {}'
util.error(err_msg.format(package.name, p))
# copy plugin binaries
util.mkdir(plugin_dir)
binary_path = os.path.join(package_dir, '..', 'plugins', p + '.smx')
shutil.copy2(binary_path, plugin_dir)
# copy source files
if not nosource:
util.mkdir(output_source_dir)
for source_file in plugins[p].source_files:
source_path = os.path.join(plugins[p].source_dir, source_file)
output_file_path = os.path.join(output_source_dir, source_file)
util.mkdir(os.path.dirname(output_file_path))
shutil.copyfile(source_path, output_file_path)
# copy filegroup definitions
for filegroup in package.filegroups:
filegroup_out_dir = os.path.join(package_dir, filegroup)
util.mkdir(filegroup_out_dir)
for f in package.filegroups[filegroup]:
if os.path.isdir(f):
util.copytree()
else:
shutil.copy2(f, filegroup_out_dir)
# copy everything else
util.safe_copytree(package.configs, config_dir)
util.safe_copytree(package.translations, translation_dir)
util.safe_copytree(package.cfg, cfg_dir)
util.safe_copytree(package.data, data_dir)
util.safe_copytree(package.gamedata, gamedata_dir)
def create_lambda_zip(self, prefix='lambda_package', handler_file=None,
minify=True, exclude=None, use_precompiled_packages=True, include=None, venv=None):
"""
Creates a Lambda-ready zip file of the current virtualenvironment and working directory.
Returns path to that file.
"""
print("Packaging project as zip...")
if not venv:
if 'VIRTUAL_ENV' in os.environ:
venv = os.environ['VIRTUAL_ENV']
elif os.path.exists('.python-version'): # pragma: no cover
logger.debug("Pyenv's local virtualenv detected.")
try:
subprocess.check_output('pyenv', stderr=subprocess.STDOUT)
except OSError as e:
print("This directory seems to have pyenv's local venv"
"but pyenv executable was not found.")
with open('.python-version', 'r') as f:
env_name = f.read()[:-1]
logger.debug('env name = {}'.format(env_name))
bin_path = subprocess.check_output(['pyenv', 'which', 'python']).decode('utf-8')
venv = bin_path[:bin_path.rfind(env_name)] + env_name
logger.debug('env path = {}'.format(venv))
else: # pragma: no cover
print("Zappa requires an active virtual environment.")
quit()
cwd = os.getcwd()
zip_fname = prefix + '-' + str(int(time.time())) + '.zip'
zip_path = os.path.join(cwd, zip_fname)
# Files that should be excluded from the zip
if exclude is None:
exclude = list()
# Exclude the zip itself
exclude.append(zip_path)
def splitpath(path):
parts = []
(path, tail) = os.path.split(path)
while path and tail:
parts.append(tail)
(path, tail) = os.path.split(path)
parts.append(os.path.join(path, tail))
return map(os.path.normpath, parts)[::-1]
split_venv = splitpath(venv)
split_cwd = splitpath(cwd)
# Ideally this should be avoided automatically,
# but this serves as an okay stop-gap measure.
if split_venv[-1] == split_cwd[-1]: # pragma: no cover
print("Warning! Your project and virtualenv have the same name! You may want to re-create your venv with a new name, or explicitly define a 'project_name', as this may cause errors.")
# First, do the project..
temp_project_path = os.path.join(tempfile.gettempdir(), str(int(time.time())))
if minify:
excludes = ZIP_EXCLUDES + exclude + [split_venv[-1]]
copytree(cwd, temp_project_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
else:
copytree(cwd, temp_project_path, symlinks=False)
# Then, do the site-packages..
# TODO Windows: %VIRTUAL_ENV%\Lib\site-packages
temp_package_path = os.path.join(tempfile.gettempdir(), str(int(time.time() + 1)))
site_packages = os.path.join(venv, 'lib', 'python2.7', 'site-packages')
if minify:
excludes = ZIP_EXCLUDES + exclude
copytree(site_packages, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
else:
copytree(site_packages, temp_package_path, symlinks=False)
# We may have 64-bin specific packages too.
site_packages_64 = os.path.join(venv, 'lib64', 'python2.7', 'site-packages')
if os.path.exists(site_packages_64):
if minify:
excludes = ZIP_EXCLUDES + exclude
copytree(site_packages_64, temp_package_path, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
else:
copytree(site_packages_64, temp_package_path, symlinks=False)
copy_tree(temp_package_path, temp_project_path, update=True)
# Then the pre-compiled packages..
if use_precompiled_packages:
installed_packages_name_set = {package.project_name.lower() for package in
pip.get_installed_distributions()}
for name, details in lambda_packages.items():
if name.lower() in installed_packages_name_set:
tar = tarfile.open(details['path'], mode="r:gz")
for member in tar.getmembers():
# If we can, trash the local version.
if member.isdir():
shutil.rmtree(os.path.join(temp_project_path, member.name), ignore_errors=True)
continue
tar.extract(member, temp_project_path)
# If a handler_file is supplied, copy that to the root of the package,
# because that's where AWS Lambda looks for it. It can't be inside a package.
if handler_file:
filename = handler_file.split(os.sep)[-1]
shutil.copy(handler_file, os.path.join(temp_project_path, filename))
# Then zip it all up..
try:
import zlib
compression_method = zipfile.ZIP_DEFLATED
except ImportError as e: # pragma: no cover
compression_method = zipfile.ZIP_STORED
zipf = zipfile.ZipFile(zip_path, 'w', compression_method)
for root, dirs, files in os.walk(temp_project_path):
for filename in files:
# If there is a .pyc file in this package,
# we can skip the python source code as we'll just
# use the compiled bytecode anyway..
if filename[-3:] == '.py':
abs_filname = os.path.join(root, filename)
abs_pyc_filename = abs_filname + 'c'
if os.path.isfile(abs_pyc_filename):
# but only if the pyc is older than the py,
# otherwise we'll deploy outdated code!
py_time = os.stat(abs_filname).st_mtime
pyc_time = os.stat(abs_pyc_filename).st_mtime
if pyc_time > py_time:
continue
zipf.write(os.path.join(root, filename), os.path.join(root.replace(temp_project_path, ''), filename))
# And, we're done!
zipf.close()
# Trash the temp directory
shutil.rmtree(temp_project_path)
shutil.rmtree(temp_package_path)
# Warn if this is too large for Lambda.
file_stats = os.stat(zip_path)
if file_stats.st_size > 52428800: # pragma: no cover
print("\n\nWarning: Application zip package is likely to be too large for AWS Lambda.\n\n")
return zip_fname
def create_lambda_zip(self,
prefix='lambda_package',
handler_file=None,
minify=True,
exclude=None,
use_precompiled_packages=True,
include=None,
venv=None):
"""
Creates a Lambda-ready zip file of the current virtualenvironment and working directory.
Returns path to that file.
"""
print("Packaging project as zip...")
if not venv:
if 'VIRTUAL_ENV' in os.environ:
venv = os.environ['VIRTUAL_ENV']
elif os.path.exists('.python-version'): # pragma: no cover
logger.debug("Pyenv's local virtualenv detected.")
try:
subprocess.check_output('pyenv', stderr=subprocess.STDOUT)
except OSError as e:
print("This directory seems to have pyenv's local venv"
"but pyenv executable was not found.")
with open('.python-version', 'r') as f:
env_name = f.read()[:-1]
logger.debug('env name = {}'.format(env_name))
bin_path = subprocess.check_output(
['pyenv', 'which', 'python']).decode('utf-8')
venv = bin_path[:bin_path.rfind(env_name)] + env_name
logger.debug('env path = {}'.format(venv))
else: # pragma: no cover
print("Zappa requires an active virtual environment.")
quit()
cwd = os.getcwd()
zip_fname = prefix + '-' + str(int(time.time())) + '.zip'
zip_path = os.path.join(cwd, zip_fname)
# Files that should be excluded from the zip
if exclude is None:
exclude = list()
# Exclude the zip itself
exclude.append(zip_path)
def splitpath(path):
parts = []
(path, tail) = os.path.split(path)
while path and tail:
parts.append(tail)
(path, tail) = os.path.split(path)
parts.append(os.path.join(path, tail))
return map(os.path.normpath, parts)[::-1]
split_venv = splitpath(venv)
split_cwd = splitpath(cwd)
# Ideally this should be avoided automatically,
# but this serves as an okay stop-gap measure.
if split_venv[-1] == split_cwd[-1]: # pragma: no cover
print(
"Warning! Your project and virtualenv have the same name! You may want to re-create your venv with a new name, or explicitly define a 'project_name', as this may cause errors."
)
# First, do the project..
temp_project_path = os.path.join(tempfile.gettempdir(),
str(int(time.time())))
if minify:
excludes = ZIP_EXCLUDES + exclude + [split_venv[-1]]
copytree(cwd,
temp_project_path,
symlinks=False,
ignore=shutil.ignore_patterns(*excludes))
else:
copytree(cwd, temp_project_path, symlinks=False)
# Then, do the site-packages..
temp_package_path = os.path.join(tempfile.gettempdir(),
str(int(time.time() + 1)))
if os.sys.platform == 'win32':
site_packages = os.path.join(venv, 'Lib', 'site-packages')
else:
site_packages = os.path.join(venv, 'lib', 'python2.7',
'site-packages')
if minify:
excludes = ZIP_EXCLUDES + exclude
copytree(site_packages,
temp_package_path,
symlinks=False,
ignore=shutil.ignore_patterns(*excludes))
else:
copytree(site_packages, temp_package_path, symlinks=False)
# We may have 64-bin specific packages too.
site_packages_64 = os.path.join(venv, 'lib64', 'python2.7',
'site-packages')
if os.path.exists(site_packages_64):
if minify:
excludes = ZIP_EXCLUDES + exclude
copytree(site_packages_64,
temp_package_path,
symlinks=False,
ignore=shutil.ignore_patterns(*excludes))
else:
copytree(site_packages_64, temp_package_path, symlinks=False)
copy_tree(temp_package_path, temp_project_path, update=True)
# Then the pre-compiled packages..
if use_precompiled_packages:
installed_packages_name_set = {
package.project_name.lower()
for package in pip.get_installed_distributions()
}
for name, details in lambda_packages.items():
if name.lower() in installed_packages_name_set:
tar = tarfile.open(details['path'], mode="r:gz")
for member in tar.getmembers():
# If we can, trash the local version.
if member.isdir():
shutil.rmtree(os.path.join(temp_project_path,
member.name),
ignore_errors=True)
continue
tar.extract(member, temp_project_path)
# If a handler_file is supplied, copy that to the root of the package,
# because that's where AWS Lambda looks for it. It can't be inside a package.
if handler_file:
filename = handler_file.split(os.sep)[-1]
shutil.copy(handler_file, os.path.join(temp_project_path,
filename))
# Then zip it all up..
try:
import zlib
compression_method = zipfile.ZIP_DEFLATED
except ImportError as e: # pragma: no cover
compression_method = zipfile.ZIP_STORED
zipf = zipfile.ZipFile(zip_path, 'w', compression_method)
for root, dirs, files in os.walk(temp_project_path):
for filename in files:
# If there is a .pyc file in this package,
# we can skip the python source code as we'll just
# use the compiled bytecode anyway..
if filename[-3:] == '.py':
abs_filname = os.path.join(root, filename)
abs_pyc_filename = abs_filname + 'c'
if os.path.isfile(abs_pyc_filename):
# but only if the pyc is older than the py,
# otherwise we'll deploy outdated code!
py_time = os.stat(abs_filname).st_mtime
pyc_time = os.stat(abs_pyc_filename).st_mtime
if pyc_time > py_time:
continue
zipf.write(
os.path.join(root, filename),
os.path.join(root.replace(temp_project_path, ''),
filename))
# And, we're done!
zipf.close()
# Trash the temp directory
shutil.rmtree(temp_project_path)
shutil.rmtree(temp_package_path)
# Warn if this is too large for Lambda.
file_stats = os.stat(zip_path)
if file_stats.st_size > 52428800: # pragma: no cover
print(
"\n\nWarning: Application zip package is likely to be too large for AWS Lambda.\n\n"
)
return zip_fname
def generate_static():
"""
Generate CSS and JavaScript files
"""
util.copytree('static', os.path.join(_deploy_dir, 'static'))
def setup_data_files():
template_data = os.path.join(dir_path, "..", "..", "assets")
target_data = os.path.join(os.getcwd(), "assets")
util.copytree(template_data,
target_data,
ignore=shutil.ignore_patterns('config.json'))
def generate_static():
"""
Generate CSS and JavaScript files
"""
util.copytree('static', os.path.join(_deploy_dir, 'static'))
