def init(self):
"""Initiating the project and provide a sample lmdo.yaml file"""
if self._args.get('<project_name>'):
mkdir('./{}'.format(self._args.get('<project_name>')))
"""Copy lmdo.yaml over"""
# Do not copy over unless it's a clearn dir
if os.path.isfile(
os.path.join(self._args.get('<project_name>'),
PROJECT_CONFIG_FILE)):
Oprint.err(
'Your have existing {} already, exiting...'.format(
PROJECT_CONFIG_FILE), 'lmdo')
pkg_dir = self.get_installed_path()
if pkg_dir:
copytree(os.path.join(pkg_dir, 'template'),
'./{}'.format(self._args.get('<project_name>')))
elif self._args.get('config'):
pkg_dir = self.get_installed_path()
# Don't override existing lmdo.yaml
if os.path.isfile(PROJECT_CONFIG_FILE):
Oprint.warn(
'You have existing {} file, a copy will be created with name {}.copy'
.format(PROJECT_CONFIG_FILE, PROJECT_CONFIG_FILE), 'lmdo')
shutil.copyfile(
os.path.join(pkg_dir, 'template', PROJECT_CONFIG_FILE),
'{}.copy'.format(PROJECT_CONFIG_FILE))
else:
shutil.copyfile(
os.path.join(pkg_dir, 'template', PROJECT_CONFIG_FILE),
PROJECT_CONFIG_FILE)
def get_zipped_package(self, func_name, func_type):
"""Packaging lambda"""
# Create packaging temp dir
lambda_temp_dir = tempfile.mkdtemp()
# Create zip file temp dir
target_temp_dir = tempfile.mkdtemp()
target = '{}/{}'.format(target_temp_dir, self.get_zip_name(func_name))
self.add_init_file_to_root(lambda_temp_dir)
if func_type == self.FUNCTION_TYPE_WSGI:
self.pip_wsgi_install(lambda_temp_dir)
# Heater is one file only from lmdo. don't need packages
if func_type != self.FUNCTION_TYPE_HEATER:
# Copy project files
copytree(os.getcwd(),
lambda_temp_dir,
ignore=shutil.ignore_patterns('*.git*'))
# Installing package
self.dependency_packaging(lambda_temp_dir)
replace_path = [{'from_path': lambda_temp_dir, 'to_path': '.'}]
# Zip what we've got so far
zipper(lambda_temp_dir, target, LAMBDA_EXCLUDE, False,
replace_path)
# Default type function doesn't need lmdo's lambda wrappers
if func_type != self.FUNCTION_TYPE_DEFAULT:
# Don't load lmdo __init__.py
if LAMBDA_EXCLUDE.get('LAMBDA_EXCLUDE'):
LAMBDA_EXCLUDE['file_with_path'].append(
'*{}/{}/__init__.py'.format(self.LMDO_HANDLER_DIR,
func_type))
else:
LAMBDA_EXCLUDE['file_with_path'] = [
'*{}/{}/__init__.py'.format(self.LMDO_HANDLER_DIR,
func_type)
]
replace_path = [{
'from_path': self.get_lmdo_function_dir(func_type),
'to_path': '.'
}]
# Zip extra lmdo function handler
zipper(self.get_lmdo_function_dir(func_type), target,
LAMBDA_EXCLUDE, False, replace_path)
shutil.rmtree(lambda_temp_dir)
return (target_temp_dir, target)
def copy_editable_packages(self, egg_links, temp_package_path):
"""Copy editable packages"""
Oprint.info('Copying editable packages over', 'pip')
for egg_link in egg_links:
with open(egg_link) as df:
egg_path = df.read().decode('utf-8').splitlines()[0].strip()
pkgs = {
x.split(".")[0]
for x in find_packages(egg_path, exclude=['test', 'tests'])
}
for pkg in pkgs:
copytree(os.path.join(egg_path, pkg),
os.path.join(temp_package_path, pkg),
symlinks=False)
if temp_package_path:
# now remove any egg-links as they will cause issues if they still exist
for link in glob.glob(os.path.join(temp_package_path,
"*.egg-link")):
os.remove(link)
def fetch(self):
"""Fetch template repo to local"""
try:
Oprint.info(
'Start downloading repo to your project from {}'.format(
self._args.get('<url>')), 'lmdo')
spinner.start()
tmp = tempfile.mkdtemp()
self.git_clone(self._args.get('<url>'), tmp)
copytree(tmp, './', ignore=shutil.ignore_patterns('*.git*'))
shutil.rmtree(tmp)
spinner.stop()
Oprint.info(
'Complete downloading repo to your project from {}'.format(
self._args.get('<url>')), 'lmdo')
except Exception as e:
spinner.stop()
raise e
def venv_package_install(self, tmp_path):
"""Install virtualenv packages"""
import pip
venv = self.get_current_venv_path()
cwd = os.getcwd()
def splitpath(path):
parts = []
(path, tail) = os.path.split(path)
while path and tail:
parts.append(tail)
(path, tail) = os.path.split(path)
parts.append(os.path.join(path, tail))
return map(os.path.normpath, parts)[::-1]
split_venv = splitpath(venv)
split_cwd = splitpath(cwd)
# Ideally this should be avoided automatically,
# but this serves as an okay stop-gap measure.
if split_venv[-1] == split_cwd[-1]: # pragma: no cover
Oprint.warn(
"Warning! Your project and virtualenv have the same name! You may want to re-create your venv with a new name, or explicitly define a 'project_name', as this may cause errors.",
'lambda')
# Then, do site site-packages..
egg_links = []
site_packages = os.path.join(venv, 'lib', 'python2.7', 'site-packages')
egg_links.extend(glob.glob(os.path.join(site_packages, '*.egg-link')))
Oprint.info('Copying lib packages over', 'pip')
copytree(site_packages,
tmp_path,
symlinks=False,
ignore=shutil.ignore_patterns(*self.VIRTUALENV_ZIP_EXCLUDES))
# We may have 64-bin specific packages too.
site_packages_64 = os.path.join(venv, 'lib64', 'python2.7',
'site-packages')
if os.path.exists(site_packages_64):
egg_links.extend(
glob.glob(os.path.join(site_packages_64, '*.egg-link')))
Oprint.info('Copying lib64 packages over', 'pip')
copytree(
site_packages_64,
tmp_path,
symlinks=False,
ignore=shutil.ignore_patterns(*self.VIRTUALENV_ZIP_EXCLUDES))
if egg_links:
self.copy_editable_packages(egg_links, tmp_path)
package_to_keep = []
if os.path.isdir(site_packages):
package_to_keep += os.listdir(site_packages)
if os.path.isdir(site_packages_64):
package_to_keep += os.listdir(site_packages_64)
installed_packages_name_set = self.get_virtualenv_installed_package()
# First, try lambda packages
for name, details in lambda_packages.iteritems():
if name.lower() in installed_packages_name_set:
Oprint.info(
'Installing Lambda_package Amazon Linux AMI bianry package {} to {}'
.format(name, tmp_path), 'pip')
tar = tarfile.open(details['path'], mode="r:gz")
for member in tar.getmembers():
# If we can, trash the local version.
if member.isdir():
shutil.rmtree(os.path.join(tmp_path, member.name),
ignore_errors=True)
continue
tar.extract(member, tmp_path)
installed_packages_name_set.remove(name.lower())
# Then try to use manylinux packages from PyPi..
# Related: https://github.com/Miserlou/Zappa/issues/398
try:
Oprint.info(
'Installing virtualenv python package dependancies to {}'.
format(tmp_path), 'pip')
spinner.start()
for installed_package_name in installed_packages_name_set:
wheel_url = self.get_manylinux_wheel(installed_package_name)
if wheel_url:
resp = requests.get(wheel_url, timeout=2, stream=True)
resp.raw.decode_content = True
zipresp = resp.raw
with zipfile.ZipFile(BytesIO(zipresp.read())) as zfile:
zfile.extractall(tmp_path)
spinner.stop()
except Exception as e:
spinner.stop()
Oprint.warn(e, 'pip')
def get_zipped_package(self, function_config):
"""Packaging lambda"""
func_name = function_config.get('FunctionName')
func_type = function_config.get('Type')
# Create packaging temp dir
lambda_temp_dir = tempfile.mkdtemp()
# Create zip file temp dir
target_temp_dir = tempfile.mkdtemp()
target = '{}/{}'.format(target_temp_dir, self.get_zip_name(func_name))
self.add_init_file_to_root(lambda_temp_dir)
if func_type == self.FUNCTION_TYPE_WSGI:
self.pip_wsgi_install(lambda_temp_dir)
# Heater is one file only from lmdo. don't need packages
if func_type != self.FUNCTION_TYPE_HEATER:
# Go only need executables
if func_type == self.FUNCTION_TYPE_GO:
if not function_config.get('ExecutableName'):
Oprint.err(
'ExecutableName is not defined in lmdo config, function {} won\'t be deployed'
.format(func_name), self.NAME)
return False, False
# We only have on executable needed
shutil.copy(
os.path.join(os.getcwd(),
function_config.get('ExecutableName')),
lambda_temp_dir)
else:
# Copy project files
copytree(os.getcwd(),
lambda_temp_dir,
ignore=shutil.ignore_patterns('*.git*'))
# Installing package
self.dependency_packaging(lambda_temp_dir)
replace_path = [{'from_path': lambda_temp_dir, 'to_path': '.'}]
# Zip what we've got so far
zipper(lambda_temp_dir, target, LAMBDA_EXCLUDE, False,
replace_path)
# Default type function doesn't need lmdo's lambda wrappers
if func_type != self.FUNCTION_TYPE_DEFAULT:
# Don't load lmdo __init__.py
if LAMBDA_EXCLUDE.get('LAMBDA_EXCLUDE'):
LAMBDA_EXCLUDE['file_with_path'].append(
'*{}/{}/__init__.py'.format(self.LMDO_HANDLER_DIR,
func_type))
else:
LAMBDA_EXCLUDE['file_with_path'] = [
'*{}/{}/__init__.py'.format(self.LMDO_HANDLER_DIR,
func_type)
]
replace_path = [{
'from_path': self.get_lmdo_function_dir(func_type),
'to_path': '.'
}]
# Zip extra lmdo function handler
zipper(self.get_lmdo_function_dir(func_type), target,
LAMBDA_EXCLUDE, False, replace_path)
shutil.rmtree(lambda_temp_dir)
return (target_temp_dir, target)