def sync_requirements(args):
if not os.path.exists(os.path.join(WORKING_DIR, 'Pipfile')) and \
os.path.exists(os.path.join(WORKING_DIR, 'Pipfile.lock')):
logger.error(
"This feature requires a root level 'Pipfile' and 'Pipfile.lock'")
sys.exit(1)
template = SAMTemplate(args.template)
pipfile_packages = get_pipfile_packages()
logger.info('Evaluating Lambda function dependencies...\n')
for k, v in template.lambda_resources.items():
handler_file = template.get_lambda_handler(k)
if not handler_file:
logger.error(f"{k}: There was no 'Handler' found for the Lambda "
f"function and it is being skipped!\n")
continue
lambda_code_dir = v['Properties']['CodeUri']
requirements = parse_requirements(lambda_code_dir)
if not requirements:
logger.info(f"{k}: No requirements.txt file for this Lambda")
continue
results = write_requirements(pipfile_packages, requirements,
lambda_code_dir)
if results:
logger.info(
f"{k}: A requirements.txt file has been generated with "
"the following packages:")
logger.info(f"{k}: {', '.join(results)}\n")
else:
logger.info(f"{k}: No requirements.txt file generated\n")
def get_possum_path(user_dir):
possum_dir = os.path.join(user_dir, '.possum')
if not os.path.exists(possum_dir):
logger.info('Creating Possum directory...')
os.mkdir(possum_dir)
elif not os.path.isdir(possum_dir):
logger.error(f"'{possum_dir}' is not a directory! Delete to "
"allow Possum to recreate the directory")
sys.exit(1)
return os.path.join(possum_dir, _possum_name())
def build_docker_image(version):
logger.info(f"Building 'possum:{version}' Docker image (this may take "
"several minutes)...")
client = docker.from_env()
try:
image = client.images.build(
fileobj=dockerfile(version),
tag=f'possum:{version}',
quiet=False,
rm=True,
pull=True
)
logger.info("Tagging as 'latest'...")
if image[0].tag('possum', 'latest'):
image[0].reload()
else:
logger.info('Unable to add additional tag')
except APIError as err:
logger.error(f'Unable to build the Docker image: {err.explanation}')
sys.exit(1)
except BuildError as err:
logger.error(f'Unable to build the Docker image: {err.msg}')
sys.exit(1)
image_id = image[0].short_id.split(':')[-1]
logger.info("Image successfully created:\n"
f" ID: {image_id}\n"
f" Tags: {', '.join(image[0].tags)}")
def upload_packages(package_directory, bucket_name,
bucket_dir, profile_name=None):
session = boto3.Session(profile_name=profile_name)
s3_client = session.resource('s3')
os.chdir(package_directory)
logger.info(f'Uploading all Lambda packages to: {bucket_dir}')
for artifact in os.listdir('.'):
logger.info(f'Uploading package: {artifact}')
try:
s3_client.Bucket(bucket_name).upload_file(
artifact, os.path.join(bucket_dir, artifact))
except NoCredentialsError:
logger.error('Unable to upload packages to the S3 bucket. Boto3 '
'was unable to locate credentials!')
sys.exit(1)
except (S3UploadFailedError, ClientError)as err:
logger.error('Failed to upload the package to the S3 bucket! '
f'Encountered:\n{err}')
sys.exit(1)
def main_legacy(args):
try:
possum_file = PossumFile(USER_DIR)
except scanner.ScannerError:
logger.error(f"The Possum file could not be loaded!")
sys.exit(1)
if args.docker:
if not args.docker_image:
logger.error('A Docker image must be specified')
sys.exit(1)
run_in_docker(USER_DIR, possum_file.path, args.docker_image)
sys.exit()
try:
pipenvw = PipenvWrapper()
except PipenvPathNotFound:
logger.error("'pipenv' could not be found!")
sys.exit(1)
global S3_BUCKET_NAME
global S3_ARTIFACT_DIR
S3_BUCKET_NAME, S3_ARTIFACT_DIR = get_s3_bucket_and_dir(args.s3_bucket)
try:
with open(args.template) as fobj:
template_file = YAML().load(fobj)
except Exception as error:
logger.error('Failed to load template file! Encountered: '
f'{type(error).__name__}\n')
sys.exit(1)
api_resources = dict()
lambda_functions = dict()
for resource in template_file['Resources']:
if template_file['Resources'][resource]['Type'] == \
'AWS::Serverless::Function':
runtime = \
template_file['Resources'][resource]['Properties'].get(
'Runtime',
get_global(template_file, 'Function', 'Runtime')
)
if not runtime.lower().startswith('python'):
logger.warning(
'Possum only packages Python based Lambda '
f' functions! Found runtime "{runtime}" for '
f'function "{resource}" This Lambda function '
'will be skipped. You will need to package this '
'function separately before deploying.')
continue
else:
lambda_functions[resource] = \
template_file['Resources'][resource]
elif template_file['Resources'][resource]['Type'] == \
'AWS::Serverless::Api':
# Server::Api resource. If it has a 'DefinitionUri' parameter
# that DOES NOT start with s3://, this swagger file must be
# shipped and the template updated.
if 'DefinitionUri' in \
template_file['Resources'][resource]['Properties'] \
and \
not (
template_file['Resources'][resource]['Properties']
['DefinitionUri'].startswith('s3://')
):
api_resources[resource] = template_file['Resources'][resource]
logger.info("\nThe following functions will be packaged and deployed:")
for func in lambda_functions.keys():
logger.info(f" - {func}")
logger.info("\nThe following swagger files will be deployed:")
for api in api_resources.keys():
logger.info(f" - {api}")
build_directory = tempfile.mkdtemp(suffix='-build', prefix='possum-')
logger.info(f'\nBuild directory: {build_directory}\n')
build_artifact_directory = os.path.join(build_directory, 's3_artifacts')
os.mkdir(build_artifact_directory)
for logical_id, api_resource in api_resources.items():
swagger_src = os.path.join(WORKING_DIR,
api_resource['Properties']['DefinitionUri'])
swagger_dst = os.path.join(build_artifact_directory,
f'{logical_id}.swagger')
shutil.copyfile(swagger_src, swagger_dst)
update_template_resource(template_file,
logical_id,
S3_BUCKET_NAME,
S3_ARTIFACT_DIR,
s3_object=os.path.basename(swagger_dst),
resource_param='DefinitionUri')
for func, values in lambda_functions.items():
func_source_dir = os.path.join(WORKING_DIR,
values['Properties']['CodeUri'])
func_build_dir = os.path.join(build_directory, func)
if possum_file.check_hash(func, func_source_dir) and not args.clean:
last_s3_uri = possum_file.get_last_s3_uri(func)
if last_s3_uri:
logger.info(f'{func}: No changes detected')
logger.info(f'{func}: Using S3 artifact: {last_s3_uri}\n')
update_template_resource(template_file,
func,
S3_BUCKET_NAME,
S3_ARTIFACT_DIR,
s3_uri=last_s3_uri)
continue
shutil.copytree(func_source_dir, func_build_dir)
os.chdir(func_build_dir)
logger.info(f'{func}: Working dir: {func_build_dir}')
requirements_files = ('Pipfile', 'Pipfile.lock', 'requirements.txt')
if [i for i in os.listdir('.') if i in requirements_files]:
logger.info(f'{func}: Creating virtual environment...')
pipenvw.create_virtual_environment()
pipenvw.get_virtual_environment_path()
logger.info(f'{func}: Environment created at {pipenvw.venv_path}')
do_not_copy = get_existing_site_packages(pipenvw.venv_path)
logger.info(f'{func}: Installing requirements...')
pipenvw.install_packages()
logger.info(f'{func}: Copying installed packages...')
move_installed_packages(pipenvw.venv_path, do_not_copy)
logger.info(f'{func}: Removing Lambda build environment...')
pipenvw.remove_virtualenv()
logger.info(f'{func}: Creating Lambda package...')
artifact = create_lambda_package(func_build_dir,
build_artifact_directory)
update_template_resource(template_file,
func,
S3_BUCKET_NAME,
S3_ARTIFACT_DIR,
s3_object=artifact)
possum_file.set_s3_uri(
func, template_file['Resources'][func]['Properties']['CodeUri'])
logger.info('')
upload_packages(build_artifact_directory, S3_BUCKET_NAME, S3_ARTIFACT_DIR,
args.profile)
logger.info('\nRemoving build directory...')
shutil.rmtree(build_directory)
stream = io.StringIO()
YAML().dump(template_file, stream)
deployment_template = stream.getvalue()
if not args.output_template:
logger.info('\nUpdated SAM deployment template:\n')
print(deployment_template)
else:
logger.info("Writing deployment template to "
f"'{args.output_template}'...\n")
with open(os.path.join(WORKING_DIR, args.output_template),
'wt') as fobj:
fobj.write(deployment_template)
possum_file.save()
def run_in_docker(user_dir, possum_path, image_name):
command = sys.argv
command[0] = 'possum'
command.pop(command.index('--docker'))
try:
command.pop(command.index('--docker-image') + 1)
command.pop(command.index('--docker-image'))
except ValueError:
pass
# docker_directory = tempfile.mkdtemp(
# suffix='-docker', prefix='possum-', dir='/tmp')
# logger.info(f'Working Docker directory: {docker_directory}')
client = docker.from_env()
# Copy invocation environment parameters into Docker for use-cases where
# AWS credentials are not available via the ~/.aws directory.
container_env = {
k: v
for k, v in os.environ.items()
if k in [
'AWS_ACCESS_KEY_ID',
'AWS_SECRET_ACCESS_KEY',
'AWS_SESSION_TOKEN',
]
}
logger.info('Running Docker container...')
try:
container = client.containers.create(
image=image_name,
command=command,
environment=container_env,
volumes={
os.path.join(user_dir, '.aws'): {
'bind': '/root/.aws',
'mode': 'ro'
},
os.path.dirname(possum_path): {
'bind': '/root/.possum',
'mode': 'rw'
},
os.getcwd(): {
'bind': '/var/task',
'mode': 'rw'
}
},
detach=True
)
except ImageNotFound:
logger.error(f"The Docker image '{image_name}' could not be found")
sys.exit(1)
try:
container.start()
except APIError as err:
logger.error(f'Unable to start the Docker container: {err.explanation}')
sys.exit(1)
for event in container.logs(stream=True):
logger.info(event.decode().strip())
time.sleep(.01)
container.remove()