def get(self, name, params=[]):
"""
Get template, applying params and func
An attempt is made to parse intergers, this is done so that formatting
parameters for zero passing can be used in the templates. Useful for
dealing with inconsistent date formatting.
:param name: name of template to retrieve
:param params: list of strings 'name:value' applied to all strings in template
:return: dict of selected template
"""
if name not in self.templates:
fatal("Template '{}' not found".format(name))
t = copy.deepcopy(self.templates[name])
dict_params = {}
if params:
for param in params:
key, value = param.split('=')
try:
value = int(value)
except ValueError:
pass
dict_params[key] = value
logging.info('Template parameters: {}'.format(dict_params))
try:
Templates.assign_params(t, dict_params)
except KeyError as keyError:
fatal("Missing template parameter: {}".format(keyError))
return t
def task_create(mod, func, cache):
"""
Build archive of lambda function, and deploy it
:param modname: filename containing it
:param funcname: name of function
:param region: region where it should be created and deployed
:return:
"""
cli = boto3.client('lambda')
archive_contents = build_archive(mod, cache)
try:
res = cli.create_function(Handler="{mod}.{func}".format(mod=mod, func=func),
FunctionName=func,
Runtime=AWS_LAMBDA_RUNTIME,
Role=AWS_LAMBDA_ROLE,
Code={"ZipFile": archive_contents},
Timeout=AWS_LAMBDA_TIMEOUT,
MemorySize=AWS_LAMBDA_MEMORYSIZE
)
except Exception as e:
fatal("Operation failed: {}".format(e))
logging.info("Operation completed: {}".format(res["FunctionArn"]))
def task_delete(name):
"""
Delete lambda function
:param name: function name or ARN
:return:
"""
cli = boto3.client('lambda')
try:
cli.delete_function(FunctionName=name)
except Exception as e:
fatal("Operation failed: {}".format(e))
logging.info("Operation completed.")
def task_update(mod, func, cache):
"""
Update existing lambda function
:param modname: filename containing it
:param funcname: name of function
:param region: region where it should be created and deployed
:param fast: if true, download all dependencies again from scratch
:return:
"""
cli = boto3.client('lambda')
archive_contents = build_archive(mod, cache)
try:
res = cli.update_function_code(FunctionName=func,
ZipFile=archive_contents)
except Exception as e:
fatal("Operation failed: {}".format(e))
logging.info("Operation completed: {}".format(res["FunctionArn"]))
def main():
"""
Application entry point
:return:
"""
parser = argparse.ArgumentParser(description="AWS Lambda admin tool v{}".format(__version__))
parser.add_argument('task', choices=['list', 'create', 'update', 'delete'])
parser.add_argument('--region', default=AWS_DEFAULT_REGION, help="region to consider")
parser.add_argument('--include', action='append', help="include Python script")
parser.add_argument('--mod', help="module name")
parser.add_argument('--func', help="function name")
parser.add_argument('--name', help="function to delete")
parser.add_argument('--cache', help="cache key")
args = parser.parse_args()
if args.include:
for pathname in args.include:
logging.info('Including {} ...'.format(pathname))
exec(open(pathname).read(), globals())
elif args.task == 'list':
task_list()
elif args.task == 'create':
if args.mod is None or args.func is None:
fatal("create task requires --mod or --func")
task_create(args.mod, args.func, args.cache)
elif args.task == 'update':
if args.mod is None or args.func is None:
fatal("update task requires --mod or --func")
task_update(args.mod, args.func, args.cache)
elif args.task == 'delete':
if args.name is None:
fatal("delete task requires --name")
task_delete(args.name)
else:
fatal("Task not recognized")
def main():
"""
Application entry point
:return:
"""
parser = argparse.ArgumentParser(
description="AWS EMR admin tool v{}".format(__version__))
parser.add_argument('task',
choices=[
'active', 'create', 'terminate', 'ssh', 'tunnel',
'step', 'render', 'templates'
])
parser.add_argument('--region',
default=AWS_DEFAULT_REGION,
help="region to consider")
parser.add_argument('--cluster', help="name of cluster template")
parser.add_argument('--step', help="add step to running cluster")
parser.add_argument('--bootstrap', help="name of bootstrap template")
parser.add_argument('--param', action='append', help="template parameter")
parser.add_argument('--include',
action='append',
help="include Python script")
parser.add_argument('--id', help="ID of EMR cluster")
parser.add_argument('--cmd',
default='',
help="command to execute on master node")
parser.add_argument('--tunnel',
action='store_true',
help="start tunnel once cluster running")
args = parser.parse_args()
if args.include:
for pathname in args.include:
logging.info('Including {} ...'.format(pathname))
exec(open(pathname).read(), globals())
try:
if args.task == 'active':
task_list_active(region_name=args.region)
elif args.task == 'create':
if args.cluster is None and args.id is None:
fatal("create task requires --cluster or --id")
cluster_id = task_create(region_name=args.region,
template=args.cluster,
params=args.param,
cluster_id=args.id)
if args.tunnel:
# if --tunnel, start tunnel afterwards
task_tunnel(region_name=args.region, cluster_id=cluster_id)
elif args.task == 'terminate':
task_terminate(region_name=args.region, cluster_id=args.id)
elif args.task == 'ssh':
task_ssh(region_name=args.region, cluster_id=args.id, cmd=args.cmd)
elif args.task == 'tunnel':
task_tunnel(region_name=args.region, cluster_id=args.id)
elif args.task == 'step':
task_add_step(region_name=args.region,
template=args.step,
params=args.param,
cluster_id=args.id)
elif args.task == 'render':
if args.cluster:
t, n = clusterTemplates.get(args.cluster,
args.param), "Cluster"
elif args.bootstrap:
t, n = bootstrapTemplates.get(args.bootstrap,
args.param), "Bootstrap"
elif args.step:
t, n = stepTemplates.get(args.step, args.param), "Step"
else:
fatal(
"render task requires either --cluster, --bootstrap, or --step"
)
logging.info('{n} template:\n\n{s}\n\n'.format(
n=n, s=json.dumps(t, indent=4, sort_keys=True)))
elif args.task == 'templates':
logging.info('EMR cluster templates: {}'.format(
clusterTemplates.get_names()))
logging.info('EMR step templates: {}'.format(
stepTemplates.get_names()))
logging.info('EMR bootstrap templates: {}'.format(
bootstrapTemplates.get_names()))
else:
fatal("Task not recognized")
except KeyboardInterrupt:
fatal("Interrupted")