try:
print('\nhelp msg from spark-ec2 script:\n')
stdout, stderr = runScript('{} -h'.format(ec2_spark_script), [], output_opt='display')
except ScriptException as se:
print(se)
exit()
mode_keys = None
if args.launch:
mode_keys = DEFAULT_LAUNCH_ARGS
elif args.login:
mode_keys = DEFAULT_LOGIN_ARGS
elif args.destroy:
mode_keys = DEFAULT_DESTROY_ARGS
else:
log.printf('unknown mode!', type='ERROR')
exit()
second_lvl_arg_dict = {k:DEFAULT_EC2_ARGS[k] for k in mode_keys}
second_lvl_arg_list = map(lambda i: '{} {}'.format(list(second_lvl_arg_dict.keys())[i],
list(second_lvl_arg_dict.values())[i]), range(len(second_lvl_arg_dict)))
# NOTE: don't use '+=', cuz if the same arg is overwritten in cmd line,
# you need to put the default value prior to the overwritten one
args.spark_ec2_flag = ' {} {}'.format(' '.join(second_lvl_arg_list), args.spark_ec2_flag)
args.spark_ec2_flag += ' -i {} -k {}' \
.format(args.identity_file, args.identity_file.split('.pem')[0].split('/')[-1])
print('args to spark-ec2 script: \n\t{}'.format(args.spark_ec2_flag))
ec2_credential = args.credential_file
aws_access_key_id = None
aws_secret_access_key = None
if __name__ == '__main__':
args = parseArgs()
# setup cli settings
try:
scriptGetKey = """
set -eu
credential_file={}
echo $(cat $credential_file | awk 'NR==2' | awk -F ',' '{{print $(NF-1)}}')
echo $(cat $credential_file | awk 'NR==2' | awk -F ',' '{{print $NF}}')
""".format(args.credential_file)
stdout, stderr = runScript(scriptGetKey, [], output_opt='pipe')
key_id, secret_key = stdout.decode('utf-8').split('\n')[:-1]
stdout, stderr = runScript('aws configure', [], output_opt='display',
input_opt='pipe', input_pipe=[key_id, secret_key, args.region, OUTPUT_FORMAT])
print()
log.printf('AWS conf done', type='INFO')
except ScriptException as se:
print(se)
#########################################
# find master id, then get public-dns #
#########################################
master_dns = ''
try:
stdout, stderr = runScript('aws ec2 describe-instances', [], output_opt='pipe')
#inst = json.loads(stdout.decode('utf-8'))
log.printf('instance info got, target: {}-master' \
.format(args.cluster_name), type='INFO')
out_str = stdout.decode('utf-8')
master_id_regex = '{}-master-{}'.format(args.cluster_name, '\S*')