def check_public_ip(self, env_vars, DB): thepublicip = AWSsgFuncs.get_ip_address() if thepublicip != DB['public_ip']: print('Your public_ip has changed from {0} to {1}'.format( DB['public_ip'], thepublicip)) DB['public_ip'] = thepublicip save_database(DB, env_vars['db_path'])
def attach_volume(boxname, volume_type, volume_size, env_vars, DB): region = env_vars['aws_region'] access_key = env_vars['access_key'] secret_key = env_vars['secret_key'] awsf = AWSvolFuncs(region, access_key, secret_key) params = {} try: nametuple = DB['running_instances'][boxname]['sdrive_names'].pop(0) params['name'] = nametuple[0] params['mnt'] = nametuple[1] except: raise Exception('Maximum number of volumes you can attach reached') params['az'] = DB['running_instances'][boxname]['az'] params['instance_id'] = DB['running_instances'][boxname]['id'] params['size'] = volume_size params['vtype'] = volume_type params['vol_id'] = awsf.create_volume(params) DB['running_instances'][boxname]['sdrive'][params['name']] = params save_database(DB, env_vars['db_path']) out = run_command( ['ssh', boxname, 'sudo', 'mkfs', '-t', 'xfs', params['mnt']]) drive_name = 'Project/{0}'.format( len(DB['running_instances'][boxname]['sdrive'])) out = run_command(['ssh', boxname, 'sudo', 'mkdir', '-p', drive_name]) out = run_command([ 'ssh', boxname, "echo 'sudo mount {0} ~/{1}' >> ~/.bashrc".format( params['mnt'], drive_name) ])
def start_instance(boxname, env_vars, DB): if boxname not in DB['stopped_instances']: print('the box is not available check again:') sys.exit() print('Waiting for confirmation from AWS') region = env_vars['aws_region'] home_folder = env_vars['HOME'] access_key = env_vars['access_key'] secret_key = env_vars['secret_key'] awsf = AWSec2Funcs(region, access_key, secret_key) my_ssh_key_path = env_vars['key_pair_path'] ssh_key_name = env_vars['key_pair_name'] id = DB['stopped_instances'][boxname]['id'] DB['running_instances'][boxname] = awsf.start_ec2_instance(id) DB['running_instances'][boxname]['sdrive'] = DB['stopped_instances'][ boxname]['sdrive'] del (DB['stopped_instances'][boxname]) # DB['available_names'].append(boxname) save_database(DB, env_vars['db_path']) write_into_text( boxname, ''' Host {0} HostName {1} User ubuntu IdentityFile {2} ForwardAgent yes StrictHostKeyChecking no '''.format(boxname, DB['running_instances'][boxname]['public_ip'], my_ssh_key_path), os.path.join(home_folder, '.ssh/config')) print('ec2 instance {0} started successfully'.format(boxname))
def terminate_instance(boxname, env_vars, DB): if boxname in DB['stopped_instances']: raise ValueError( textwrap.dedent('''\ Instance {name} is in stopped state and can not be terminated To terminate the instance you need to first start it: propad start {name}; then you can stop it '''.format(name=boxname))) region = env_vars['aws_region'] home_folder = env_vars['HOME'] access_key = env_vars['access_key'] secret_key = env_vars['secret_key'] awsf = AWSec2Funcs(region, access_key, secret_key) awsvolf = AWSvolFuncs(region, access_key, secret_key) for _, val in DB['running_instances'][boxname]['sdrive'].items(): id = val['vol_id'] print(val['vol_id'], val['name']) out = run_command(['ssh', boxname, 'umount', val['mnt']]) awsvolf.delete_vol(id) id = DB['running_instances'][boxname]['id'] awsf.terminate_ec2_instance(id) del (DB['running_instances'][boxname]) if boxname[0:3] == 'box': DB['available_names'].append(boxname) save_database(DB, env_vars['db_path']) delete_text_from_file(boxname, os.path.join(home_folder, '.ssh/config')) print('ec2 instance {0} terminated successfully'.format(boxname))
def stop_instance(boxname, env_vars, DB): region = env_vars['aws_region'] home_folder = env_vars['HOME'] access_key = env_vars['access_key'] secret_key = env_vars['secret_key'] awsf = AWSec2Funcs(region, access_key, secret_key) id = DB['running_instances'][boxname]['id'] awsf.stop_ec2_instance(id) DB['stopped_instances'][boxname] = DB['running_instances'][boxname] del (DB['running_instances'][boxname]) save_database(DB, env_vars['db_path']) delete_text_from_file(boxname, os.path.join(home_folder, '.ssh/config')) print('ec2 instance {0} stopped successfully'.format(boxname))
def get_box_name(DB, dbpath): """ gets redis variables names and returns the best name for the newly created instance """ if len(DB['available_names']) > 0: dname = DB['available_names'].popleft() if dname in DB['running_instances'] or\ dname in DB['stopped_instances'] : raise Exception( 'Was not able to find a proper name. Please report the bug') boxn = dname else: boxi = DB['created_instances'] + 1 boxn = 'box{0}'.format(boxi) DB['created_instances'] += 1 save_database(DB, dbpath) return boxn
def terminate_instance(boxname, env_vars, DB): if boxname in DB['stopped_instances']: raise ValueError( textwrap.dedent('''\ Instance {name} is in stopped state and can not be terminated To terminate the instance you need to first start it: propad start {name}; then you can stop it '''.format(name=boxname))) region = env_vars['aws_region'] home_folder = env_vars['HOME'] access_key = env_vars['access_key'] secret_key = env_vars['secret_key'] awsf = AWSec2Funcs(region, access_key, secret_key) id = DB['running_instances'][boxname]['id'] awsf.terminate_ec2_instance(id) del (DB['running_instances'][boxname]) if boxname[0:3] == 'box': DB['available_names'].append(boxname) save_database(DB, env_vars['db_path']) delete_text_from_file(boxname, os.path.join(home_folder, '.ssh/config')) print('ec2 instance {0} terminated successfully'.format(boxname)) # if __name__ == "__main__": # import argparse # parser = argparse.ArgumentParser(description='A function to create instance', # usage='%(prog)s [OPTIONS]') # parser.add_argument("-n", "--name", dest="boxname", default="", # help="Enter the name of the sandbox:") # args = parser.parse_args() # boxname = args.boxname # if not boxname: # print('Please enter the name of the box you want to remove') # sys.exit() # DB = load_database() # terminate_instance(boxname, DB)
def create_vpc(env_vars, DB): region = env_vars['aws_region'] access_key = env_vars['access_key'] secret_key = env_vars['secret_key'] awsf = AWSec2Funcs(region, access_key, secret_key) thename = env_vars['vpc_name'] filters = [{'Name':'tag:Name', 'Values':[thename]}] vpcs = list(awsf.ec2.vpcs.filter(Filters=filters)) if len(vpcs)==0: pass elif len(vpcs)==1 and thename in DB: print ('The VPC with the name: {0} exists.'.format(thename)) print ('will continue to use it') return elif thename not in DB: pass elif len(vpcs)>1: raise Exception('There are more than one VPCs with the name. Contact ...') else: raise Exception('Either VPC does not exists or DB has changed') cidrblock1 = '172.16.0.0/28' cidrblock2 = '172.16.0.0/28' vpc_params = awsf.create_vpc(thename, cidrblock1, cidrblock2) vpcid = vpc_params.vpc_id if type(vpc_params.vpc_id)==type(-1): raise Exception('Was not able to create VPC. Check your permissions') elif type(vpc_params.sg_id)==type('s') and type(vpc_params.vpc_id)==type(-1): awsf.delete_vpc(vpcid) raise Exception('Was able to create basic VPC but fails further down with VPC dependencies') print (vpc_params.sg_id, vpc_params.subnet_id, vpc_params.vpc_id) DB[thename] = {} DB[thename]['vpc_id'] = vpc_params.vpc_id DB[thename]['sg_id'] = vpc_params.sg_id DB[thename]['subnet_id'] = vpc_params.subnet_id save_database(DB, env_vars['db_path'])
def create_instance(boxname, boxtype, shut_down_time, env_vars, DB): update_status(env_vars, DB) region = env_vars['aws_region'] home_folder = env_vars['HOME'] access_key = env_vars['access_key'] secret_key = env_vars['secret_key'] awsf = AWSec2Funcs(region, access_key, secret_key) my_ssh_key_path = env_vars['key_pair_path'] ssh_key_name = env_vars['key_pair_name'] if not boxname: boxname = get_box_name(DB, env_vars['db_path']) else: if boxname[:3] == 'box' or \ boxname in DB['running_instances'] or \ boxname in DB['stopped_instances']: print("enter a better name. either exists or starts with box") sys.exit() params = {} params['ssh_key_name'] = ssh_key_name params['aws_ami'] = env_vars['aws_ami'] params['aws_iam_role'] = env_vars['role_name'] params['vpc'] = DB[env_vars['vpc_name']] params['box_type'] = boxtype params['name'] = env_vars['your_name'] + boxname print('wainting for confirmation from AWS') DB['running_instances'][boxname] = awsf.create_ec2_instance(params) write_into_text( boxname, ''' Host {0} HostName {1} User ubuntu IdentityFile {2} ForwardAgent yes StrictHostKeyChecking no '''.format(boxname, DB['running_instances'][boxname]['public_ip'], my_ssh_key_path), os.path.join(home_folder, '.ssh/config')) save_database(DB, env_vars['db_path']) tmp_dir = os.path.join(env_vars['env_dir'], 'tmp') tmp_tclock = os.path.join(tmp_dir, 'tclock.py') tmp_cron = os.path.join(tmp_dir, 'cron') if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) dir_path = os.path.dirname(os.path.realpath(__file__)) tmp_tclock = os.path.join(dir_path, 'scripts', 'tclock.py') with open(tmp_cron, 'wb') as f: towrite = '*/{0} * * * * python /home/ubuntu/.provisionpad/tclock.py\n'.format( shut_down_time) f.write(towrite.encode('UTF-8')) print('Setting up ec2 instance') time.sleep(30) out = run_command(['ssh', boxname, 'pip', 'install', 'psutil']) if out != 0: raise Exception('Failed to install psutil on server') out = run_command(['ssh', boxname, 'mkdir', '-p', '.provisionpad/data']) if out != 0: raise Exception('Failed to create .provisionpad/data') out = run_command( ['ssh', boxname, 'touch', '.provisionpad/data/total_idle.out']) if out != 0: raise Exception('Failed to create .provisionpad/data/total_idle.out') out = run_command( ['scp', tmp_cron, '{0}:~/.provisionpad/'.format(boxname)]) if out != 0: raise Exception('Failed to copy the crontab file') out = run_command( ['scp', tmp_tclock, '{0}:~/.provisionpad/'.format(boxname)]) if out != 0: raise Exception('Failed to copy the tmp_tclock') out = run_command( ['ssh', boxname, 'crontab', '/home/ubuntu/.provisionpad/cron']) if out != 0: raise Exception('schedule a cron job') os.remove(tmp_cron) print('ec2 instance {} created successfully'.format(boxname)) show_status(env_vars, DB)
def initiate(): home = os.path.expanduser("~") env_dir = os.path.join(home, '.provisionpad') if not os.path.isdir(env_dir): os.mkdir(env_dir) dbpath = os.path.join(env_dir, 'database.p') DB = load_database(dbpath) home = os.path.expanduser("~") env_dir = os.path.join(home, '.provisionpad') if not os.path.isdir(env_dir): os.mkdir(env_dir) env_var_path = os.path.join(env_dir, 'env_variable.json') input_var_path = os.path.join(env_dir, 'input_variable.json') if not os.path.isfile(input_var_path): env_vars = {} ask_for_credentials = True print ('Initiating a new propad environment') print ('Searching for default AWS credentials...') session = boto3.Session() credentials = session.get_credentials() if credentials: use_default_cred = input(textwrap.dedent('''\ Default AWS credentials found. Do you want to use them?(y/n) ''')) if str(use_default_cred).strip()[0] == 'y': env_vars['access_key'] = credentials.access_key env_vars['secret_key'] = credentials.secret_key ask_for_credentials = False if ask_for_credentials: print (' You can find aws access keys under user tab (top third from right)') print (' My security credentials for the root info or under IAM users section') print (' For more information please visit: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html') access_key = input('Enter AWS access key ID: ') env_vars['access_key'] = str(access_key).strip() if not env_vars['access_key']: print ('Invalid input') sys.exit() secret_key = input('Enter AWS secret access key: ') env_vars['secret_key'] = str(secret_key).strip() if not env_vars['secret_key']: print ('Invalid input') sys.exit() your_name = input('Enter your name (optional, will be added as a tag to the instance): ') env_vars['your_name'] = re.sub('[^a-zA-Z0-9]', '', your_name).upper() if not env_vars['your_name']: print ('Invalid input') sys.exit() # env_vars['your_email'] = input('Enter your email (): ') print ('\n') print ('Note: AMI (Image) should be in the same defined AWS region.') env_vars['aws_region'] = input ('Enter AWS region (us-east-2): ') if not env_vars['aws_region']: env_vars['aws_region'] = 'us-east-2' env_vars['aws_ami'] = input ('Enter your AWS AMI. (Ubuntu 18): ') if not env_vars['aws_ami']: env_vars['aws_ami'] = 'ami-029f8374ffdc9a057' #'ami-00df714b389c23925' with open(input_var_path, 'w') as f: json.dump(env_vars, f, indent=4) else: with open(input_var_path, 'r') as f: env_vars = json.load(f) env_vars['db_path'] = dbpath env_vars['env_path'] = env_var_path env_vars['env_dir'] = env_dir key_pair_name = 'ec2_keypair_{0}_{1}.pem'.format(env_vars['your_name'], env_vars['aws_region']) key_pair_path = os.path.join(env_dir, key_pair_name) env_vars['key_pair_name'] = key_pair_name env_vars['key_pair_path'] = key_pair_path env_vars['vpc_name'] = '{0}_VPC'.format(env_vars['your_name']) role_name = [env_vars['your_name'] ] policies = ['S3FULL'] role_name.extend(policies) role_names = ''.join(role_name) env_vars['policy'] = policies env_vars['role_name'] = role_names env_vars['HOME'] = home # get the public ip address of local machine DB['public_ip'] = AWSsgFuncs.get_ip_address() save_database(DB, env_vars['db_path']) create_vpc(env_vars, DB) awsec2f = AWSec2Funcs(env_vars['aws_region'], env_vars['access_key'], env_vars['secret_key']) awsstsf = AWSstsFuncs(env_vars['aws_region'], env_vars['access_key'], env_vars['secret_key']) awsiamf = AWSiamFuncs(env_vars['aws_region'], env_vars['access_key'], env_vars['secret_key']) if not os.path.isfile(env_vars['key_pair_path']): if not awsec2f.check_key_pair(env_vars['key_pair_name']): try: print ('creating key pair') with open(env_vars['key_pair_path'], 'w') as f: key_pair = str(awsec2f.create_key_pair(key_pair_name)) print (key_pair) f.write(key_pair) os.chmod(env_vars['key_pair_path'], 0o600) except: os.remove(env_vars['key_pair_path']) raise Exception('You do not have access to create key-pair check your permissions') else: raise Exception('we can find the public key but pem is not available') else: print ('the key pair exists') account_id = awsstsf.get_account_id() policy_attach = [] for policy in env_vars['policy']: policy_arn = 'arn:aws:iam::{0}:policy/{1}'.format(account_id, policy ) if not awsiamf.check_policy_exists(policy_arn): if policy == 'S3FULL': awsiamf.ec2_policy_access_full(policy) policy_attach.append(policy_arn) else: print ('the policy {0} not implemented yet'.format(policy)) else: print ('the policy {0} exists'.format(policy)) policy_attach.append(policy_arn) if not awsiamf.check_role_exists(env_vars['role_name']): print (awsiamf.create_role_for_ec2(env_vars['role_name']) ) awsiamf.create_instance_profile(env_vars['role_name']) if awsiamf.check_role_exists(env_vars['role_name'], 1, 5): for policy in policy_attach: print ('attaching policy arn: {0}'.format(policy)) awsiamf.attach_policy_to_role(env_vars['role_name'], policy) print ('policy attached') else: raise Exception(' was not able to find the role') with open(env_var_path, 'w') as f: json.dump(env_vars, f, indent=4)
def update_status(env_vars, DB): region = env_vars['aws_region'] home_folder = env_vars['HOME'] access_key = env_vars['access_key'] secret_key = env_vars['secret_key'] awsec2f = AWSec2Funcs(region, access_key, secret_key) aws_inst_info = awsec2f.instance_state(env_vars['your_name']) aws_inst_info_d = deepcopy(aws_inst_info) clean_propad_from_file(os.path.join(home_folder, '.ssh/config')) for ins in aws_inst_info: if aws_inst_info[ins][0] == 'terminated': aws_inst_info_d.pop(ins) if not (aws_inst_info[ins][0] == 'stopped' or aws_inst_info[ins][0] == 'running' or aws_inst_info[ins][0] == 'terminated'): print('Waiting to complete the transition. Try again a bit later.') sys.exit() aws_inst_info = aws_inst_info_d DBD = deepcopy(DB) for ins, ins_info in DBD['running_instances'].items(): if ins_info['id'] not in aws_inst_info: print('seems like the instance you have created ') print('has been removed from the aws manually most likely') print('removing it from the database') del (DB['running_instances'][ins]) if ins[0:3] == 'box': DB['available_names'].append(ins) save_database(DB, env_vars['db_path']) delete_text_from_file(ins, os.path.join(home_folder, '.ssh/config')) elif ins_info['id'] in aws_inst_info and aws_inst_info[ ins_info['id']][0] == 'stopped': print('seems like the instance has been stopped') print('removing it from the running instances') DB['stopped_instances'][ins] = DB['running_instances'][ins] del (DB['running_instances'][ins]) save_database(DB, env_vars['db_path']) delete_text_from_file(ins, os.path.join(home_folder, '.ssh/config')) else: write_into_text( ins, ''' Host {0} HostName {1} User ubuntu IdentityFile {2} ForwardAgent yes StrictHostKeyChecking no '''.format(ins, DB['running_instances'][ins]['public_ip'], env_vars['key_pair_path']), os.path.join(home_folder, '.ssh/config')) print('{0} is fine as expected'.format(ins)) DBD = deepcopy(DB) for ins, ins_info in DBD['stopped_instances'].items(): if ins_info['id'] not in aws_inst_info: print('seems like the instance you have created ') print('has been removed from the aws manually most likely') print('removing it from the database') del (DB['stopped_instances'][ins]) if ins[0:3] == 'box': DB['available_names'].append(ins) save_database(DB, env_vars['db_path']) delete_text_from_file(ins, os.path.join(home_folder, '.ssh/config')) elif ins_info['id'] in aws_inst_info and aws_inst_info[ ins_info['id']][0] == 'running': print('seems like the instance has started manually') print('moving from stopped to running') DB['running_instances'][ins] = DB['stopped_instances'][ins] DB['running_instances'][ins]['public_ip'] = aws_inst_info[ ins_info['id']][1] del (DB['stopped_instances'][ins]) save_database(DB, env_vars['db_path']) write_into_text( ins, ''' Host {0} HostName {1} User ubuntu IdentityFile {2} ForwardAgent yes StrictHostKeyChecking no '''.format(ins, DB['running_instances'][ins]['public_ip'], env_vars['key_pair_path']), os.path.join(home_folder, '.ssh/config')) else: print('{0} is fine as expected'.format(ins)) ids_db = set([]) for x in DB['running_instances']: ids_db.add(DB['running_instances'][x]['id']) for x in DB['stopped_instances']: ids_db.add(DB['stopped_instances'][x]['id']) to_create = set(aws_inst_info) - ids_db for x in to_create: ins = get_box_name(DB, env_vars['db_path']) thekeyname = 'stopped_instances' if aws_inst_info[x][0] == 'running': thekeyname = 'running_instances' boxname = get_box_name(DB, env_vars['db_path']) DB[thekeyname][boxname] = awsec2f.get_instance_info(x) write_into_text( ins, ''' Host {0} HostName {1} User ubuntu IdentityFile {2} ForwardAgent yes StrictHostKeyChecking no '''.format(ins, DB[thekeyname][boxname]['public_ip'], env_vars['key_pair_path']), os.path.join(home_folder, '.ssh/config')) if len(ids_db - set(aws_inst_info)) > 0: raise Exception('this should not happen in status') save_database(DB, env_vars['db_path'])
def create_instance(boxname, boxtype, shut_down_time, env_vars, DB): update_status(env_vars, DB) region = env_vars['aws_region'] home_folder = env_vars['HOME'] access_key = env_vars['access_key'] secret_key = env_vars['secret_key'] awsf = AWSec2Funcs(region, access_key, secret_key) my_ssh_key_path = env_vars['key_pair_path'] ssh_key_name = env_vars['key_pair_name'] if not boxname: boxname = get_box_name(DB, env_vars['db_path']) else: if boxname[:3] == 'box' or \ boxname in DB['running_instances'] or \ boxname in DB['stopped_instances']: print("enter a better name. either exists or starts with box") sys.exit() params = {} params['ssh_key_name'] = ssh_key_name params['aws_ami'] = env_vars['aws_ami'] params['aws_iam_role'] = env_vars['role_name'] params['vpc'] = DB[env_vars['vpc_name']] params['box_type'] = boxtype params['name'] = env_vars['your_name'] + boxname print('Waiting for confirmation from AWS') DB['running_instances'][boxname] = awsf.create_ec2_instance(params) write_into_text( boxname, ''' Host {0} HostName {1} User ubuntu IdentityFile {2} ForwardAgent yes StrictHostKeyChecking no '''.format(boxname, DB['running_instances'][boxname]['public_ip'], my_ssh_key_path), os.path.join(home_folder, '.ssh/config')) drivel = 'fgh' DB['running_instances'][boxname]['sdrive_names'] = [ (params['name'] + 'VOL{0}'.format(i), '/dev/xvd{0}'.format(drivel[i])) for i in range(len(drivel)) ] DB['running_instances'][boxname]['sdrive'] = {} save_database(DB, env_vars['db_path']) tmp_dir = os.path.join(env_vars['env_dir'], 'tmp') tmp_tclock = os.path.join(tmp_dir, 'tclock.py') tmp_cron = os.path.join(tmp_dir, 'cron') if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) dir_path = os.path.dirname(os.path.realpath(__file__)) tmp_tclock = os.path.join(dir_path, 'scripts', 'tclock.py') # The following line is OS dependent if it changes from ubuntu to something else then # It will cause trouble for auto shutdown -- Amir with open(tmp_cron, 'wb') as f: towrite = '*/{0} * * * * python /home/ubuntu/.provisionpad/tclock.py\n'.format( shut_down_time) f.write(towrite.encode('UTF-8')) print('Setting up EC2 instance') awssgf = AWSsgFuncs(region, access_key, secret_key) awssgf.check_public_ip(env_vars, DB) vpcparams = DB[env_vars['vpc_name']] awssgf.revoke_sg_permissions_all(vpcparams['vpc_id']) awssgf.set_sg_sshonly_local_ip(vpcparams['sg_id'], DB['public_ip']) awssgf.set_sg_http_egress(vpcparams['sg_id']) # some times it has problem ssh ing into instance # I have added this 30 seconds sleep to be safe # Is there a better way? -- Amir time.sleep(30) out = run_command(['ssh', boxname, 'pip', 'install', 'psutil']) if out != 0: raise Exception('Failed to install psutil on server') out = run_command(['ssh', boxname, 'mkdir', '-p', '.provisionpad/data']) if out != 0: raise Exception('Failed to create .provisionpad/data') out = run_command( ['ssh', boxname, 'touch', '.provisionpad/data/total_idle.out']) if out != 0: raise Exception('Failed to create .provisionpad/data/total_idle.out') out = run_command( ['scp', tmp_cron, '{0}:~/.provisionpad/'.format(boxname)]) if out != 0: raise Exception('Failed to copy the crontab file') out = run_command( ['scp', tmp_tclock, '{0}:~/.provisionpad/'.format(boxname)]) if out != 0: raise Exception('Failed to copy the tmp_tclock') out = run_command( ['ssh', boxname, 'crontab', '/home/ubuntu/.provisionpad/cron']) if out != 0: raise Exception('schedule a cron job') os.remove(tmp_cron) awssgf.revoke_sg_permissions_all(vpcparams['vpc_id']) awssgf.set_sg_sshonly_local_ip(vpcparams['sg_id'], DB['public_ip']) print('ec2 instance {} created successfully'.format(boxname)) show_status(env_vars, DB)