def demand(opts, conf): run_args = { 'image_id': utils.get_opt(opts.ami, conf, 'AMI_ID', must_exist=True), 'max_count': opts.n_instances, 'instance_type': utils.get_opt(opts.instance_type, conf, 'INSTANCE_TYPE', must_exist=True), 'instance_profile_name': utils.get_opt(opts.instance_profile, conf, 'INSTANCE_PROFILE', must_exist=True), 'user_data': startup_script(opts, conf) if not opts.idle else None, 'key_name': conf.get("SSH_KEY_NAME", "brenda"), 'security_groups': [conf.get("SECURITY_GROUP", "brenda")], 'monitoring_enabled': opts.monitoring, 'dry_run': opts.dry_run, } logging.debug('Instance parameters: %s', run_args) node.get_done(opts, conf) # sanity check on DONE var # Request instances ec2 = aws.get_ec2_conn(conf) reservation = ec2.run_instances(**run_args) logging.info(reservation) if not opts.dry_run: tag_demand_resources(conf, reservation, dict(opts.tags))
def create_instance_with_ebs(opts, conf, new): ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True) itype = utils.get_opt(opts.ebs_manage_instance_type, conf, 'EBS_MANAGE_INSTANCE_TYPE', default="t2.micro") zone = utils.get_opt(opts.ebs_manage_availability_zone, conf, 'EBS_MANAGE_AVAILABILITY_ZONE') ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") sec_groups = (conf.get("SECURITY_GROUP", "brenda"), ) blkprops = {} if new: blkprops['size'] = opts.size else: if opts.size > 1: blkprops['size'] = opts.size if not opts.snapshot: raise ValueError("--snapshot must be specified") blkprops['snapshot_id'] = aws.translate_snapshot_name( conf, opts.snapshot) bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping() bdm[utils.blkdev(0)] = boto.ec2.blockdevicemapping.EBSBlockDeviceType( delete_on_termination=False, **blkprops) istore_dev = aws.add_instance_store(opts, conf, bdm, itype) script = None if opts.mount: dev = utils.blkdev(0, mount_form=True) script = "#!/bin/bash\n" if new: script += "/sbin/mkfs -t ext4 %s\n" % (dev, ) script += "/bin/mount %s /mnt\n" % (dev, ) run_args = { 'image_id': ami_id, 'instance_type': itype, 'key_name': ssh_key_name, 'security_groups': sec_groups, 'placement': zone, 'block_device_map': bdm, } if script: run_args['user_data'] = script print "RUN ARGS" for k, v in sorted(run_args.items()): print " %s : %r" % (k, v) print "BLK DEV PROPS", blkprops print "ISTORE DEV", istore_dev if not opts.dry_run: ec2 = aws.get_ec2_conn(conf) reservation = ec2.run_instances(**run_args) print reservation
def spot(opts, conf): ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True) price = utils.get_opt(opts.price, conf, 'BID_PRICE', must_exist=True) reqtype = 'persistent' if opts.persistent else 'one-time' itype = brenda_instance_type(opts, conf) snapshots = aws.get_snapshots(conf) bdm, snap_description, istore_dev = aws.blk_dev_map( opts, conf, itype, snapshots) script = startup_script(opts, conf, istore_dev) user_data = None if not opts.idle: user_data = script ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") sec_groups = (conf.get("SECURITY_GROUP", "brenda"), ) run_args = { 'image_id': ami_id, 'price': price, 'type': reqtype, 'count': opts.n_instances, 'instance_type': itype, 'user_data': user_data, 'key_name': ssh_key_name, 'security_groups': sec_groups, 'block_device_map': bdm, } if opts.availability_zone: run_args['placement'] = opts.availability_zone print "----------------------------" print "AMI ID:", ami_id print "Max bid price", price print "Request type:", reqtype print "Instance type:", itype print "Instance count:", opts.n_instances if opts.availability_zone: print "Availability zone:", opts.availability_zone if snap_description: print "Project EBS snapshot:", snap_description if istore_dev: print "Instance store device:", istore_dev print "SSH key name:", ssh_key_name print "Security groups:", sec_groups print_script(opts, conf, script) aws.get_done(opts, conf) # sanity check on DONE var if not opts.dry_run: ec2 = aws.get_ec2_conn(conf) reservation = ec2.request_spot_instances(**run_args) print reservation
def create_instance_with_ebs(opts, conf, new): ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True) itype = utils.get_opt(opts.ebs_manage_instance_type, conf, 'EBS_MANAGE_INSTANCE_TYPE', default="t1.micro") zone = utils.get_opt(opts.ebs_manage_availability_zone, conf, 'EBS_MANAGE_AVAILABILITY_ZONE') ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") sec_groups = (conf.get("SECURITY_GROUP", "brenda"),) blkprops = {} if new: blkprops['size'] = opts.size else: if opts.size > 1: blkprops['size'] = opts.size if not opts.snapshot: raise ValueError("--snapshot must be specified") blkprops['snapshot_id'] = aws.translate_snapshot_name(conf, opts.snapshot) bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping() bdm[utils.blkdev(0)] = boto.ec2.blockdevicemapping.EBSBlockDeviceType(delete_on_termination=False, **blkprops) istore_dev = aws.add_instance_store(opts, conf, bdm, itype) script = None if opts.mount: dev = utils.blkdev(0, mount_form=True) script = "#!/bin/bash\n" if new: script += "/sbin/mkfs -t ext4 %s\n" % (dev,) script += "/bin/mount %s /mnt\n" % (dev,) run_args = { 'image_id' : ami_id, 'instance_type' : itype, 'key_name' : ssh_key_name, 'security_groups' : sec_groups, 'placement' : zone, 'block_device_map' : bdm, } if script: run_args['user_data'] = script print "RUN ARGS" for k, v in sorted(run_args.items()): print " %s : %r" % (k, v) print "BLK DEV PROPS", blkprops print "ISTORE DEV", istore_dev if not opts.dry_run: ec2 = aws.get_ec2_conn(conf) reservation = ec2.run_instances(**run_args) print reservation
def spot(opts, conf): ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True) price = utils.get_opt(opts.price, conf, 'BID_PRICE', must_exist=True) reqtype = 'persistent' if opts.persistent else 'one-time' itype = brenda_instance_type(opts, conf) snapshots = aws.get_snapshots(conf) bdm, snap_description, istore_dev = aws.blk_dev_map(opts, conf, itype, snapshots) script = startup_script(opts, conf, istore_dev) user_data = None if not opts.idle: user_data = script ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") sec_groups = (conf.get("SECURITY_GROUP", "brenda"),) run_args = { 'image_id' : ami_id, 'price' : price, 'type' : reqtype, 'count' : opts.n_instances, 'instance_type' : itype, 'user_data' : user_data, 'key_name' : ssh_key_name, 'security_groups' : sec_groups, 'block_device_map' : bdm, } if opts.availability_zone: run_args['placement'] = opts.availability_zone print "----------------------------" print "AMI ID:", ami_id print "Max bid price", price print "Request type:", reqtype print "Instance type:", itype print "Instance count:", opts.n_instances if opts.availability_zone: print "Availability zone:", opts.availability_zone if snap_description: print "Project EBS snapshot:", snap_description if istore_dev: print "Instance store device:", istore_dev print "SSH key name:", ssh_key_name print "Security groups:", sec_groups print_script(opts, conf, script) aws.get_done(opts, conf) # sanity check on DONE var if not opts.dry_run: ec2 = aws.get_ec2_conn(conf) reservation = ec2.request_spot_instances(**run_args) print reservation
def ssh_args(opts, conf): user = utils.get_opt(opts.user, conf, "AWS_USER", default="root") args = ["ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "LogLevel=quiet"] if user: args.extend(["-o", "User="******"-i", aws.get_adaptive_ssh_identity_fn(opts, conf)]) return args
def ssh_args(opts, conf): user = utils.get_opt(opts.user, conf, 'AWS_USER', default='root') args = ['ssh', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', '-o', 'LogLevel=quiet'] if user: args.extend(['-o', 'User='******'-i', aws.get_adaptive_ssh_identity_fn(opts, conf)]) return args
def worker(): while True: try: item = q.get(block=False) except Queue.Empty, e: break else: node, cmd = item #output = utils.system_return_output(cmd, capture_stderr=capture_stderr) #new ssh on windows code------------------------------------- import paramiko #read ssh args from end of cmd object immediately following the node hostname entry sshArgs = " ".join(cmd[cmd.index(node)+1:]) #print "sshargs: " +ssArgs #get username #print "trying new user" user = utils.get_opt(opts.user, conf, 'AWS_USER', default='ubuntu') #print "user: "******"cd /mnt/brenda", sshArgs] for command in commands: stdin , stdout, stderr = c.exec_command(command) output = stdout.read() c.close() data = (node, output) with lock: if show_output: print "------- %s\n%s" % data, ret.append(data) q.task_done() except Exception as e: print e q.task_done()
def ssh_args(opts, conf): user = utils.get_opt(opts.user, conf, 'AMI_USER', default='root') args = [ 'ssh', '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', '-o', 'LogLevel=quiet' ] if user: args.extend(['-o', 'User='******'brenda-run init'?") args.extend(['-i', ssh_local_fn]) return args
def price(opts, conf): ec2 = aws.get_ec2_conn(conf) itype = utils.get_opt(opts.instance_type, conf, 'INSTANCE_TYPE', must_exist=True) data = {} for item in ec2.get_spot_price_history(instance_type=itype, product_description="Linux/UNIX"): # show the most recent price for each availability zone if item.availability_zone in data: if item.timestamp > data[item.availability_zone].timestamp: data[item.availability_zone] = item else: data[item.availability_zone] = item print "Spot price data for instance", itype for k, v in sorted(data.items()): print "%s %s $%s" % (v.availability_zone, v.timestamp, v.price)
def demand(opts, conf): ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True) itype = brenda_instance_type(opts, conf) snapshots = aws.get_snapshots(conf) bdm, snap_description, istore_dev = aws.blk_dev_map( opts, conf, itype, snapshots) script = startup_script(opts, conf, istore_dev) user_data = None if not opts.idle: user_data = script ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") sec_groups = (conf.get("SECURITY_GROUP", "brenda"), ) run_args = { 'image_id': ami_id, 'max_count': opts.n_instances, 'instance_type': itype, 'user_data': user_data, 'key_name': ssh_key_name, 'security_groups': sec_groups, 'block_device_map': bdm, } print "----------------------------" print "AMI ID:", ami_id print "Instance type:", itype print "Max instances:", opts.n_instances if snap_description: print "Project EBS snapshot:", snap_description if istore_dev: print "Instance store device:", istore_dev print "SSH key name:", ssh_key_name print "Security groups:", sec_groups print_script(opts, conf, script) aws.get_done(opts, conf) # sanity check on DONE var if not opts.dry_run: ec2 = aws.get_ec2_conn(conf) reservation = ec2.run_instances(**run_args) print reservation
def demand(opts, conf): ami_id = utils.get_opt(opts.ami, conf, 'AMI_ID', default=AMI_ID, must_exist=True) itype = brenda_instance_type(opts, conf) snapshots = aws.get_snapshots(conf) bdm, snap_description, istore_dev = aws.blk_dev_map(opts, conf, itype, snapshots) script = startup_script(opts, conf, istore_dev) user_data = None if not opts.idle: user_data = script ssh_key_name = conf.get("SSH_KEY_NAME", "brenda") sec_groups = (conf.get("SECURITY_GROUP", "brenda"),) run_args = { 'image_id' : ami_id, 'max_count' : opts.n_instances, 'instance_type' : itype, 'user_data' : user_data, 'key_name' : ssh_key_name, 'security_groups' : sec_groups, 'block_device_map' : bdm, } print "----------------------------" print "AMI ID:", ami_id print "Instance type:", itype print "Max instances:", opts.n_instances if snap_description: print "Project EBS snapshot:", snap_description if istore_dev: print "Instance store device:", istore_dev print "SSH key name:", ssh_key_name print "Security groups:", sec_groups print_script(opts, conf, script) aws.get_done(opts, conf) # sanity check on DONE var if not opts.dry_run: ec2 = aws.get_ec2_conn(conf) reservation = ec2.run_instances(**run_args) print reservation
def brenda_instance_type(opts, conf): return utils.get_opt(opts.instance_type, conf, 'INSTANCE_TYPE', default="m2.xlarge")