def start_servers(processes, ami_id, user_data, setup_disks, instance_type,
obs_ids, created_by, name, instance_details, spot_price,
frequency_channels, force):
cvel_data = get_cvel()
# Create the queue
tasks = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(processes):
consumer = Consumer(tasks)
consumer.start()
counter = 1
for obs_id in obs_ids:
snapshot_id = OBS_IDS.get(obs_id)
if snapshot_id is None:
LOGGER.warning(
'The obs-id: {0} does not exist in the settings file')
else:
obs_id_dashes = obs_id.replace('_', '-')
for frequency_groups in get_frequency_groups(
frequency_channels, obs_id_dashes, cvel_data, force):
tasks.put(
Task(ami_id, user_data, setup_disks, instance_type, obs_id,
snapshot_id, created_by, name, spot_price,
instance_details, frequency_groups, counter))
counter += 1
# Add a poison pill to shut things down
for x in range(processes):
tasks.put(None)
# Wait for the queue to terminate
tasks.join()
def start_servers(
processes,
ami_id,
user_data,
setup_disks,
instance_type,
obs_ids,
created_by,
name,
instance_details,
spot_price,
frequency_channels,
force):
cvel_data = get_cvel()
# Create the queue
tasks = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(processes):
consumer = Consumer(tasks)
consumer.start()
counter = 1
for obs_id in obs_ids:
snapshot_id = OBS_IDS.get(obs_id)
if snapshot_id is None:
LOGGER.warning('The obs-id: {0} does not exist in the settings file')
else:
obs_id_dashes = obs_id.replace('_', '-')
for frequency_groups in get_frequency_groups(frequency_channels, obs_id_dashes, cvel_data, force):
tasks.put(
Task(
ami_id,
user_data,
setup_disks,
instance_type,
obs_id,
snapshot_id,
created_by,
name,
spot_price,
instance_details,
frequency_groups,
counter
)
)
counter += 1
# Add a poison pill to shut things down
for x in range(processes):
tasks.put(None)
# Wait for the queue to terminate
tasks.join()
def check_args(args):
"""
Check the arguments and prompt for new ones
"""
map_args = {}
if args['obs_ids'] is None:
return None
elif len(args['obs_ids']) == 1 and args['obs_ids'][0] == '*':
map_args['obs_ids'] = OBS_IDS.keys()
else:
map_args['obs_ids'] = args['obs_ids']
if args['instance_type'] is None:
return None
if args['name'] is None:
return None
instance_details = AWS_INSTANCES.get(args['instance_type'])
if instance_details is None:
LOGGER.error('The instance type {0} is not supported.'.format(
args['instance_type']))
return None
else:
LOGGER.info(
'instance: {0}, vCPU: {1}, RAM: {2}GB, Disks: {3}x{4}GB, IOPS: {5}'
.format(args['instance_type'], instance_details.vCPU,
instance_details.memory, instance_details.number_disks,
instance_details.size, instance_details.iops_support))
map_args.update({
'ami_id':
args['ami_id'] if args['ami_id'] is not None else AWS_AMI_ID,
'created_by':
args['created_by']
if args['created_by'] is not None else getpass.getuser(),
'spot_price':
args['spot_price'] if args['spot_price'] is not None else None,
'user_data':
get_script(args['bash_script']
if args['bash_script'] is not None else BASH_SCRIPT_CVEL),
'setup_disks':
get_script(BASH_SCRIPT_SETUP_DISKS),
'instance_details':
instance_details,
})
return map_args
def check_args(args):
"""
Check the arguments and prompt for new ones
"""
map_args = {}
if args['obs_ids'] is None:
return None
elif len(args['obs_ids']) == 1 and args['obs_ids'][0] == '*':
map_args['obs_ids'] = OBS_IDS.keys()
else:
map_args['obs_ids'] = args['obs_ids']
if args['instance_type'] is None:
return None
if args['name'] is None:
return None
instance_details = AWS_INSTANCES.get(args['instance_type'])
if instance_details is None:
LOGGER.error('The instance type {0} is not supported.'.format(args['instance_type']))
return None
else:
LOGGER.info(
'instance: {0}, vCPU: {1}, RAM: {2}GB, Disks: {3}x{4}GB, IOPS: {5}'.format(
args['instance_type'],
instance_details.vCPU,
instance_details.memory,
instance_details.number_disks,
instance_details.size,
instance_details.iops_support))
map_args.update({
'ami_id': args['ami_id'] if args['ami_id'] is not None else AWS_AMI_ID,
'created_by': args['created_by'] if args['created_by'] is not None else getpass.getuser(),
'spot_price': args['spot_price'] if args['spot_price'] is not None else None,
'user_data': get_script(args['bash_script'] if args['bash_script'] is not None else BASH_SCRIPT_CVEL),
'setup_disks': get_script(BASH_SCRIPT_SETUP_DISKS),
'instance_details': instance_details,
})
return map_args
def start_servers(
ami_id,
user_data,
setup_disks,
instance_type,
obs_id,
created_by,
name,
instance_details,
spot_price):
snapshot_id = OBS_IDS.get(obs_id)
if snapshot_id is None:
LOGGER.warning('The obs-id: {0} does not exist in the settings file')
else:
ec2_helper = EC2Helper()
iops = None
if instance_details.iops_support:
iops = 500
zone = ec2_helper.get_cheapest_spot_price(instance_type, spot_price)
if zone is not None:
volume, snapshot_name = ec2_helper.create_volume(snapshot_id, zone, iops=iops)
LOGGER.info('obs_id: {0}, volume_name: {1}'.format(obs_id, snapshot_name))
now = datetime.datetime.now()
user_data_mime = get_mime_encoded_user_data(volume.id, setup_disks, user_data, now.strftime('%Y-%m-%dT%H-%M-%S'))
if spot_price is not None:
ec2_helper.run_spot_instance(
ami_id,
spot_price,
user_data_mime,
instance_type,
volume.id,
created_by,
'{1}-{0}'.format(name, snapshot_name),
instance_details,
zone,
ephemeral=True)
else:
LOGGER.error('Cannot get a spot instance of {0} for ${1}'.format(instance_type, spot_price))
def start_servers(ami_id, user_data, setup_disks, instance_type, obs_id,
created_by, name, instance_details, spot_price):
snapshot_id = OBS_IDS.get(obs_id)
if snapshot_id is None:
LOGGER.warning('The obs-id: {0} does not exist in the settings file')
else:
ec2_helper = EC2Helper()
iops = None
if instance_details.iops_support:
iops = 500
zone = ec2_helper.get_cheapest_spot_price(instance_type, spot_price)
if zone is not None:
volume, snapshot_name = ec2_helper.create_volume(snapshot_id,
zone,
iops=iops)
LOGGER.info('obs_id: {0}, volume_name: {1}'.format(
obs_id, snapshot_name))
now = datetime.datetime.now()
user_data_mime = get_mime_encoded_user_data(
volume.id, setup_disks, user_data,
now.strftime('%Y-%m-%dT%H-%M-%S'))
if spot_price is not None:
ec2_helper.run_spot_instance(ami_id,
spot_price,
user_data_mime,
instance_type,
volume.id,
created_by,
'{1}-{0}'.format(
name, snapshot_name),
instance_details,
zone,
ephemeral=True)
else:
LOGGER.error('Cannot get a spot instance of {0} for ${1}'.format(
instance_type, spot_price))
