def start_servers(processes, ami_id, user_data, setup_disks, instance_type, obs_ids, created_by, name, instance_details, spot_price, frequency_channels, force): cvel_data = get_cvel() # Create the queue tasks = multiprocessing.JoinableQueue() # Start the consumers for x in range(processes): consumer = Consumer(tasks) consumer.start() counter = 1 for obs_id in obs_ids: snapshot_id = OBS_IDS.get(obs_id) if snapshot_id is None: LOGGER.warning( 'The obs-id: {0} does not exist in the settings file') else: obs_id_dashes = obs_id.replace('_', '-') for frequency_groups in get_frequency_groups( frequency_channels, obs_id_dashes, cvel_data, force): tasks.put( Task(ami_id, user_data, setup_disks, instance_type, obs_id, snapshot_id, created_by, name, spot_price, instance_details, frequency_groups, counter)) counter += 1 # Add a poison pill to shut things down for x in range(processes): tasks.put(None) # Wait for the queue to terminate tasks.join()
def start_servers( processes, ami_id, user_data, setup_disks, instance_type, obs_ids, created_by, name, instance_details, spot_price, frequency_channels, force): cvel_data = get_cvel() # Create the queue tasks = multiprocessing.JoinableQueue() # Start the consumers for x in range(processes): consumer = Consumer(tasks) consumer.start() counter = 1 for obs_id in obs_ids: snapshot_id = OBS_IDS.get(obs_id) if snapshot_id is None: LOGGER.warning('The obs-id: {0} does not exist in the settings file') else: obs_id_dashes = obs_id.replace('_', '-') for frequency_groups in get_frequency_groups(frequency_channels, obs_id_dashes, cvel_data, force): tasks.put( Task( ami_id, user_data, setup_disks, instance_type, obs_id, snapshot_id, created_by, name, spot_price, instance_details, frequency_groups, counter ) ) counter += 1 # Add a poison pill to shut things down for x in range(processes): tasks.put(None) # Wait for the queue to terminate tasks.join()
def check_args(args): """ Check the arguments and prompt for new ones """ map_args = {} if args['obs_ids'] is None: return None elif len(args['obs_ids']) == 1 and args['obs_ids'][0] == '*': map_args['obs_ids'] = OBS_IDS.keys() else: map_args['obs_ids'] = args['obs_ids'] if args['instance_type'] is None: return None if args['name'] is None: return None instance_details = AWS_INSTANCES.get(args['instance_type']) if instance_details is None: LOGGER.error('The instance type {0} is not supported.'.format( args['instance_type'])) return None else: LOGGER.info( 'instance: {0}, vCPU: {1}, RAM: {2}GB, Disks: {3}x{4}GB, IOPS: {5}' .format(args['instance_type'], instance_details.vCPU, instance_details.memory, instance_details.number_disks, instance_details.size, instance_details.iops_support)) map_args.update({ 'ami_id': args['ami_id'] if args['ami_id'] is not None else AWS_AMI_ID, 'created_by': args['created_by'] if args['created_by'] is not None else getpass.getuser(), 'spot_price': args['spot_price'] if args['spot_price'] is not None else None, 'user_data': get_script(args['bash_script'] if args['bash_script'] is not None else BASH_SCRIPT_CVEL), 'setup_disks': get_script(BASH_SCRIPT_SETUP_DISKS), 'instance_details': instance_details, }) return map_args
def check_args(args): """ Check the arguments and prompt for new ones """ map_args = {} if args['obs_ids'] is None: return None elif len(args['obs_ids']) == 1 and args['obs_ids'][0] == '*': map_args['obs_ids'] = OBS_IDS.keys() else: map_args['obs_ids'] = args['obs_ids'] if args['instance_type'] is None: return None if args['name'] is None: return None instance_details = AWS_INSTANCES.get(args['instance_type']) if instance_details is None: LOGGER.error('The instance type {0} is not supported.'.format(args['instance_type'])) return None else: LOGGER.info( 'instance: {0}, vCPU: {1}, RAM: {2}GB, Disks: {3}x{4}GB, IOPS: {5}'.format( args['instance_type'], instance_details.vCPU, instance_details.memory, instance_details.number_disks, instance_details.size, instance_details.iops_support)) map_args.update({ 'ami_id': args['ami_id'] if args['ami_id'] is not None else AWS_AMI_ID, 'created_by': args['created_by'] if args['created_by'] is not None else getpass.getuser(), 'spot_price': args['spot_price'] if args['spot_price'] is not None else None, 'user_data': get_script(args['bash_script'] if args['bash_script'] is not None else BASH_SCRIPT_CVEL), 'setup_disks': get_script(BASH_SCRIPT_SETUP_DISKS), 'instance_details': instance_details, }) return map_args
def start_servers( ami_id, user_data, setup_disks, instance_type, obs_id, created_by, name, instance_details, spot_price): snapshot_id = OBS_IDS.get(obs_id) if snapshot_id is None: LOGGER.warning('The obs-id: {0} does not exist in the settings file') else: ec2_helper = EC2Helper() iops = None if instance_details.iops_support: iops = 500 zone = ec2_helper.get_cheapest_spot_price(instance_type, spot_price) if zone is not None: volume, snapshot_name = ec2_helper.create_volume(snapshot_id, zone, iops=iops) LOGGER.info('obs_id: {0}, volume_name: {1}'.format(obs_id, snapshot_name)) now = datetime.datetime.now() user_data_mime = get_mime_encoded_user_data(volume.id, setup_disks, user_data, now.strftime('%Y-%m-%dT%H-%M-%S')) if spot_price is not None: ec2_helper.run_spot_instance( ami_id, spot_price, user_data_mime, instance_type, volume.id, created_by, '{1}-{0}'.format(name, snapshot_name), instance_details, zone, ephemeral=True) else: LOGGER.error('Cannot get a spot instance of {0} for ${1}'.format(instance_type, spot_price))
def start_servers(ami_id, user_data, setup_disks, instance_type, obs_id, created_by, name, instance_details, spot_price): snapshot_id = OBS_IDS.get(obs_id) if snapshot_id is None: LOGGER.warning('The obs-id: {0} does not exist in the settings file') else: ec2_helper = EC2Helper() iops = None if instance_details.iops_support: iops = 500 zone = ec2_helper.get_cheapest_spot_price(instance_type, spot_price) if zone is not None: volume, snapshot_name = ec2_helper.create_volume(snapshot_id, zone, iops=iops) LOGGER.info('obs_id: {0}, volume_name: {1}'.format( obs_id, snapshot_name)) now = datetime.datetime.now() user_data_mime = get_mime_encoded_user_data( volume.id, setup_disks, user_data, now.strftime('%Y-%m-%dT%H-%M-%S')) if spot_price is not None: ec2_helper.run_spot_instance(ami_id, spot_price, user_data_mime, instance_type, volume.id, created_by, '{1}-{0}'.format( name, snapshot_name), instance_details, zone, ephemeral=True) else: LOGGER.error('Cannot get a spot instance of {0} for ${1}'.format( instance_type, spot_price))