def command_interactive(args): LOG.info(args) sleep(0.5) # Allow the logging time to print path_dirname, filename = os.path.split(__file__) config_file_name = '{0}/aws-chiles02.settings'.format(path_dirname) if os.path.exists(config_file_name): config = ConfigObj(config_file_name) else: config = ConfigObj() config.filename = config_file_name get_argument(config, 'ami', 'AMI Id', help_text='the AMI to use', default=AWS_AMI_ID) get_argument(config, 'bucket_name', 'Bucket name', help_text='the bucket to access', default='13b-266') get_argument(config, 'volume', 'Volume', help_text='the directory on the host to bind to the Docker Apps', default='/mnt/dfms/dfms_root') get_argument(config, 'width', 'Frequency width', data_type=int, help_text='the frequency width', default=4) get_argument(config, 'nodes_ingest', 'Number of ingest nodes', data_type=int, help_text='the number of ingest nodes', default=4) # Write the arguments config.write() create_and_generate( **config )
def command_interactive(args): LOG.info(args) sleep(0.5) # Allow the logging time to print path_dirname, filename = os.path.split(__file__) config_file_name = '{0}/aws-chiles02.settings'.format(path_dirname) if os.path.exists(config_file_name): config = ConfigObj(config_file_name) else: config = ConfigObj() config.filename = config_file_name get_argument(config, 'create_use', 'Create or use', allowed=['create', 'use'], help_text='the use a network or create a network') if config['create_use'] == 'create': get_argument(config, 'ami', 'AMI Id', help_text='the AMI to use', default=AWS_AMI_ID) get_argument(config, 'spot_price', 'Spot Price for i2.2xlarge', help_text='the spot price') get_argument(config, 'bucket_name', 'Bucket name', help_text='the bucket to access', default='13b-266') get_argument(config, 'volume', 'Volume', help_text='the directory on the host to bind to the Docker Apps') get_argument(config, 'width', 'Frequency width', data_type=int, help_text='the frequency width', default=4) get_argument(config, 'bottom_frequency', 'Bottom frequency', data_type=int, help_text='the bottom frequency to look at') get_argument(config, 'nodes', 'Number nodes', data_type=int, help_text='the number of nodes', default=8) get_argument(config, 'shutdown', 'Add the shutdown node', data_type=bool, help_text='add a shutdown drop', default=True) else: get_argument(config, 'dim', 'Data Island Manager', help_text='the IP to the DataIsland Manager') get_argument(config, 'bucket_name', 'Bucket name', help_text='the bucket to access', default='13b-266') get_argument(config, 'volume', 'Volume', help_text='the directory on the host to bind to the Docker Apps') get_argument(config, 'width', 'Frequency width', data_type=int, help_text='the frequency width', default=4) get_argument(config, 'bottom_frequency', 'Bottom frequency', data_type=int, help_text='the bottom frequency to look at') get_argument(config, 'shutdown', 'Add the shutdown node', data_type=bool, help_text='add a shutdown drop', default=True) # Write the arguments config.write() # Run the command if config['create_use'] == 'create': create_and_generate( config['bucket_name'], config['width'], config['ami'], config['spot_price'], config['volume'], config['bottom_frequency'], config['nodes'], config['shutdown'], ) else: use_and_generate( config['dim'], DIM_PORT, config['bucket_name'], config['width'], config['volume'], config['bottom_frequency'], config['shutdown'], )
def command_interactive(args): LOG.info(args) sleep(0.5) # Allow the logging time to print path_dirname, filename = os.path.split(__file__) config_file_name = '{0}/aws-chiles02.settings'.format(path_dirname) if os.path.exists(config_file_name): config = ConfigObj(config_file_name) else: config = ConfigObj() config.filename = config_file_name get_argument(config, 'run_type', 'Create, use or json', allowed=['create', 'use', 'json'], help_text=' use a network or create a network or just produce the JSON') get_argument(config, 'bucket_name', 'Bucket name', help_text='the bucket to access', default='13b-266') get_argument(config, 'volume', 'Volume', help_text='the directory on the host to bind to the Docker Apps') get_argument(config, 'width', 'Frequency width', data_type=int, help_text='the frequency width', default=4) get_argument(config, 'w_projection_planes', 'W Projection planes', data_type=int, help_text='the number of w projections planes', default=24) get_argument(config, 'shutdown', 'Add the shutdown node', data_type=bool, help_text='add a shutdown drop', default=True) get_argument(config, 'scan_statistics', 'Generate scan statistics', data_type=bool, help_text='generate scan statistics', default=True) get_argument(config, 'frequency_range', 'Do you want to specify a range of frequencies', data_type=bool, help_text='Do you want to specify a range of frequencies', default=False) if config['frequency_range']: get_argument(config, 'min_frequency', 'The minimum frequency', data_type=int, help_text='the minimum frequency', default=944) get_argument(config, 'max_frequency', 'The maximum frequency', data_type=int, help_text='the maximum frequency', default=1420) if config['run_type'] == 'create': get_argument(config, 'ami', 'AMI Id', help_text='the AMI to use', default=AWS_AMI_ID) get_argument(config, 'spot_price', 'Spot Price for i2.2xlarge', help_text='the spot price') get_argument(config, 'nodes', 'Number of nodes', data_type=int, help_text='the number of nodes', default=1) get_argument(config, 'dump_json', 'Dump the json', data_type=bool, help_text='dump the json', default=False) elif config['run_type'] == 'use': get_argument(config, 'dim', 'Data Island Manager', help_text='the IP to the DataIsland Manager') get_argument(config, 'dump_json', 'Dump the json', data_type=bool, help_text='dump the json', default=False) else: get_argument(config, 'nodes', 'Number of nodes', data_type=int, help_text='the number of nodes', default=1) # Write the arguments config.write() # Run the command if config['run_type'] == 'create': create_and_generate( bucket_name=config['bucket_name'], frequency_width=config['width'], w_projection_planes=config['w_projection_planes'], ami_id=config['ami'], spot_price=config['spot_price'], volume=config['volume'], nodes=config['nodes'], add_shutdown=config['shutdown'], min_frequency=config['min_frequency'] if config['frequency_range'] else None, max_frequency=config['max_frequency'] if config['frequency_range'] else None, scan_statistics=config['scan_statistics'], dump_json=config['dump_json'], ) elif config['run_type'] == 'use': use_and_generate( host=config['dim'], port=DIM_PORT, bucket_name=config['bucket_name'], frequency_width=config['width'], w_projection_planes=config['w_projection_planes'], volume=config['volume'], add_shutdown=config['shutdown'], min_frequency=config['min_frequency'] if config['frequency_range'] else None, max_frequency=config['max_frequency'] if config['frequency_range'] else None, scan_statistics=config['scan_statistics'], dump_json=config['dump_json'], ) else: generate_json( width=config['width'], w_projection_planes=config['w_projection_planes'], bucket=config['bucket_name'], nodes=config['nodes'], volume=config['volume'], shutdown=config['shutdown'], min_frequency=config['min_frequency'] if config['frequency_range'] else None, max_frequency=config['max_frequency'] if config['frequency_range'] else None, scan_statistics=config['scan_statistics'], )
def command_interactive(args): LOG.info(args) sleep(0.5) # Allow the logging time to print path_dirname, filename = os.path.split(__file__) config_file_name = '{0}/aws-chiles02.settings'.format(path_dirname) if os.path.exists(config_file_name): config = ConfigObj(config_file_name) else: config = ConfigObj() config.filename = config_file_name get_argument(config, 'run_type', 'Create, use or json', allowed=['create', 'use', 'json'], help_text=' use a network or create a network or just produce the JSON') if config['run_type'] == 'create': get_argument(config, 'ami', 'AMI Id', help_text='the AMI to use', default=AWS_AMI_ID) get_argument(config, 'spot_price', 'Spot Price for i2.2xlarge', help_text='the spot price') get_argument(config, 'bucket_name', 'Bucket name', help_text='the bucket to access', default='13b-266') get_argument(config, 'volume', 'Volume', help_text='the directory on the host to bind to the Docker Apps') get_argument(config, 'input_dir', 'Input directory', help_text='the input directory', default='uvsub_4') get_argument(config, 'nodes', 'Number of nodes', data_type=int, help_text='the number of nodes', default=1) get_argument(config, 'log_level', 'Log level', allowed=['v', 'vv', 'vvv'], help_text='the log level', default='vvv') get_argument(config, 'shutdown', 'Add the shutdown node', data_type=bool, help_text='add a shutdown drop', default=True) elif config['run_type'] == 'use': get_argument(config, 'dim', 'Data Island Manager', help_text='the IP to the DataIsland Manager') get_argument(config, 'bucket_name', 'Bucket name', help_text='the bucket to access', default='13b-266') get_argument(config, 'volume', 'Volume', help_text='the directory on the host to bind to the Docker Apps') get_argument(config, 'input_dir', 'Input directory', help_text='the input directory', default='uvsub_4') get_argument(config, 'shutdown', 'Add the shutdown node', data_type=bool, help_text='add a shutdown drop', default=True) else: get_argument(config, 'bucket_name', 'Bucket name', help_text='the bucket to access', default='13b-266') get_argument(config, 'volume', 'Volume', help_text='the directory on the host to bind to the Docker Apps') get_argument(config, 'input_dir', 'Input directory', help_text='the input directory', default='uvsub_4') get_argument(config, 'nodes', 'Number of nodes', data_type=int, help_text='the number of nodes', default=1) get_argument(config, 'shutdown', 'Add the shutdown node', data_type=bool, help_text='add a shutdown drop', default=True) # Write the arguments config.write() # Run the command if config['run_type'] == 'create': create_and_generate( config['bucket_name'], config['input_dir'], config['ami'], config['spot_price'], config['volume'], config['nodes'], config['shutdown'], config['log_level'], ) elif config['run_type'] == 'use': use_and_generate( config['dim'], DIM_PORT, config['bucket_name'], config['input_dir'], config['volume'], config['shutdown'], ) else: generate_json( config['input_dir'], config['bucket_name'], config['nodes'], config['volume'], config['shutdown'], )
def interactive(config): get_argument(config, 'create_use_json', 'Create, use or json', allowed=['create', 'use', 'json'], help_text='the use a network or create a network') get_argument(config, 'bucket_name', 'Bucket name', help_text='the bucket to access', default='13b-266') get_argument(config, 'volume', 'Volume', help_text='the directory on the host to bind to the Docker Apps') get_argument(config, 'width', 'Frequency width', data_type=int, help_text='the frequency width', default=4) get_argument(config, 'shutdown', 'Add the shutdown node', data_type=bool, help_text='add a shutdown drop', default=True) if config['create_use_json'] == 'create': get_argument(config, 'ami', 'AMI Id', help_text='the AMI to use', default=AWS_AMI_ID) get_argument(config, 'spot_price_i2.2xlarge', 'Spot Price for i2.2xlarge', help_text='the spot price') get_argument(config, 'spot_price_i2_4xlarge', 'Spot Price for i2.4xlarge', help_text='the spot price') get_argument(config, 'days_per_node', 'Number of days per node', data_type=int, help_text='the number of days per node', default=1) elif config['create_use_json'] == 'use': get_argument(config, 'dim', 'Data Island Manager', help_text='the IP to the DataIsland Manager') else: get_argument(config, 'nodes', 'Number nodes', data_type=int, help_text='the number of nodes', default=8) get_argument(config, 'parallel_streams', 'Parallel streams', data_type=int, help_text='the number of parallel streams', default=4) # Run the command if config['create_use_json'] == 'create': command_line = 'create {0} {1} {2} {3} {4} {5} {6} {7}'.format( config['bucket_name'], config['volume'], config['ami'], config['spot_price_i2.2xlarge'], config['spot_price_i2_4xlarge'], '--days_per_node ' + config['days_per_node'], '--width ' + config['width'], '--shutdown' if config['shutdown'] else '' ) create_and_generate( bucket_name=config['bucket_name'], frequency_width=config['width'], ami_id=config['ami'], spot_price1=config['spot_price_i2.2xlarge'], spot_price2=config['spot_price_i2_4xlarge'], volume=config['volume'], days_per_node=config['days_per_node'], add_shutdown=config['shutdown'], ) elif config['create_use_json'] == 'use': command_line = 'use {0} {1} {2} {3} {4} {5}'.format( config['bucket_name'], config['volume'], config['dim'], '--port ' + DIM_PORT, '--width ' + config['width'], '--shutdown' if config['shutdown'] else '' ) use_and_generate( host=config['dim'], port=DIM_PORT, bucket_name=config['bucket_name'], frequency_width=config['width'], volume=config['volume'], add_shutdown=config['shutdown'], ) else: command_line = 'json'.format( ) build_json( bucket=config['bucket_name'], width=config['width'], volume=config['volume'], nodes=config['nodes'], parallel_streams=config['parallel_streams'], add_shutdown=config['shutdown'], ) return parser_arguments(command_line)
def command_interactive(args): LOG.info(args) path_dirname, filename = os.path.split(__file__) root, ext = os.path.splitext(filename) config_file_name = '{0}/{1}.settings'.format(path_dirname, root) if os.path.exists(config_file_name): config = ConfigObj(config_file_name) else: config = ConfigObj() config.filename = config_file_name get_argument(config, 'create_use', 'Create or use', allowed=['create', 'use'], help_text='the use a network or create a network') if config['create_use'] == 'create': get_argument(config, 'ami', 'AMI Id', help_text='the AMI to use', default=AWS_AMI_ID) get_argument(config, 'spot_price', 'Spot Price for i2.4xlarge', help_text='the spot price') get_argument(config, 'bucket_name', 'Bucket name', help_text='the bucket to access', default='13b-266') get_argument(config, 'volume', 'Volume', help_text='the directory on the host to bind to the Docker Apps') get_argument(config, 'width', 'Frequency width', data_type=int, help_text='the frequency width', default=4) get_argument(config, 'iterations', 'Clean iterations', data_type=int, help_text='the clean iterations', default=10) get_argument(config, 'concatenation_type', 'Image, Virtual', allowed=['image', 'virtual'], help_text='the type of iteration') get_argument(config, 'shutdown', 'Add the shutdown node', data_type=bool, help_text='add a shutdown drop', default=True) else: get_argument(config, 'dim', 'Data Island Manager', help_text='the IP to the DataIsland Manager') get_argument(config, 'bucket_name', 'Bucket name', help_text='the bucket to access', default='13b-266') get_argument(config, 'volume', 'Volume', help_text='the directory on the host to bind to the Docker Apps') get_argument(config, 'width', 'Frequency width', data_type=int, help_text='the frequency width', default=4) get_argument(config, 'iterations', 'Clean iterations', data_type=int, help_text='the clean iterations', default=10) get_argument(config, 'concatenation_type', 'Image, Virtual', allowed=['image', 'virtual'], help_text='the type of iteration') get_argument(config, 'shutdown', 'Add the shutdown node', data_type=bool, help_text='add a shutdown drop', default=True) # Write the arguments config.write() # Run the command if config['create_use'] == 'create': create_and_generate( config['bucket_name'], config['width'], config['ami'], config['spot_price'], config['volume'], config['shutdown'], config['iterations'], config['concatenation_type'], ) else: use_and_generate( config['dim'], DIM_PORT, config['bucket_name'], config['width'], config['volume'], config['shutdown'], config['iterations'], config['concatenation_type'], )
def command_interactive(args): LOG.info(args) sleep(0.5) # Allow the logging time to print path_dirname, filename = os.path.split(__file__) config_file_name = '{0}/aws-chiles02.settings'.format(path_dirname) if os.path.exists(config_file_name): config = ConfigObj(config_file_name) else: config = ConfigObj() config.filename = config_file_name get_argument(config, 'run_type', 'Create, use or json', allowed=['create', 'use', 'json'], help_text='the use a network or create a network') get_argument(config, 'bucket_name', 'Bucket name', help_text='the bucket to access', default='13b-266') get_argument(config, 'volume', 'Volume', help_text='the directory on the host to bind to the Docker Apps') get_argument(config, 'width', 'Frequency width', data_type=int, help_text='the frequency width', default=4) get_argument(config, 'iterations', 'Clean iterations', data_type=int, help_text='the clean iterations', default=1) get_argument(config, 'arcsec', 'How many arc seconds', help_text='the arc seconds', default='2') get_argument(config, 'w_projection_planes', 'W Projection planes', data_type=int, help_text='the number of w projections planes', default=24) get_argument(config, 'robust', 'Clean robust value', data_type=float, help_text='the robust value for clean', default=0.8) get_argument(config, 'image_size', 'The image size', data_type=int, help_text='the image size for clean', default=2048) get_argument(config, 'clean_channel_average', 'The number of input channels to average', data_type=int, help_text='the number of input channels to average', default=1) get_argument(config, 'only_image', 'Only the image to S3', data_type=bool, help_text='only copy the image to S3', default=False) get_argument(config, 'shutdown', 'Add the shutdown node', data_type=bool, help_text='add a shutdown drop', default=True) get_argument(config, 'clean_directory_name', 'The directory name for clean', help_text='the directory name for clean', use_stored=False) get_argument(config, 'frequency_range', 'Do you want to specify a range of frequencies', data_type=bool, help_text='Do you want to specify a range of frequencies', default=False) if config['frequency_range']: get_argument(config, 'min_frequency', 'The minimum frequency', data_type=int, help_text='the minimum frequency', default=944) get_argument(config, 'max_frequency', 'The maximum frequency', data_type=int, help_text='the maximum frequency', default=1420) if config['run_type'] == 'create': get_argument(config, 'ami', 'AMI Id', help_text='the AMI to use', default=AWS_AMI_ID) get_argument(config, 'spot_price_i2_4xlarge', 'Spot Price for i2.4xlarge', help_text='the spot price') get_argument(config, 'frequencies_per_node', 'Number of frequencies per node', data_type=int, help_text='the number of frequencies per node', default=1) get_argument(config, 'log_level', 'Log level', allowed=['v', 'vv', 'vvv'], help_text='the log level', default='vvv') elif config['run_type'] == 'use': get_argument(config, 'dim', 'Data Island Manager', help_text='the IP to the DataIsland Manager') else: get_argument(config, 'nodes', 'Number of nodes', data_type=int, help_text='the number of nodes', default=1) # Write the arguments config.write() # Run the command if config['run_type'] == 'create': create_and_generate( bucket_name=config['bucket_name'], frequency_width=config['width'], ami_id=config['ami'], spot_price=config['spot_price_i2_4xlarge'], volume=config['volume'], frequencies_per_node=config['frequencies_per_node'], add_shutdown=config['shutdown'], iterations=config['iterations'], arcsec=config['arcsec'] + 'arcsec', w_projection_planes=config['w_projection_planes'], robust=config['robust'], only_image=config['only_image'], image_size=config['image_size'], clean_channel_average=config['clean_channel_average'], min_frequency=config['min_frequency'] if config['frequency_range'] else None, max_frequency=config['max_frequency'] if config['frequency_range'] else None, clean_directory_name=config['clean_directory_name'], log_level=config['log_level'], ) elif config['run_type'] == 'use': use_and_generate( host=config['dim'], port=DIM_PORT, bucket_name=config['bucket_name'], frequency_width=config['width'], volume=config['volume'], add_shutdown=config['shutdown'], iterations=config['iterations'], arcsec=config['arcsec'] + 'arcsec', w_projection_planes=config['w_projection_planes'], robust=config['robust'], image_size=config['image_size'], clean_channel_average=config['clean_channel_average'], min_frequency=config['min_frequency'] if config['frequency_range'] else None, max_frequency=config['max_frequency'] if config['frequency_range'] else None, clean_directory_name=config['clean_directory_name'], only_image=config['only_image'], ) else: generate_json( width=config['width'], bucket=config['bucket_name'], iterations=config['iterations'], arcsec=config['arcsec'] + 'arcsec', nodes=config['nodes'], volume=config['volume'], parallel_streams=PARALLEL_STREAMS, shutdown=config['shutdown'], w_projection_planes=config['w_projection_planes'], robust=config['robust'], image_size=config['image_size'], clean_channel_average=config['clean_channel_average'], min_frequency=config['min_frequency'] if config['frequency_range'] else None, max_frequency=config['max_frequency'] if config['frequency_range'] else None, clean_directory_name=config['clean_directory_name'], only_image=config['only_image'] )
def command_interactive(args): LOG.info(args) sleep(0.5) # Allow the logging time to print path_dirname, filename = os.path.split(__file__) config_file_name = '{0}/aws-chiles02.settings'.format(path_dirname) if os.path.exists(config_file_name): config = ConfigObj(config_file_name) else: config = ConfigObj() config.filename = config_file_name get_argument(config, 'create_use', 'Create or use', allowed=['create', 'use'], help_text='the use a network or create a network') get_argument(config, 'bucket_name', 'Bucket name', help_text='the bucket to access', default='13b-266') get_argument(config, 'volume', 'Volume', help_text='the directory on the host to bind to the Docker Apps') get_argument(config, 'width', 'Frequency width', data_type=int, help_text='the frequency width', default=4) get_argument(config, 'iterations', 'Clean iterations', data_type=int, help_text='the clean iterations', default=1) get_argument(config, 'arcsec', 'How many arc seconds', help_text='the arc seconds', default='1.25') get_argument(config, 'only_image', 'Only the image to S3', data_type=bool, help_text='only copy the image to S3', default=False) get_argument(config, 'shutdown', 'Add the shutdown node', data_type=bool, help_text='add a shutdown drop', default=True) if config['create_use'] == 'create': get_argument(config, 'ami', 'AMI Id', help_text='the AMI to use', default=AWS_AMI_ID) get_argument(config, 'spot_price_i2_4xlarge', 'Spot Price for i2.4xlarge', help_text='the spot price') get_argument(config, 'frequencies_per_node', 'Number of frequencies per node', data_type=int, help_text='the number of frequencies per node', default=1) get_argument(config, 'log_level', 'Log level', allowed=['v', 'vv', 'vvv'], help_text='the log level', default='vvv') else: get_argument(config, 'dim', 'Data Island Manager', help_text='the IP to the DataIsland Manager') # Write the arguments config.write() # Run the command if config['create_use'] == 'create': create_and_generate( config['bucket_name'], config['width'], config['ami'], config['spot_price_i2_4xlarge'], config['volume'], config['frequencies_per_node'], config['shutdown'], config['iterations'], config['arcsec'] + 'arcsec', config['only_image'], config['log_level'], ) else: use_and_generate( config['dim'], DIM_PORT, config['bucket_name'], config['width'], config['volume'], config['shutdown'], config['iterations'], config['arcsec'] + 'arcsec', config['only_image'], )
def command_interactive(args): LOG.info(args) sleep(0.5) # Allow the logging time to print path_dirname, filename = os.path.split(__file__) config_file_name = '{0}/aws-chiles02.settings'.format(path_dirname) if os.path.exists(config_file_name): config = ConfigObj(config_file_name) else: config = ConfigObj() config.filename = config_file_name get_argument(config, 'create_use', 'Create or use', allowed=['create', 'use'], help_text='the use a network or create a network') get_argument(config, 'bucket_name', 'Bucket name', help_text='the bucket to access', default='13b-266') get_argument( config, 'volume', 'Volume', help_text='the directory on the host to bind to the Docker Apps') get_argument(config, 'width', 'Frequency width', data_type=int, help_text='the frequency width', default=4) get_argument(config, 'iterations', 'Clean iterations', data_type=int, help_text='the clean iterations', default=1) get_argument(config, 'arcsec', 'How many arc seconds', help_text='the arc seconds', default='1.25') get_argument(config, 'only_image', 'Only the image to S3', data_type=bool, help_text='only copy the image to S3', default=False) get_argument(config, 'shutdown', 'Add the shutdown node', data_type=bool, help_text='add a shutdown drop', default=True) if config['create_use'] == 'create': get_argument(config, 'ami', 'AMI Id', help_text='the AMI to use', default=AWS_AMI_ID) get_argument(config, 'spot_price_i2_4xlarge', 'Spot Price for i2.4xlarge', help_text='the spot price') get_argument(config, 'frequencies_per_node', 'Number of frequencies per node', data_type=int, help_text='the number of frequencies per node', default=1) get_argument(config, 'log_level', 'Log level', allowed=['v', 'vv', 'vvv'], help_text='the log level', default='vvv') else: get_argument(config, 'dim', 'Data Island Manager', help_text='the IP to the DataIsland Manager') # Write the arguments config.write() # Run the command if config['create_use'] == 'create': create_and_generate( config['bucket_name'], config['width'], config['ami'], config['spot_price_i2_4xlarge'], config['volume'], config['frequencies_per_node'], config['shutdown'], config['iterations'], config['arcsec'] + 'arcsec', config['only_image'], config['log_level'], ) else: use_and_generate( config['dim'], DIM_PORT, config['bucket_name'], config['width'], config['volume'], config['shutdown'], config['iterations'], config['arcsec'] + 'arcsec', config['only_image'], )