def proper_name_validator(value, default): """ Validate proper_name user input. """ # Check for default if value == default: return True, value # Validate and sanitize user input proper_name_error_regex = re.compile(r'^[a-zA-Z0-9\s]+$') proper_name_warn_regex = re.compile(r'^[a-zA-Z0-9-\s_\"\']+$') if not proper_name_error_regex.match(value): # If offending characters are dashes, underscores or quotes, replace and notify user if proper_name_warn_regex.match(value): before = value value = value.replace('_', ' ') value = value.replace('-', ' ') value = value.replace('"', '') value = value.replace("'", "") write_pretty_output( 'Warning: Illegal characters were detected in proper name "{0}". They have been replaced or ' 'removed with valid characters: "{1}"'.format(before, value), FG_YELLOW) # Otherwise, throw error else: write_pretty_output( 'Error: Proper name can only contain letters and numbers and spaces.', FG_RED) return False, value return True, value
def get_container_options(self, defaults): # Default environmental vars options = self.default_container_options() if not defaults: write_pretty_output( "Provide contact information for the 52 North Web Processing Service or press enter to " "accept the defaults shown in square brackets: ") options['environment'].update( NAME=UserInputHelper.get_input_with_default('Name', 'NONE'), POSITION=UserInputHelper.get_input_with_default( 'Position', 'NONE'), ADDRESS=UserInputHelper.get_input_with_default( 'Address', 'NONE'), CITY=UserInputHelper.get_input_with_default('City', 'NONE'), STATE=UserInputHelper.get_input_with_default('State', 'NONE'), COUNTRY=UserInputHelper.get_input_with_default( 'Country', 'NONE'), POSTAL_CODE=UserInputHelper.get_input_with_default( 'Postal Code', 'NONE'), EMAIL=UserInputHelper.get_input_with_default('Email', 'NONE'), PHONE=UserInputHelper.get_input_with_default('Phone', 'NONE'), FAX=UserInputHelper.get_input_with_default('Fax', 'NONE'), USERNAME=UserInputHelper.get_input_with_default( 'Admin Username', 'wps'), PASSWORD=UserInputHelper.get_verified_password( 'Admin Password', 'wps')) return options
def get_container_options(self, defaults): # Default options options = self.default_container_options() # User environmental variables if not defaults: write_pretty_output( "Provide passwords for the three Tethys database users or press enter to accept the " "default passwords shown in square brackets:") default_password = '******' # tethys_default prompt = 'Password for "tethys_default" database user' tethys_default_pass = UserInputHelper.get_verified_password( prompt, default_password) # tethys_db_manager prompt = 'Password for "tethys_db_manager" database user' tethys_db_manager_pass = UserInputHelper.get_verified_password( prompt, default_password) # tethys_super prompt = 'Password for "tethys_super" database user' tethys_super_pass = UserInputHelper.get_verified_password( prompt, default_password) options['environment'].update( TETHYS_DEFAULT_PASS=tethys_default_pass, TETHYS_DB_MANAGER_PASS=tethys_db_manager_pass, TETHYS_SUPER_PASS=tethys_super_pass) return options
def create(self, defaults=False): write_pretty_output("\nInstalling the {} Docker container...".format( self.display_name)) options = self.get_container_options(defaults) options['host_config'] = self.docker_client.api.create_host_config( **options['host_config']) self.docker_client.api.create_container(**options)
def start(self): msg = 'Starting {} container...' write_pretty_output(msg.format(self.display_name)) msg = None try: self.container.start() except Exception as e: msg = 'There was an error while attempting to start container {}: {}'.format( self.display_name, str(e)) return msg
def stop(self, silent=False): msg = 'Stopping {} container...' if not silent: write_pretty_output(msg.format(self.display_name)) try: self.container.stop() msg = None except Exception as e: msg = 'There was an error while attempting to stop container {}: {}'.format( self.display_name, str(e)) return msg
def install_docker_containers(containers_to_install, defaults=False): """ Install all Docker containers Args: containers_to_install(list[ContainerMetadata], required): list of containers to install defaults(bool, optional, default=False): if True use all of the default options instead of prompting user to specify options """ for container_metadata in containers_to_install: container_metadata.create(defaults) write_pretty_output("Finished installing Docker containers.")
def pull_docker_images(containers_to_install): """ Pull Docker container images Args: containers_to_install(list[ContainerMetadata], required): list of containers to install """ if len(containers_to_install) < 1: write_pretty_output("Docker images already pulled.") else: write_pretty_output("Pulling Docker images...") for container in containers_to_install: container.pull()
def docker_status(containers=None): """ Returns the status of the Docker containers: either Running or Stopped. """ # Get container dicts container_statuses = get_docker_container_statuses(containers=containers) for container, is_running in container_statuses.items(): if is_running is None: msg = '{}: Not Installed' elif is_running: msg = '{}: Running' else: msg = '{}: Not Running' write_pretty_output(msg.format(container.display_name))
def docker_ip(containers=None): """ Returns the hosts and ports of the Docker containers. """ # Containers container_statuses = get_docker_container_statuses(containers=containers) for container_metadata, is_running in container_statuses.items(): if is_running is None: msg = '{name}: Not Installed' elif not is_running: msg = '{name}: Not Running' else: msg = container_metadata.ip write_pretty_output(msg.format(name=container_metadata.display_name))
def get_verified_password(prompt, default): password_1 = getpass.getpass('{} [{}]: '.format(prompt, default)) if password_1 == '': return default else: password_2 = getpass.getpass('Confirm Password: '******'Passwords do not match, please try again.') password_1 = getpass.getpass('{} [{}]: '.format( prompt, default)) password_2 = getpass.getpass('Confirm Password: ') return password_1
def docker_stop(containers=None): """ Stop Docker containers """ container_statuses = get_docker_container_statuses(containers=containers) for container_metadata, status in container_statuses.items(): already_stopped = not status if status is None: msg = '{} container not installed.' elif already_stopped: msg = '{} container already stopped.' else: msg = container_metadata.stop() if msg is not None: write_pretty_output(msg.format(container_metadata.display_name))
def docker_start(containers=None): """ Start the docker containers """ container_statuses = get_docker_container_statuses(containers=containers) for container_metadata, status in container_statuses.items(): already_running = status if already_running is None: msg = '{} container not installed.' elif already_running: msg = '{} container already running.' else: msg = container_metadata.start() if msg is not None: write_pretty_output(msg.format(container_metadata.display_name))
def get_container_options(self, defaults): # Default options options = self.default_container_options() # User environmental variables if not defaults: write_pretty_output( "Tethys uses the mdillon/postgis image on Docker Hub. " "See: https://hub.docker.com/r/mdillon/postgis/") # POSTGRES_PASSWORD prompt = 'Password for postgres user (i.e. POSTGRES_PASSWORD)' postgres_password = \ UserInputHelper.get_verified_password(prompt, options['environment']['POSTGRES_PASSWORD']) options['environment'].update( POSTGRES_PASSWORD=postgres_password, ) return options
def theme_color_validator(value, default): """ Validate theme_color user input. """ # Generate random color if default option provided if value == default: return True, get_random_color() # Validate hexadecimal if provided try: if len(value) > 0 and '#' in value: value = value[1:] int(value, 16) value = '#' + value return True, value except ValueError: write_pretty_output( "Error: Value given is not a valid hexadecimal color.", FG_RED) return False, value
def get_valid_directory_input(prompt, default=None): default = default or '' pre_prompt = '' prompt = '{} [{}]: '.format(prompt, default) while True: value = input('{}{}'.format(pre_prompt, prompt)) or str(default) if len(value) > 0 and value[0] != '/': value = '/' + value if not os.path.isdir(value): try: os.makedirs(value) except OSError as e: write_pretty_output('{0}: {1}'.format(repr(e), value)) pre_prompt = 'Please provide a valid directory\n' continue break return value
def get_container_options(self, defaults): # Default environmental vars options = self.default_container_options() if not defaults: environment = dict() write_pretty_output( "Provide configuration options for the THREDDS container or or press enter to " "accept the defaults shown in square brackets: ") environment['TDM_PW'] = UserInputHelper.get_verified_password( prompt='TDM Password', default=options['environment']['TDM_PW'], ) environment['TDS_HOST'] = UserInputHelper.get_input_with_default( prompt='TDS Host', default=options['environment']['TDS_HOST'], ) environment[ 'THREDDS_XMX_SIZE'] = UserInputHelper.get_input_with_default( prompt='TDS JVM Max Heap Size', default=options['environment']['THREDDS_XMX_SIZE'], ) environment[ 'THREDDS_XMS_SIZE'] = UserInputHelper.get_input_with_default( prompt='TDS JVM Min Heap Size', default=options['environment']['THREDDS_XMS_SIZE'], ) environment[ 'TDM_XMX_SIZE'] = UserInputHelper.get_input_with_default( prompt='TDM JVM Max Heap Size', default=options['environment']['TDM_XMX_SIZE'], ) environment[ 'TDM_XMS_SIZE'] = UserInputHelper.get_input_with_default( prompt='TDM JVM Min Heap Size', default=options['environment']['TDM_XMS_SIZE'], ) options.update(environment=environment) mount_data_dir = UserInputHelper.get_valid_choice_input( prompt='Bind the THREDDS data directory to the host?', choices=['y', 'n'], default='y', ) if mount_data_dir.lower() == 'y': tethys_home = get_tethys_home_dir() default_mount_location = os.path.join(tethys_home, 'thredds') thredds_data_volume = '/usr/local/tomcat/content/thredds' mount_location = UserInputHelper.get_valid_directory_input( prompt= 'Specify location to bind the THREDDS data directory', default=default_mount_location) mounts = [ Mount(thredds_data_volume, mount_location, type='bind') ] options['host_config'].update(mounts=mounts) return options
def remove(self): write_pretty_output('Removing {} container...'.format( self.display_name)) self.container.remove()
def scaffold_command(args): """ Create a new Tethys app projects in the current directory. """ # Log log = logging.getLogger('tethys') # log.setLevel(logging.DEBUG) log.debug('Command args: {}'.format(args)) # Get template dirs log.debug('APP_PATH: {}'.format(APP_PATH)) log.debug('EXTENSION_PATH: {}'.format(EXTENSION_PATH)) # Get template root directory is_extension = False if args.extension: is_extension = True template_name = args.template template_root = os.path.join(EXTENSION_PATH, args.template) else: template_name = args.template template_root = os.path.join(APP_PATH, args.template) log.debug('Template root directory: {}'.format(template_root)) # Validate template if not os.path.isdir(template_root): write_pretty_output( 'Error: "{}" is not a valid template.'.format(template_name), FG_WHITE) exit(1) # Validate project name project_name = args.name # Only lowercase contains_uppers = False for letter in project_name: if letter.isupper(): contains_uppers = True break if contains_uppers: before = project_name project_name = project_name.lower() write_pretty_output( 'Warning: Uppercase characters in project name "{0}" ' 'changed to lowercase: "{1}".'.format(before, project_name), FG_YELLOW) # Check for valid characters name project_error_regex = re.compile(r'^[a-zA-Z0-9_]+$') project_warning_regex = re.compile(r'^[a-zA-Z0-9_-]+$') # Only letters, numbers and underscores allowed in app names if not project_error_regex.match(project_name): # If the only offending character is a dash, replace dashes with underscores and notify user if project_warning_regex.match(project_name): before = project_name project_name = project_name.replace('-', '_') write_pretty_output( 'Warning: Dashes in project name "{0}" have been replaced ' 'with underscores "{1}"'.format(before, project_name), FG_YELLOW) # Otherwise, throw error else: write_pretty_output( 'Error: Invalid characters in project name "{0}". ' 'Only letters, numbers, and underscores.'.format(project_name), FG_YELLOW) exit(1) # Project name derivatives project_dir = '{0}-{1}'.format( EXTENSION_PREFIX if is_extension else APP_PREFIX, project_name) split_project_name = project_name.split('_') title_case_project_name = [x.title() for x in split_project_name] default_proper_name = ' '.join(title_case_project_name) class_name = ''.join(title_case_project_name) default_theme_color = get_random_color() write_pretty_output( 'Creating new Tethys project named "{0}".'.format(project_dir), FG_WHITE) # Get metadata from user if not is_extension: metadata_input = ( { 'name': 'proper_name', 'prompt': 'Proper name for the app (e.g.: "My First App")', 'default': default_proper_name, 'validator': proper_name_validator }, { 'name': 'description', 'prompt': 'Brief description of the app', 'default': '', 'validator': None }, { 'name': 'color', 'prompt': 'App theme color (e.g.: "#27AE60")', 'default': default_theme_color, 'validator': theme_color_validator }, { 'name': 'tags', 'prompt': 'Tags: Use commas to delineate tags and ' 'quotes around each tag (e.g.: "Hydrology","Reference Timeseries")', 'default': '', 'validator': None }, { 'name': 'author', 'prompt': 'Author name', 'default': '', 'validator': None }, { 'name': 'author_email', 'prompt': 'Author email', 'default': '', 'validator': None }, { 'name': 'license_name', 'prompt': 'License name', 'default': '', 'validator': None }, ) else: metadata_input = ( { 'name': 'proper_name', 'prompt': 'Proper name for the extension (e.g.: "My First Extension")', 'default': default_proper_name, 'validator': proper_name_validator }, { 'name': 'description', 'prompt': 'Brief description of the extension', 'default': '', 'validator': None }, { 'name': 'author', 'prompt': 'Author name', 'default': '', 'validator': None }, { 'name': 'author_email', 'prompt': 'Author email', 'default': '', 'validator': None }, { 'name': 'license_name', 'prompt': 'License name', 'default': '', 'validator': None }, ) # Build up template context context = { 'project': project_name, 'project_dir': project_dir, 'project_url': project_name.replace('_', '-'), 'class_name': class_name, 'proper_name': default_proper_name, 'description': '', 'color': default_theme_color, 'tags': '', 'author': '', 'author_email': '', 'license_name': '' } if not args.use_defaults: # Collect metadata input from user for item in metadata_input: valid = False response = item['default'] while not valid: try: response = input('{0} ["{1}"]: '.format( item['prompt'], item['default'])) or item['default'] except (KeyboardInterrupt, SystemExit): write_pretty_output('\nScaffolding cancelled.', FG_YELLOW) exit(1) if callable(item['validator']): valid, response = item['validator'](response, item['default']) else: valid = True if not valid: write_pretty_output( 'Invalid response: {}'.format(response), FG_RED) context[item['name']] = response log.debug('Template context: {}'.format(context)) # Create root directory project_root = os.path.join(os.getcwd(), project_dir) log.debug('Project root path: {}'.format(project_root)) if os.path.isdir(project_root): if not args.overwrite: valid = False negative_choices = ['n', 'no', ''] valid_choices = ['y', 'n', 'yes', 'no'] default = 'y' response = '' while not valid: try: response = input('Directory "{}" already exists. ' 'Would you like to overwrite it? [Y/n]: '. format(project_root)) or default except (KeyboardInterrupt, SystemExit): write_pretty_output('\nScaffolding cancelled.', FG_YELLOW) exit(1) if response.lower() in valid_choices: valid = True if response.lower() in negative_choices: write_pretty_output('Scaffolding cancelled.', FG_YELLOW) exit(0) try: shutil.rmtree(project_root) except OSError: write_pretty_output( 'Error: Unable to overwrite "{}". ' 'Please remove the directory and try again.'.format( project_root), FG_YELLOW) exit(1) # Walk the template directory, creating the templates and directories in the new project as we go for curr_template_root, dirs, template_files in os.walk(template_root): curr_project_root = curr_template_root.replace(template_root, project_root) curr_project_root = render_path(curr_project_root, context) # Create Root Directory os.makedirs(curr_project_root) write_pretty_output('Created: "{}"'.format(curr_project_root), FG_WHITE) # Create Files for template_file in template_files: template_file_path = os.path.join(curr_template_root, template_file) project_file = template_file.replace(TEMPLATE_SUFFIX, '') project_file_path = os.path.join(curr_project_root, project_file) # Load the template log.debug('Loading template: "{}"'.format(template_file_path)) try: with open(template_file_path, 'r') as tfp: template = Template(tfp.read()) except UnicodeDecodeError: with open(template_file_path, 'br') as tfp: with open(project_file_path, 'bw') as pfp: pfp.write(tfp.read()) continue # Render template if loaded log.debug('Rendering template: "{}"'.format(template_file_path)) if template: with open(project_file_path, 'w') as pfp: pfp.write(template.render(context)) write_pretty_output('Created: "{}"'.format(project_file_path), FG_WHITE) write_pretty_output( 'Successfully scaffolded new project "{}"'.format(project_name), FG_WHITE)
def log_pull_stream(stream): """ Handle the printing of pull statuses """ if platform.system() == 'Windows': # i.e. can't uses curses for block in stream: lines = [l for l in block.split(b'\r\n') if l] for line in lines: json_line = json.loads(line) current_id = "{}:".format( json_line['id']) if 'id' in json_line else '' current_status = json_line[ 'status'] if 'status' in json_line else '' current_progress = json_line[ 'progress'] if 'progress' in json_line else '' write_pretty_output("{id}{status} {progress}".format( id=current_id, status=current_status, progress=current_progress)) else: TERMINAL_STATUSES = [ 'Already exists', 'Download complete', 'Pull complete' ] PROGRESS_STATUSES = ['Downloading', 'Extracting'] STATUSES = TERMINAL_STATUSES + PROGRESS_STATUSES + [ 'Pulling fs layer', 'Waiting', 'Verifying Checksum' ] NUMBER_OF_HEADER_ROWS = 2 header_rows = list() message_log = list() progress_messages = dict() messages_to_print = list() # prepare console for curses window printing stdscr = curses.initscr() curses.noecho() curses.cbreak() try: for block in stream: lines = [l for l in block.split(b'\r\n') if l] for line in lines: json_line = json.loads(line) current_id = json_line['id'] if 'id' in json_line else None current_status = json_line[ 'status'] if 'status' in json_line else '' current_progress = json_line[ 'progress'] if 'progress' in json_line else '' if current_id is None: # save messages to print after docker images are pulled messages_to_print.append(current_status.strip()) else: # use curses window to properly display progress if current_status not in STATUSES: # Assume this is the header header_rows.append(current_status) header_rows.append('-' * len(current_status)) elif current_status in PROGRESS_STATUSES: # add messages with progress to dictionary to print at the bottom of the screen progress_messages[current_id] = { 'id': current_id, 'status': current_status, 'progress': current_progress } else: # add all other messages to list to show above progress messages message_log.append( "{id}: {status} {progress}".format( id=current_id, status=current_status, progress=current_progress)) # remove messages from progress that have completed if current_id in progress_messages: del progress_messages[current_id] # update window # row/column calculations for proper display on screen maxy, maxx = stdscr.getmaxyx() number_of_rows, number_of_columns = maxy, maxx current_progress_messages = sorted( progress_messages.values(), key=lambda message: STATUSES.index(message['status' ])) # row/column calculations for proper display on screen number_of_progress_rows = len( current_progress_messages) number_of_message_rows = number_of_rows - number_of_progress_rows - NUMBER_OF_HEADER_ROWS # slice messages to only those that will fit on the screen current_messages = [ '' ] * number_of_message_rows + message_log current_messages = current_messages[ -number_of_message_rows:] rows = header_rows + current_messages + [ '{id}: {status} {progress}'.format( **current_message) for current_message in current_progress_messages ] for row, message in enumerate(rows): message += ' ' * number_of_columns message = message[:number_of_columns - 1] stdscr.addstr(row, 0, message) stdscr.refresh() finally: # always reset console to normal regardless of success or failure curses.echo() curses.nocbreak() curses.endwin() write_pretty_output('\n'.join(messages_to_print))
def get_container_options(self, defaults): # default configuration options = self.default_container_options() if not self.is_cluster: # Then all of the other options are irrelevant defaults = True if not defaults: # Environmental variables from user input environment = dict() write_pretty_output( "The GeoServer docker can be configured to run in a clustered mode (multiple instances of " "GeoServer running in the docker container) for better performance.\n" ) environment[ 'ENABLED_NODES'] = UserInputHelper.get_valid_numeric_input( prompt='Number of GeoServer Instances Enabled', max_val=4, ) environment['REST_NODES'] = UserInputHelper.get_valid_numeric_input( prompt='Number of GeoServer Instances with REST API Enabled', max_val=int(environment['ENABLED_NODES']), ) write_pretty_output( "\nGeoServer can be configured with limits to certain types of requests to prevent it from " "becoming overwhelmed. This can be done automatically based on a number of processors or " "each " "limit can be set explicitly.\n") flow_control_mode = UserInputHelper.get_valid_choice_input( prompt= 'Would you like to specify number of Processors (c) OR set request limits explicitly (e)', choices=['c', 'e'], default='c', ) if flow_control_mode.lower() == 'c': environment[ 'NUM_CORES'] = UserInputHelper.get_valid_numeric_input( prompt='Number of Processors', max_val=4, # TODO dynamically figure out what the max is ) else: environment[ 'MAX_OWS_GLOBAL'] = UserInputHelper.get_valid_numeric_input( prompt= 'Maximum number of simultaneous OGC web service requests (e.g.: WMS, WCS, WFS)', default=100) environment[ 'MAX_WMS_GETMAP'] = UserInputHelper.get_valid_numeric_input( prompt='Maximum number of simultaneous GetMap requests', default=8) environment[ 'MAX_OWS_GWC'] = UserInputHelper.get_valid_numeric_input( prompt= 'Maximum number of simultaneous GeoWebCache tile renders', default=16) environment[ 'MAX_TIMEOUT'] = UserInputHelper.get_valid_numeric_input( prompt='Maximum request timeout in seconds', default=60) environment['MAX_MEMORY'] = UserInputHelper.get_valid_numeric_input( prompt= 'Maximum memory to allocate to each GeoServer instance in MB', max_val=4096, # TODO dynamically figure out what the max is default=1024) max_memory = int(environment['MAX_MEMORY']) environment['MIN_MEMORY'] = UserInputHelper.get_valid_numeric_input( prompt= 'Minimum memory to allocate to each GeoServer instance in MB', max_val=max_memory, default=max_memory) options.update(environment=environment, ) mount_data_dir = UserInputHelper.get_valid_choice_input( prompt='Bind the GeoServer data directory to the host?', choices=['y', 'n'], default='y', ) if mount_data_dir.lower() == 'y': tethys_home = os.environ.get('TETHYS_HOME', os.path.expanduser('~/tethys/')) default_mount_location = os.path.join(tethys_home, 'geoserver', 'data') gs_data_volume = '/var/geoserver/data' mount_location = UserInputHelper.get_valid_directory_input( prompt='Specify location to bind data directory', default=default_mount_location) mounts = [Mount(gs_data_volume, mount_location, type='bind')] options['host_config'].update(mounts=mounts) return options