def proper_name_validator(value, default):
"""
Validate proper_name user input.
"""
# Check for default
if value == default:
return True, value
# Validate and sanitize user input
proper_name_error_regex = re.compile(r'^[a-zA-Z0-9\s]+$')
proper_name_warn_regex = re.compile(r'^[a-zA-Z0-9-\s_\"\']+$')
if not proper_name_error_regex.match(value):
# If offending characters are dashes, underscores or quotes, replace and notify user
if proper_name_warn_regex.match(value):
before = value
value = value.replace('_', ' ')
value = value.replace('-', ' ')
value = value.replace('"', '')
value = value.replace("'", "")
write_pretty_output(
'Warning: Illegal characters were detected in proper name "{0}". They have been replaced or '
'removed with valid characters: "{1}"'.format(before, value),
FG_YELLOW)
# Otherwise, throw error
else:
write_pretty_output(
'Error: Proper name can only contain letters and numbers and spaces.',
FG_RED)
return False, value
return True, value
def get_container_options(self, defaults):
# Default environmental vars
options = self.default_container_options()
if not defaults:
write_pretty_output(
"Provide contact information for the 52 North Web Processing Service or press enter to "
"accept the defaults shown in square brackets: ")
options['environment'].update(
NAME=UserInputHelper.get_input_with_default('Name', 'NONE'),
POSITION=UserInputHelper.get_input_with_default(
'Position', 'NONE'),
ADDRESS=UserInputHelper.get_input_with_default(
'Address', 'NONE'),
CITY=UserInputHelper.get_input_with_default('City', 'NONE'),
STATE=UserInputHelper.get_input_with_default('State', 'NONE'),
COUNTRY=UserInputHelper.get_input_with_default(
'Country', 'NONE'),
POSTAL_CODE=UserInputHelper.get_input_with_default(
'Postal Code', 'NONE'),
EMAIL=UserInputHelper.get_input_with_default('Email', 'NONE'),
PHONE=UserInputHelper.get_input_with_default('Phone', 'NONE'),
FAX=UserInputHelper.get_input_with_default('Fax', 'NONE'),
USERNAME=UserInputHelper.get_input_with_default(
'Admin Username', 'wps'),
PASSWORD=UserInputHelper.get_verified_password(
'Admin Password', 'wps'))
return options
def get_container_options(self, defaults):
# Default options
options = self.default_container_options()
# User environmental variables
if not defaults:
write_pretty_output(
"Provide passwords for the three Tethys database users or press enter to accept the "
"default passwords shown in square brackets:")
default_password = '******'
# tethys_default
prompt = 'Password for "tethys_default" database user'
tethys_default_pass = UserInputHelper.get_verified_password(
prompt, default_password)
# tethys_db_manager
prompt = 'Password for "tethys_db_manager" database user'
tethys_db_manager_pass = UserInputHelper.get_verified_password(
prompt, default_password)
# tethys_super
prompt = 'Password for "tethys_super" database user'
tethys_super_pass = UserInputHelper.get_verified_password(
prompt, default_password)
options['environment'].update(
TETHYS_DEFAULT_PASS=tethys_default_pass,
TETHYS_DB_MANAGER_PASS=tethys_db_manager_pass,
TETHYS_SUPER_PASS=tethys_super_pass)
return options
def create(self, defaults=False):
write_pretty_output("\nInstalling the {} Docker container...".format(
self.display_name))
options = self.get_container_options(defaults)
options['host_config'] = self.docker_client.api.create_host_config(
**options['host_config'])
self.docker_client.api.create_container(**options)
def start(self):
msg = 'Starting {} container...'
write_pretty_output(msg.format(self.display_name))
msg = None
try:
self.container.start()
except Exception as e:
msg = 'There was an error while attempting to start container {}: {}'.format(
self.display_name, str(e))
return msg
def stop(self, silent=False):
msg = 'Stopping {} container...'
if not silent:
write_pretty_output(msg.format(self.display_name))
try:
self.container.stop()
msg = None
except Exception as e:
msg = 'There was an error while attempting to stop container {}: {}'.format(
self.display_name, str(e))
return msg
def install_docker_containers(containers_to_install, defaults=False):
"""
Install all Docker containers
Args:
containers_to_install(list[ContainerMetadata], required):
list of containers to install
defaults(bool, optional, default=False):
if True use all of the default options instead of prompting user to specify options
"""
for container_metadata in containers_to_install:
container_metadata.create(defaults)
write_pretty_output("Finished installing Docker containers.")
def pull_docker_images(containers_to_install):
"""
Pull Docker container images
Args:
containers_to_install(list[ContainerMetadata], required):
list of containers to install
"""
if len(containers_to_install) < 1:
write_pretty_output("Docker images already pulled.")
else:
write_pretty_output("Pulling Docker images...")
for container in containers_to_install:
container.pull()
def docker_status(containers=None):
"""
Returns the status of the Docker containers: either Running or Stopped.
"""
# Get container dicts
container_statuses = get_docker_container_statuses(containers=containers)
for container, is_running in container_statuses.items():
if is_running is None:
msg = '{}: Not Installed'
elif is_running:
msg = '{}: Running'
else:
msg = '{}: Not Running'
write_pretty_output(msg.format(container.display_name))
def docker_ip(containers=None):
"""
Returns the hosts and ports of the Docker containers.
"""
# Containers
container_statuses = get_docker_container_statuses(containers=containers)
for container_metadata, is_running in container_statuses.items():
if is_running is None:
msg = '{name}: Not Installed'
elif not is_running:
msg = '{name}: Not Running'
else:
msg = container_metadata.ip
write_pretty_output(msg.format(name=container_metadata.display_name))
def get_verified_password(prompt, default):
password_1 = getpass.getpass('{} [{}]: '.format(prompt, default))
if password_1 == '':
return default
else:
password_2 = getpass.getpass('Confirm Password: '******'Passwords do not match, please try again.')
password_1 = getpass.getpass('{} [{}]: '.format(
prompt, default))
password_2 = getpass.getpass('Confirm Password: ')
return password_1
def docker_stop(containers=None):
"""
Stop Docker containers
"""
container_statuses = get_docker_container_statuses(containers=containers)
for container_metadata, status in container_statuses.items():
already_stopped = not status
if status is None:
msg = '{} container not installed.'
elif already_stopped:
msg = '{} container already stopped.'
else:
msg = container_metadata.stop()
if msg is not None:
write_pretty_output(msg.format(container_metadata.display_name))
def docker_start(containers=None):
"""
Start the docker containers
"""
container_statuses = get_docker_container_statuses(containers=containers)
for container_metadata, status in container_statuses.items():
already_running = status
if already_running is None:
msg = '{} container not installed.'
elif already_running:
msg = '{} container already running.'
else:
msg = container_metadata.start()
if msg is not None:
write_pretty_output(msg.format(container_metadata.display_name))
def get_container_options(self, defaults):
# Default options
options = self.default_container_options()
# User environmental variables
if not defaults:
write_pretty_output(
"Tethys uses the mdillon/postgis image on Docker Hub. "
"See: https://hub.docker.com/r/mdillon/postgis/")
# POSTGRES_PASSWORD
prompt = 'Password for postgres user (i.e. POSTGRES_PASSWORD)'
postgres_password = \
UserInputHelper.get_verified_password(prompt, options['environment']['POSTGRES_PASSWORD'])
options['environment'].update(
POSTGRES_PASSWORD=postgres_password, )
return options
def theme_color_validator(value, default):
"""
Validate theme_color user input.
"""
# Generate random color if default option provided
if value == default:
return True, get_random_color()
# Validate hexadecimal if provided
try:
if len(value) > 0 and '#' in value:
value = value[1:]
int(value, 16)
value = '#' + value
return True, value
except ValueError:
write_pretty_output(
"Error: Value given is not a valid hexadecimal color.", FG_RED)
return False, value
def get_valid_directory_input(prompt, default=None):
default = default or ''
pre_prompt = ''
prompt = '{} [{}]: '.format(prompt, default)
while True:
value = input('{}{}'.format(pre_prompt, prompt)) or str(default)
if len(value) > 0 and value[0] != '/':
value = '/' + value
if not os.path.isdir(value):
try:
os.makedirs(value)
except OSError as e:
write_pretty_output('{0}: {1}'.format(repr(e), value))
pre_prompt = 'Please provide a valid directory\n'
continue
break
return value
def get_container_options(self, defaults):
# Default environmental vars
options = self.default_container_options()
if not defaults:
environment = dict()
write_pretty_output(
"Provide configuration options for the THREDDS container or or press enter to "
"accept the defaults shown in square brackets: ")
environment['TDM_PW'] = UserInputHelper.get_verified_password(
prompt='TDM Password',
default=options['environment']['TDM_PW'],
)
environment['TDS_HOST'] = UserInputHelper.get_input_with_default(
prompt='TDS Host',
default=options['environment']['TDS_HOST'],
)
environment[
'THREDDS_XMX_SIZE'] = UserInputHelper.get_input_with_default(
prompt='TDS JVM Max Heap Size',
default=options['environment']['THREDDS_XMX_SIZE'],
)
environment[
'THREDDS_XMS_SIZE'] = UserInputHelper.get_input_with_default(
prompt='TDS JVM Min Heap Size',
default=options['environment']['THREDDS_XMS_SIZE'],
)
environment[
'TDM_XMX_SIZE'] = UserInputHelper.get_input_with_default(
prompt='TDM JVM Max Heap Size',
default=options['environment']['TDM_XMX_SIZE'],
)
environment[
'TDM_XMS_SIZE'] = UserInputHelper.get_input_with_default(
prompt='TDM JVM Min Heap Size',
default=options['environment']['TDM_XMS_SIZE'],
)
options.update(environment=environment)
mount_data_dir = UserInputHelper.get_valid_choice_input(
prompt='Bind the THREDDS data directory to the host?',
choices=['y', 'n'],
default='y',
)
if mount_data_dir.lower() == 'y':
tethys_home = get_tethys_home_dir()
default_mount_location = os.path.join(tethys_home, 'thredds')
thredds_data_volume = '/usr/local/tomcat/content/thredds'
mount_location = UserInputHelper.get_valid_directory_input(
prompt=
'Specify location to bind the THREDDS data directory',
default=default_mount_location)
mounts = [
Mount(thredds_data_volume, mount_location, type='bind')
]
options['host_config'].update(mounts=mounts)
return options
def remove(self):
write_pretty_output('Removing {} container...'.format(
self.display_name))
self.container.remove()
def scaffold_command(args):
"""
Create a new Tethys app projects in the current directory.
"""
# Log
log = logging.getLogger('tethys')
# log.setLevel(logging.DEBUG)
log.debug('Command args: {}'.format(args))
# Get template dirs
log.debug('APP_PATH: {}'.format(APP_PATH))
log.debug('EXTENSION_PATH: {}'.format(EXTENSION_PATH))
# Get template root directory
is_extension = False
if args.extension:
is_extension = True
template_name = args.template
template_root = os.path.join(EXTENSION_PATH, args.template)
else:
template_name = args.template
template_root = os.path.join(APP_PATH, args.template)
log.debug('Template root directory: {}'.format(template_root))
# Validate template
if not os.path.isdir(template_root):
write_pretty_output(
'Error: "{}" is not a valid template.'.format(template_name),
FG_WHITE)
exit(1)
# Validate project name
project_name = args.name
# Only lowercase
contains_uppers = False
for letter in project_name:
if letter.isupper():
contains_uppers = True
break
if contains_uppers:
before = project_name
project_name = project_name.lower()
write_pretty_output(
'Warning: Uppercase characters in project name "{0}" '
'changed to lowercase: "{1}".'.format(before, project_name),
FG_YELLOW)
# Check for valid characters name
project_error_regex = re.compile(r'^[a-zA-Z0-9_]+$')
project_warning_regex = re.compile(r'^[a-zA-Z0-9_-]+$')
# Only letters, numbers and underscores allowed in app names
if not project_error_regex.match(project_name):
# If the only offending character is a dash, replace dashes with underscores and notify user
if project_warning_regex.match(project_name):
before = project_name
project_name = project_name.replace('-', '_')
write_pretty_output(
'Warning: Dashes in project name "{0}" have been replaced '
'with underscores "{1}"'.format(before, project_name),
FG_YELLOW)
# Otherwise, throw error
else:
write_pretty_output(
'Error: Invalid characters in project name "{0}". '
'Only letters, numbers, and underscores.'.format(project_name),
FG_YELLOW)
exit(1)
# Project name derivatives
project_dir = '{0}-{1}'.format(
EXTENSION_PREFIX if is_extension else APP_PREFIX, project_name)
split_project_name = project_name.split('_')
title_case_project_name = [x.title() for x in split_project_name]
default_proper_name = ' '.join(title_case_project_name)
class_name = ''.join(title_case_project_name)
default_theme_color = get_random_color()
write_pretty_output(
'Creating new Tethys project named "{0}".'.format(project_dir),
FG_WHITE)
# Get metadata from user
if not is_extension:
metadata_input = (
{
'name': 'proper_name',
'prompt': 'Proper name for the app (e.g.: "My First App")',
'default': default_proper_name,
'validator': proper_name_validator
},
{
'name': 'description',
'prompt': 'Brief description of the app',
'default': '',
'validator': None
},
{
'name': 'color',
'prompt': 'App theme color (e.g.: "#27AE60")',
'default': default_theme_color,
'validator': theme_color_validator
},
{
'name': 'tags',
'prompt': 'Tags: Use commas to delineate tags and '
'quotes around each tag (e.g.: "Hydrology","Reference Timeseries")',
'default': '',
'validator': None
},
{
'name': 'author',
'prompt': 'Author name',
'default': '',
'validator': None
},
{
'name': 'author_email',
'prompt': 'Author email',
'default': '',
'validator': None
},
{
'name': 'license_name',
'prompt': 'License name',
'default': '',
'validator': None
},
)
else:
metadata_input = (
{
'name': 'proper_name',
'prompt':
'Proper name for the extension (e.g.: "My First Extension")',
'default': default_proper_name,
'validator': proper_name_validator
},
{
'name': 'description',
'prompt': 'Brief description of the extension',
'default': '',
'validator': None
},
{
'name': 'author',
'prompt': 'Author name',
'default': '',
'validator': None
},
{
'name': 'author_email',
'prompt': 'Author email',
'default': '',
'validator': None
},
{
'name': 'license_name',
'prompt': 'License name',
'default': '',
'validator': None
},
)
# Build up template context
context = {
'project': project_name,
'project_dir': project_dir,
'project_url': project_name.replace('_', '-'),
'class_name': class_name,
'proper_name': default_proper_name,
'description': '',
'color': default_theme_color,
'tags': '',
'author': '',
'author_email': '',
'license_name': ''
}
if not args.use_defaults:
# Collect metadata input from user
for item in metadata_input:
valid = False
response = item['default']
while not valid:
try:
response = input('{0} ["{1}"]: '.format(
item['prompt'], item['default'])) or item['default']
except (KeyboardInterrupt, SystemExit):
write_pretty_output('\nScaffolding cancelled.', FG_YELLOW)
exit(1)
if callable(item['validator']):
valid, response = item['validator'](response,
item['default'])
else:
valid = True
if not valid:
write_pretty_output(
'Invalid response: {}'.format(response), FG_RED)
context[item['name']] = response
log.debug('Template context: {}'.format(context))
# Create root directory
project_root = os.path.join(os.getcwd(), project_dir)
log.debug('Project root path: {}'.format(project_root))
if os.path.isdir(project_root):
if not args.overwrite:
valid = False
negative_choices = ['n', 'no', '']
valid_choices = ['y', 'n', 'yes', 'no']
default = 'y'
response = ''
while not valid:
try:
response = input('Directory "{}" already exists. '
'Would you like to overwrite it? [Y/n]: '.
format(project_root)) or default
except (KeyboardInterrupt, SystemExit):
write_pretty_output('\nScaffolding cancelled.', FG_YELLOW)
exit(1)
if response.lower() in valid_choices:
valid = True
if response.lower() in negative_choices:
write_pretty_output('Scaffolding cancelled.', FG_YELLOW)
exit(0)
try:
shutil.rmtree(project_root)
except OSError:
write_pretty_output(
'Error: Unable to overwrite "{}". '
'Please remove the directory and try again.'.format(
project_root), FG_YELLOW)
exit(1)
# Walk the template directory, creating the templates and directories in the new project as we go
for curr_template_root, dirs, template_files in os.walk(template_root):
curr_project_root = curr_template_root.replace(template_root,
project_root)
curr_project_root = render_path(curr_project_root, context)
# Create Root Directory
os.makedirs(curr_project_root)
write_pretty_output('Created: "{}"'.format(curr_project_root),
FG_WHITE)
# Create Files
for template_file in template_files:
template_file_path = os.path.join(curr_template_root,
template_file)
project_file = template_file.replace(TEMPLATE_SUFFIX, '')
project_file_path = os.path.join(curr_project_root, project_file)
# Load the template
log.debug('Loading template: "{}"'.format(template_file_path))
try:
with open(template_file_path, 'r') as tfp:
template = Template(tfp.read())
except UnicodeDecodeError:
with open(template_file_path, 'br') as tfp:
with open(project_file_path, 'bw') as pfp:
pfp.write(tfp.read())
continue
# Render template if loaded
log.debug('Rendering template: "{}"'.format(template_file_path))
if template:
with open(project_file_path, 'w') as pfp:
pfp.write(template.render(context))
write_pretty_output('Created: "{}"'.format(project_file_path),
FG_WHITE)
write_pretty_output(
'Successfully scaffolded new project "{}"'.format(project_name),
FG_WHITE)
def log_pull_stream(stream):
"""
Handle the printing of pull statuses
"""
if platform.system() == 'Windows': # i.e. can't uses curses
for block in stream:
lines = [l for l in block.split(b'\r\n') if l]
for line in lines:
json_line = json.loads(line)
current_id = "{}:".format(
json_line['id']) if 'id' in json_line else ''
current_status = json_line[
'status'] if 'status' in json_line else ''
current_progress = json_line[
'progress'] if 'progress' in json_line else ''
write_pretty_output("{id}{status} {progress}".format(
id=current_id,
status=current_status,
progress=current_progress))
else:
TERMINAL_STATUSES = [
'Already exists', 'Download complete', 'Pull complete'
]
PROGRESS_STATUSES = ['Downloading', 'Extracting']
STATUSES = TERMINAL_STATUSES + PROGRESS_STATUSES + [
'Pulling fs layer', 'Waiting', 'Verifying Checksum'
]
NUMBER_OF_HEADER_ROWS = 2
header_rows = list()
message_log = list()
progress_messages = dict()
messages_to_print = list()
# prepare console for curses window printing
stdscr = curses.initscr()
curses.noecho()
curses.cbreak()
try:
for block in stream:
lines = [l for l in block.split(b'\r\n') if l]
for line in lines:
json_line = json.loads(line)
current_id = json_line['id'] if 'id' in json_line else None
current_status = json_line[
'status'] if 'status' in json_line else ''
current_progress = json_line[
'progress'] if 'progress' in json_line else ''
if current_id is None:
# save messages to print after docker images are pulled
messages_to_print.append(current_status.strip())
else:
# use curses window to properly display progress
if current_status not in STATUSES: # Assume this is the header
header_rows.append(current_status)
header_rows.append('-' * len(current_status))
elif current_status in PROGRESS_STATUSES:
# add messages with progress to dictionary to print at the bottom of the screen
progress_messages[current_id] = {
'id': current_id,
'status': current_status,
'progress': current_progress
}
else:
# add all other messages to list to show above progress messages
message_log.append(
"{id}: {status} {progress}".format(
id=current_id,
status=current_status,
progress=current_progress))
# remove messages from progress that have completed
if current_id in progress_messages:
del progress_messages[current_id]
# update window
# row/column calculations for proper display on screen
maxy, maxx = stdscr.getmaxyx()
number_of_rows, number_of_columns = maxy, maxx
current_progress_messages = sorted(
progress_messages.values(),
key=lambda message: STATUSES.index(message['status'
]))
# row/column calculations for proper display on screen
number_of_progress_rows = len(
current_progress_messages)
number_of_message_rows = number_of_rows - number_of_progress_rows - NUMBER_OF_HEADER_ROWS
# slice messages to only those that will fit on the screen
current_messages = [
''
] * number_of_message_rows + message_log
current_messages = current_messages[
-number_of_message_rows:]
rows = header_rows + current_messages + [
'{id}: {status} {progress}'.format(
**current_message)
for current_message in current_progress_messages
]
for row, message in enumerate(rows):
message += ' ' * number_of_columns
message = message[:number_of_columns - 1]
stdscr.addstr(row, 0, message)
stdscr.refresh()
finally: # always reset console to normal regardless of success or failure
curses.echo()
curses.nocbreak()
curses.endwin()
write_pretty_output('\n'.join(messages_to_print))
def get_container_options(self, defaults):
# default configuration
options = self.default_container_options()
if not self.is_cluster:
# Then all of the other options are irrelevant
defaults = True
if not defaults:
# Environmental variables from user input
environment = dict()
write_pretty_output(
"The GeoServer docker can be configured to run in a clustered mode (multiple instances of "
"GeoServer running in the docker container) for better performance.\n"
)
environment[
'ENABLED_NODES'] = UserInputHelper.get_valid_numeric_input(
prompt='Number of GeoServer Instances Enabled',
max_val=4,
)
environment['REST_NODES'] = UserInputHelper.get_valid_numeric_input(
prompt='Number of GeoServer Instances with REST API Enabled',
max_val=int(environment['ENABLED_NODES']),
)
write_pretty_output(
"\nGeoServer can be configured with limits to certain types of requests to prevent it from "
"becoming overwhelmed. This can be done automatically based on a number of processors or "
"each "
"limit can be set explicitly.\n")
flow_control_mode = UserInputHelper.get_valid_choice_input(
prompt=
'Would you like to specify number of Processors (c) OR set request limits explicitly (e)',
choices=['c', 'e'],
default='c',
)
if flow_control_mode.lower() == 'c':
environment[
'NUM_CORES'] = UserInputHelper.get_valid_numeric_input(
prompt='Number of Processors',
max_val=4, # TODO dynamically figure out what the max is
)
else:
environment[
'MAX_OWS_GLOBAL'] = UserInputHelper.get_valid_numeric_input(
prompt=
'Maximum number of simultaneous OGC web service requests (e.g.: WMS, WCS, WFS)',
default=100)
environment[
'MAX_WMS_GETMAP'] = UserInputHelper.get_valid_numeric_input(
prompt='Maximum number of simultaneous GetMap requests',
default=8)
environment[
'MAX_OWS_GWC'] = UserInputHelper.get_valid_numeric_input(
prompt=
'Maximum number of simultaneous GeoWebCache tile renders',
default=16)
environment[
'MAX_TIMEOUT'] = UserInputHelper.get_valid_numeric_input(
prompt='Maximum request timeout in seconds', default=60)
environment['MAX_MEMORY'] = UserInputHelper.get_valid_numeric_input(
prompt=
'Maximum memory to allocate to each GeoServer instance in MB',
max_val=4096, # TODO dynamically figure out what the max is
default=1024)
max_memory = int(environment['MAX_MEMORY'])
environment['MIN_MEMORY'] = UserInputHelper.get_valid_numeric_input(
prompt=
'Minimum memory to allocate to each GeoServer instance in MB',
max_val=max_memory,
default=max_memory)
options.update(environment=environment, )
mount_data_dir = UserInputHelper.get_valid_choice_input(
prompt='Bind the GeoServer data directory to the host?',
choices=['y', 'n'],
default='y',
)
if mount_data_dir.lower() == 'y':
tethys_home = os.environ.get('TETHYS_HOME',
os.path.expanduser('~/tethys/'))
default_mount_location = os.path.join(tethys_home, 'geoserver',
'data')
gs_data_volume = '/var/geoserver/data'
mount_location = UserInputHelper.get_valid_directory_input(
prompt='Specify location to bind data directory',
default=default_mount_location)
mounts = [Mount(gs_data_volume, mount_location, type='bind')]
options['host_config'].update(mounts=mounts)
return options
