def process_args(args):
"""Process the command line arguments to provide kwargs to command functions.
Args:
args: command line argument object
Returns: dict of kwargs
"""
db_settings = settings.DATABASES[args.db_alias]
db_dir = db_settings.get('DIR')
if db_dir is None:
if args.command in ['init', 'start', 'stop']:
raise RuntimeError(
f'The tethys db {args.command} command can only be used with local databases.'
)
else:
if not Path(db_dir).is_absolute():
db_dir = Path(get_tethys_home_dir()) / db_dir
options = vars(args)
options.update(
db_alias=args.db_alias,
db_dir=db_dir,
hostname=db_settings.get('HOST'),
port=db_settings.get('PORT'),
db_name=db_settings.get('NAME'),
)
return options
def run_portal_install(app_name):
file_path = Path(get_tethys_home_dir()) / 'portal_config.yml'
if not file_path.exists():
write_msg(
"No Portal Services file found. Searching for local app level services.yml..."
)
return False
write_msg("Portal install file found...Processing...")
portal_options = open_file(file_path)
app_check = portal_options and 'apps' in portal_options and portal_options[
'apps']
if app_check and app_name in portal_options[
'apps'] and 'services' in portal_options['apps'][app_name]:
services = portal_options['apps'][app_name]['services']
if services and len(services) > 0:
configure_services_from_file(services, app_name)
else:
write_msg(
"No app configuration found for app: {} in portal config file. "
"Searching for local app level services.yml... ".format(
app_name))
return False
else:
write_msg("No apps configuration found in portal config file. "
"Searching for local app level services.yml... ")
return False
return True
def gen_site_content(args):
load_apps()
from tethys_config.models import Setting, SettingsCategory
if args.restore_defaults:
from tethys_config.init import setting_defaults
Setting.objects.all().delete()
general_category = SettingsCategory.objects.get(
name="General Settings")
setting_defaults(general_category)
home_category = SettingsCategory.objects.get(name="Home Page")
setting_defaults(home_category)
if args.from_file:
portal_yaml = Path(get_tethys_home_dir()) / 'portal_config.yml'
if portal_yaml.exists():
with portal_yaml.open() as f:
site_content_settings = yaml.safe_load(f).get(
'site_content', {})
for arg in site_content_settings:
if site_content_settings[arg]:
content = site_content_settings[arg]
obj = Setting.objects.filter(
name=arg_filter[arg.lower()])
obj.update(content=content,
date_modified=timezone.now())
else:
valid_inputs = ('y', 'n', 'yes', 'no')
no_inputs = ('n', 'no')
generate_input = input(
'Would you like to generate a template portal_config.yml file that you can then'
'customize? (y/n): ')
while generate_input not in valid_inputs:
generate_input = input(
'Invalid option. Try again. (y/n): ').lower()
if generate_input in no_inputs:
write_msg(
'Generation of portal_config.yml file cancelled. Please generate one manually or provide '
'specific site content arguments.')
else:
call(['tethys', 'gen', 'portal'])
write_msg(
'\nRe-run the tethys site command with the --from-file argument.'
)
exit(0)
for arg in vars(args):
if vars(args)[arg] and arg in arg_filter:
content = vars(args)[arg].replace('\\n', '\n')
obj = Setting.objects.filter(name=arg_filter[arg])
obj.update(content=content, date_modified=timezone.now())
def test_get_tethys_home_dir__tethys_home_defined(self, mock_os):
env_tethys_home = '/foo/.bar'
conda_default_env = 'foo'
mock_os.environ.get.side_effect = [env_tethys_home, conda_default_env] # [TETHYS_HOME, CONDA_DEFAULT_ENV]
ret = utilities.get_tethys_home_dir()
mock_os.environ.get.assert_called_once_with('TETHYS_HOME')
mock_os.path.expanduser.assert_not_called()
# Returns path defined by TETHYS_HOME environment variable
self.assertEqual(env_tethys_home, ret)
def test_get_tethys_home_dir__default_env_name__tethys_home_not_defined(self, mock_os):
env_tethys_home = None
conda_default_env = 'tethys' # Default Tethys environment name
default_tethys_home = '/home/tethys/.tethys'
mock_os.environ.get.side_effect = [env_tethys_home, conda_default_env] # [TETHYS_HOME, CONDA_DEFAULT_ENV]
mock_os.path.expanduser.return_value = default_tethys_home
ret = utilities.get_tethys_home_dir()
mock_os.environ.get.assert_any_call('TETHYS_HOME')
mock_os.environ.get.assert_any_call('CONDA_DEFAULT_ENV')
mock_os.path.expanduser.assert_called_with('~/.tethys')
# Returns default tethys home environment
self.assertEqual(default_tethys_home, ret)
def gen_settings(args):
TETHYS_HOME = get_tethys_home_dir()
# Generate context variables
secret_key = ''.join([random.choice(string.ascii_letters + string.digits) for _ in range(50)])
context = {
'secret_key': secret_key,
'allowed_host': args.allowed_host,
'allowed_hosts': args.allowed_hosts,
'db_name': args.db_name,
'db_username': args.db_username,
'db_password': args.db_password,
'db_port': args.db_port,
'db_dir': args.db_dir,
'tethys_home': TETHYS_HOME,
'production': args.production,
'open_portal': args.open_portal
}
return context
def test_get_tethys_home_dir__exception(self, mock_os, mock_tethys_log):
env_tethys_home = None
conda_default_env = 'foo' # Non-default Tethys environment name
default_tethys_home = '/home/tethys/.tethys'
mock_os.environ.get.side_effect = [env_tethys_home, conda_default_env] # [TETHYS_HOME, CONDA_DEFAULT_ENV]
mock_os.path.expanduser.return_value = default_tethys_home
mock_os.path.join.side_effect = Exception
ret = utilities.get_tethys_home_dir()
mock_os.environ.get.assert_any_call('TETHYS_HOME')
mock_os.environ.get.assert_any_call('CONDA_DEFAULT_ENV')
mock_os.path.expanduser.assert_called_with('~/.tethys')
mock_os.path.join.assert_called_with(default_tethys_home, conda_default_env)
mock_tethys_log.warning.assert_called()
# Returns default tethys home environment path
self.assertEqual(default_tethys_home, ret)
def generate_command(args):
"""
Generate a settings file for a new installation.
"""
# Consts
TETHYS_HOME = get_tethys_home_dir()
TETHYS_SRC = get_tethys_src_dir()
# Setup variables
context = Context()
# Determine template path
gen_templates_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'gen_templates')
template_path = os.path.join(gen_templates_dir, args.type)
# Parse template
template = Template(open(template_path).read())
# Determine destination file name (defaults to type)
destination_file = FILE_NAMES[args.type]
# Default destination path is the current working directory
destination_dir = os.path.join(TETHYS_SRC, 'tethys_portal')
nginx_user = ''
nginx_conf_path = '/etc/nginx/nginx.conf'
if os.path.exists(nginx_conf_path):
with open(nginx_conf_path, 'r') as nginx_conf:
for line in nginx_conf.readlines():
tokens = line.split()
if len(tokens) > 0 and tokens[0] == 'user':
nginx_user = tokens[1].strip(';')
break
# Settings file setup
if args.type == GEN_SETTINGS_OPTION:
# Generate context variables
secret_key = ''.join([random.choice(string.ascii_letters + string.digits) for _ in range(50)])
context.update({'secret_key': secret_key,
'allowed_host': args.allowed_host,
'allowed_hosts': args.allowed_hosts,
'db_username': args.db_username,
'db_password': args.db_password,
'db_port': args.db_port,
'tethys_home': TETHYS_HOME,
'production': args.production,
})
if args.type == GEN_NGINX_OPTION:
hostname = str(settings.ALLOWED_HOSTS[0]) if len(settings.ALLOWED_HOSTS) > 0 else '127.0.0.1'
workspaces_root = get_settings_value('TETHYS_WORKSPACES_ROOT')
static_root = get_settings_value('STATIC_ROOT')
context.update({'hostname': hostname,
'workspaces_root': workspaces_root,
'static_root': static_root,
'client_max_body_size': args.client_max_body_size
})
if args.type == GEN_UWSGI_SERVICE_OPTION:
conda_home = get_environment_value('CONDA_HOME')
conda_env_name = get_environment_value('CONDA_ENV_NAME')
user_option_prefix = ''
try:
linux_distro = linux_distribution(full_distribution_name=0)[0]
if linux_distro in ['redhat', 'centos']:
user_option_prefix = 'http-'
except Exception:
pass
context.update({'nginx_user': nginx_user,
'conda_home': conda_home,
'conda_env_name': conda_env_name,
'tethys_src': TETHYS_SRC,
'user_option_prefix': user_option_prefix
})
if args.type == GEN_UWSGI_SETTINGS_OPTION:
conda_home = get_environment_value('CONDA_HOME')
conda_env_name = get_environment_value('CONDA_ENV_NAME')
context.update({'conda_home': conda_home,
'conda_env_name': conda_env_name,
'uwsgi_processes': args.uwsgi_processes})
if args.directory:
if os.path.isdir(args.directory):
destination_dir = args.directory
else:
print('ERROR: "{0}" is not a valid directory.'.format(destination_dir))
exit(1)
destination_path = os.path.join(destination_dir, destination_file)
# Check for pre-existing file
if os.path.isfile(destination_path):
valid_inputs = ('y', 'n', 'yes', 'no')
no_inputs = ('n', 'no')
if args.overwrite:
overwrite_input = 'yes'
else:
overwrite_input = input('WARNING: "{0}" already exists. '
'Overwrite? (y/n): '.format(destination_file)).lower()
while overwrite_input not in valid_inputs:
overwrite_input = input('Invalid option. Overwrite? (y/n): ').lower()
if overwrite_input in no_inputs:
print('Generation of "{0}" cancelled.'.format(destination_file))
exit(0)
# Render template and write to file
if template:
with open(destination_path, 'w') as f:
f.write(template.render(context))
* Created On: 2019
* Copyright: (c) Brigham Young University 2014
* License: BSD 2-Clause
********************************************************************************
"""
from pathlib import Path
from pprint import pformat
import yaml
from tethys_cli.cli_colors import write_info, write_error
from tethys_apps.utilities import get_tethys_home_dir
from django.conf import settings
TETHYS_HOME = Path(get_tethys_home_dir())
def add_settings_parser(subparsers):
settings_parser = subparsers.add_parser(
'settings', help='Tethys settings configuration command.')
settings_parser.add_argument(
'-s',
'--set',
dest='set_kwargs',
help=
'Key Value pairs to add to the settings in the portal_config.yml file. Hierarchical keys can be '
'specified with dot notation. (e.g. DATABASES.default.NAME)',
nargs=2,
action='append',
)
import sys
import yaml
import logging
import datetime as dt
from django.contrib.messages import constants as message_constants
from tethys_apps.utilities import get_tethys_home_dir
from tethys_cli.gen_commands import generate_secret_key
from bokeh.settings import settings as bokeh_settings
log = logging.getLogger(__name__)
this_module = sys.modules[__name__]
BASE_DIR = os.path.dirname(__file__)
TETHYS_HOME = get_tethys_home_dir()
portal_config_settings = {}
try:
with open(os.path.join(TETHYS_HOME, 'portal_config.yml')) as portal_yaml:
portal_config_settings = yaml.safe_load(portal_yaml).get('settings', {}) or {}
except FileNotFoundError:
log.info('Could not find the portal_config.yml file. To generate a new portal_config.yml run the command '
'"tethys gen portal_config"')
except Exception:
log.exception('There was an error while attempting to read the settings from the portal_config.yml file.')
bokeh_settings.resources = portal_config_settings.pop('BOKEH_RESOURCES', 'cdn')
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/
def get_container_options(self, defaults):
# Default environmental vars
options = self.default_container_options()
if not defaults:
environment = dict()
write_pretty_output(
"Provide configuration options for the THREDDS container or or press enter to "
"accept the defaults shown in square brackets: ")
environment['TDM_PW'] = UserInputHelper.get_verified_password(
prompt='TDM Password',
default=options['environment']['TDM_PW'],
)
environment['TDS_HOST'] = UserInputHelper.get_input_with_default(
prompt='TDS Host',
default=options['environment']['TDS_HOST'],
)
environment[
'THREDDS_XMX_SIZE'] = UserInputHelper.get_input_with_default(
prompt='TDS JVM Max Heap Size',
default=options['environment']['THREDDS_XMX_SIZE'],
)
environment[
'THREDDS_XMS_SIZE'] = UserInputHelper.get_input_with_default(
prompt='TDS JVM Min Heap Size',
default=options['environment']['THREDDS_XMS_SIZE'],
)
environment[
'TDM_XMX_SIZE'] = UserInputHelper.get_input_with_default(
prompt='TDM JVM Max Heap Size',
default=options['environment']['TDM_XMX_SIZE'],
)
environment[
'TDM_XMS_SIZE'] = UserInputHelper.get_input_with_default(
prompt='TDM JVM Min Heap Size',
default=options['environment']['TDM_XMS_SIZE'],
)
options.update(environment=environment)
mount_data_dir = UserInputHelper.get_valid_choice_input(
prompt='Bind the THREDDS data directory to the host?',
choices=['y', 'n'],
default='y',
)
if mount_data_dir.lower() == 'y':
tethys_home = get_tethys_home_dir()
default_mount_location = os.path.join(tethys_home, 'thredds')
thredds_data_volume = '/usr/local/tomcat/content/thredds'
mount_location = UserInputHelper.get_valid_directory_input(
prompt=
'Specify location to bind the THREDDS data directory',
default=default_mount_location)
mounts = [
Mount(thredds_data_volume, mount_location, type='bind')
]
options['host_config'].update(mounts=mounts)
return options
def get_container_options(self, defaults):
# default configuration
options = self.default_container_options()
if not self.is_cluster:
# Then all of the other options are irrelevant
defaults = True
if not defaults:
# Environmental variables from user input
environment = dict()
write_pretty_output(
"The GeoServer docker can be configured to run in a clustered mode (multiple instances of "
"GeoServer running in the docker container) for better performance.\n"
)
environment[
'ENABLED_NODES'] = UserInputHelper.get_valid_numeric_input(
prompt='Number of GeoServer Instances Enabled',
max_val=4,
)
environment['REST_NODES'] = UserInputHelper.get_valid_numeric_input(
prompt='Number of GeoServer Instances with REST API Enabled',
max_val=int(environment['ENABLED_NODES']),
)
write_pretty_output(
"\nGeoServer can be configured with limits to certain types of requests to prevent it from "
"becoming overwhelmed. This can be done automatically based on a number of processors or "
"each "
"limit can be set explicitly.\n")
flow_control_mode = UserInputHelper.get_valid_choice_input(
prompt=
'Would you like to specify number of Processors (c) OR set request limits explicitly (e)',
choices=['c', 'e'],
default='c',
)
if flow_control_mode.lower() == 'c':
environment[
'NUM_CORES'] = UserInputHelper.get_valid_numeric_input(
prompt='Number of Processors',
max_val=4, # TODO dynamically figure out what the max is
)
else:
environment[
'MAX_OWS_GLOBAL'] = UserInputHelper.get_valid_numeric_input(
prompt=
'Maximum number of simultaneous OGC web service requests (e.g.: WMS, WCS, WFS)',
default=100)
environment[
'MAX_WMS_GETMAP'] = UserInputHelper.get_valid_numeric_input(
prompt='Maximum number of simultaneous GetMap requests',
default=8)
environment[
'MAX_OWS_GWC'] = UserInputHelper.get_valid_numeric_input(
prompt=
'Maximum number of simultaneous GeoWebCache tile renders',
default=16)
environment[
'MAX_TIMEOUT'] = UserInputHelper.get_valid_numeric_input(
prompt='Maximum request timeout in seconds', default=60)
environment['MAX_MEMORY'] = UserInputHelper.get_valid_numeric_input(
prompt=
'Maximum memory to allocate to each GeoServer instance in MB',
max_val=4096, # TODO dynamically figure out what the max is
default=1024)
max_memory = int(environment['MAX_MEMORY'])
environment['MIN_MEMORY'] = UserInputHelper.get_valid_numeric_input(
prompt=
'Minimum memory to allocate to each GeoServer instance in MB',
max_val=max_memory,
default=max_memory)
options.update(environment=environment, )
mount_data_dir = UserInputHelper.get_valid_choice_input(
prompt='Bind the GeoServer data directory to the host?',
choices=['y', 'n'],
default='y',
)
if mount_data_dir.lower() == 'y':
tethys_home = get_tethys_home_dir()
default_mount_location = os.path.join(tethys_home, 'geoserver',
'data')
gs_data_volume = '/var/geoserver/data'
mount_location = UserInputHelper.get_valid_directory_input(
prompt='Specify location to bind data directory',
default=default_mount_location)
mounts = [Mount(gs_data_volume, mount_location, type='bind')]
options['host_config'].update(mounts=mounts)
return options
def generate_command(args):
"""
Generate a settings file for a new installation.
"""
# Consts
TETHYS_HOME = get_tethys_home_dir()
TETHYS_SRC = get_tethys_src_dir()
# Setup variables
context = Context()
# Determine template path
gen_templates_dir = os.path.join(
os.path.abspath(os.path.dirname(__file__)), 'gen_templates')
template_path = os.path.join(gen_templates_dir, args.type)
# Parse template
template = Template(open(template_path).read())
# Determine destination file name (defaults to type)
destination_file = FILE_NAMES[args.type]
# Default destination path is the current working directory
destination_dir = os.path.join(TETHYS_SRC, 'tethys_portal')
nginx_user = ''
nginx_conf_path = '/etc/nginx/nginx.conf'
if os.path.exists(nginx_conf_path):
with open(nginx_conf_path, 'r') as nginx_conf:
for line in nginx_conf.readlines():
tokens = line.split()
if len(tokens) > 0 and tokens[0] == 'user':
nginx_user = tokens[1].strip(';')
break
# Settings file setup
if args.type == GEN_SETTINGS_OPTION:
# Generate context variables
secret_key = ''.join([
random.choice(string.ascii_letters + string.digits)
for _ in range(50)
])
context.update({
'secret_key': secret_key,
'allowed_host': args.allowed_host,
'allowed_hosts': args.allowed_hosts,
'db_username': args.db_username,
'db_password': args.db_password,
'db_port': args.db_port,
'tethys_home': TETHYS_HOME,
'production': args.production,
'open_portal': args.open_portal
})
if args.type == GEN_NGINX_OPTION:
hostname = str(settings.ALLOWED_HOSTS[0]) if len(
settings.ALLOWED_HOSTS) > 0 else '127.0.0.1'
workspaces_root = get_settings_value('TETHYS_WORKSPACES_ROOT')
static_root = get_settings_value('STATIC_ROOT')
context.update({
'hostname': hostname,
'workspaces_root': workspaces_root,
'static_root': static_root,
'client_max_body_size': args.client_max_body_size
})
if args.type == GEN_UWSGI_SERVICE_OPTION:
conda_home = get_environment_value('CONDA_HOME')
conda_env_name = get_environment_value('CONDA_ENV_NAME')
user_option_prefix = ''
try:
linux_distro = linux_distribution(full_distribution_name=0)[0]
if linux_distro in ['redhat', 'centos']:
user_option_prefix = 'http-'
except Exception:
pass
context.update({
'nginx_user': nginx_user,
'conda_home': conda_home,
'conda_env_name': conda_env_name,
'tethys_src': TETHYS_SRC,
'user_option_prefix': user_option_prefix
})
if args.type == GEN_UWSGI_SETTINGS_OPTION:
conda_home = get_environment_value('CONDA_HOME')
conda_env_name = get_environment_value('CONDA_ENV_NAME')
context.update({
'conda_home': conda_home,
'conda_env_name': conda_env_name,
'uwsgi_processes': args.uwsgi_processes
})
if args.directory:
if os.path.isdir(args.directory):
destination_dir = args.directory
else:
print('ERROR: "{0}" is not a valid directory.'.format(
destination_dir))
exit(1)
destination_path = os.path.join(destination_dir, destination_file)
# Check for pre-existing file
if os.path.isfile(destination_path):
valid_inputs = ('y', 'n', 'yes', 'no')
no_inputs = ('n', 'no')
if args.overwrite:
overwrite_input = 'yes'
else:
overwrite_input = input(
'WARNING: "{0}" already exists. '
'Overwrite? (y/n): '.format(destination_file)).lower()
while overwrite_input not in valid_inputs:
overwrite_input = input(
'Invalid option. Overwrite? (y/n): ').lower()
if overwrite_input in no_inputs:
print('Generation of "{0}" cancelled.'.format(destination_file))
exit(0)
# Render template and write to file
if template:
with open(destination_path, 'w') as f:
f.write(template.render(context))
