def install_packages(conda_config, update_installed=False):
# Compile channels arguments
install_args = []
if validate_schema('channels', conda_config):
channels = conda_config['channels']
for channel in channels:
install_args.extend(['-c', channel])
# Install all Packages
if validate_schema('packages', conda_config):
if not update_installed:
install_args.extend(['--freeze-installed'])
else:
write_warning(
'Warning: Updating previously installed packages. This could break your Tethys environment.'
)
install_args.extend(conda_config['packages'])
write_msg("Running conda installation tasks...")
[resp, err, code] = conda_run(Commands.INSTALL,
*install_args,
use_exception_handler=False,
stdout=None,
stderr=None)
if code != 0:
write_error(
'Warning: Packages installation ran into an error. Please try again or a manual install'
)
def get_destination_path(args):
# Determine destination file name (defaults to type)
destination_file = FILE_NAMES[args.type]
# Default destination path is the tethys_portal source dir
destination_dir = TETHYS_HOME
# Make the Tethys Home directory if it doesn't exist yet.
if not os.path.isdir(destination_dir):
os.makedirs(destination_dir, exist_ok=True)
if args.type in [GEN_SERVICES_OPTION, GEN_INSTALL_OPTION]:
destination_dir = os.getcwd()
elif args.type == GEN_META_YAML_OPTION:
destination_dir = os.path.join(TETHYS_SRC, 'conda.recipe')
if args.directory:
destination_dir = os.path.abspath(args.directory)
if not os.path.isdir(destination_dir):
write_error('ERROR: "{0}" is not a valid directory.'.format(destination_dir))
exit(1)
destination_path = os.path.join(destination_dir, destination_file)
check_for_existing_file(destination_path, destination_file, args.overwrite)
return destination_path
def configure_services(services, app_name):
from tethys_apps.models import CustomSetting
if services['version']:
del services['version']
for service_type in services:
if services[service_type] is not None:
current_services = services[service_type]
for service_setting_name in current_services:
if service_type == 'custom_setting':
custom_setting = CustomSetting.objects.get(
name=service_setting_name)
try:
custom_setting.value = current_services[
service_setting_name]
custom_setting.clean()
custom_setting.save()
except ValidationError:
write_error(
"Incorrect value type given for custom setting '{}'. Please adjust "
"services.yml or set the value in the app's settings page."
.format(service_setting_name))
else:
find_and_link(service_type, service_setting_name,
current_services[service_setting_name],
app_name)
def open_file(file_path):
try:
with file_path.open() as f:
return yaml.safe_load(f)
except Exception as e:
write_error(str(e))
write_error(
'An unexpected error occurred reading the file. Please try again.')
exit(1)
def find_and_link(service_type, setting_name, service_id, app_name, setting):
valid_service = validate_service_id(service_type, service_id)
setting_type = get_setting_type_from_setting(setting)
if valid_service:
link_service_to_app_setting(service_type, service_id, app_name,
setting_type, setting_name)
else:
write_error(
f'Warning: Could not find service of type: "{service_type}" with the Name/ID: "{service_id}"'
)
def find_and_link(service_type, setting_name, service_name, app_name):
service = get_service_from_name(service_name)
if service:
link_service_to_app_setting(service['service_type'], service_name,
app_name, service['linkParam'],
setting_name)
else:
write_error(
'Warning: Could not find service of type: {} with the name/id: {}'.
format(service_type, service_name))
def get_docker_client(cls):
"""
Configure DockerClient
"""
if cls._docker_client is None:
try:
cls._docker_client = docker.from_env()
except docker.errors.DockerException:
write_error(
'The Docker daemon must be running to use the tethys docker command.'
)
exit(1)
return cls._docker_client
def purge_db_server(**kwargs):
"""Stop Tethys db server (if running) and then remove the db directory.
Args:
**kwargs: processed key word arguments from commandline
"""
kwargs['exit_on_error'] = False
stop_db_server(**kwargs)
if kwargs['db_dir']:
write_error(
"This action will permanently delete the database. DATA WILL BE LOST!"
)
response = 'y' if kwargs['no_confirmation'] else input(
f"Are you sure you want to continue? [y/N]: ")
if response.lower() in ['y', 'yes']:
shutil.rmtree(kwargs['db_dir'])
def _get_dict_key_handle(d, key, not_exists_okay=False):
keys = key.split('.')
values = [d]
for k in keys:
try:
values.append(values[-1][k])
except Exception:
if not_exists_okay:
if len(keys) > len(values):
values[-1][k] = {}
values.append(values[-1][k])
else:
return values[-1], k
else:
write_error(f'The setting {key} does not exists.')
return
return values[-2], k
def _run_process(args, msg, err_msg='ERROR!!!', exit_on_error=True, **kwargs):
"""Run a process while outputting messages. If error then either exit or return error code.
Args:
args: args for process
msg: Message to output before running process
err_msg: Message to output if there is an error
exit_on_error: If True then exit if process returns and error code
**kwargs: processed key word arguments from commandline
Returns: error code
"""
write_info(msg)
err_code = run_process(args)
if err_code:
write_error(err_msg)
if exit_on_error:
exit(err_code)
return err_code
def install_packages(conda_config):
# Compile channels arguments
install_args = []
if validate_schema('channels', conda_config):
channels = conda_config['channels']
for channel in channels:
install_args.extend(['-c', channel])
# Install all Packages
if validate_schema('packages', conda_config):
install_args.extend(conda_config['packages'])
write_msg("Running conda installation tasks...")
[resp, err, code] = conda_run(Commands.INSTALL,
*install_args,
use_exception_handler=False,
stdout=None,
stderr=None)
if code != 0:
write_error(
'Warning: Packages installation ran into an error. Please try again or a manual install'
)
def create_portal_superuser(portal_superuser_name='admin',
portal_superuser_email='',
portal_superuser_password='******',
**kwargs):
"""Create a superuser account for Tethys Portal
Args:
portal_superuser_name: username for the Tethys Portal superuser account
portal_superuser_email: email for the Tethys Portal superuser account
portal_superuser_password: password for the Tethys Portal superuser
**kwargs: processed key word arguments from commandline
"""
write_info(
f'Creating Tethys Portal superuser "{portal_superuser_name}"...')
load_apps()
from django.contrib.auth.models import User # noqa: E402
try:
User.objects.create_superuser(portal_superuser_name,
portal_superuser_email,
portal_superuser_password)
except IntegrityError:
write_error(
f'Tethys Portal Superuser "{portal_superuser_name}" already exists.'
)
def get_app_settings(app):
"""
Get settings related to app
Args:
app(str): name of app
Returns:
dict (linked_settings, unlinked_settings): Dictionary with two keys: linked_settings(list) - list of linked settings, unlinked_settings(list) - list of unlinked settings # noqa: E501
"""
from tethys_cli.cli_colors import write_error
from tethys_apps.models import (TethysApp, PersistentStoreConnectionSetting, PersistentStoreDatabaseSetting,
SpatialDatasetServiceSetting, DatasetServiceSetting, WebProcessingServiceSetting,
CustomSetting)
try:
app = TethysApp.objects.get(package=app)
app_settings = []
for setting in PersistentStoreConnectionSetting.objects.filter(tethys_app=app):
app_settings.append(setting)
for setting in PersistentStoreDatabaseSetting.objects.filter(tethys_app=app):
app_settings.append(setting)
for setting in SpatialDatasetServiceSetting.objects.filter(tethys_app=app):
app_settings.append(setting)
for setting in DatasetServiceSetting.objects.filter(tethys_app=app):
app_settings.append(setting)
for setting in WebProcessingServiceSetting.objects.filter(tethys_app=app):
app_settings.append(setting)
for setting in CustomSetting.objects.filter(tethys_app=app):
app_settings.append(setting)
unlinked_settings = []
linked_settings = []
for setting in app_settings:
if (hasattr(setting, 'spatial_dataset_service') and setting.spatial_dataset_service) \
or (hasattr(setting, 'persistent_store_service') and setting.persistent_store_service) \
or (hasattr(setting, 'dataset_service') and setting.dataset_service) \
or (hasattr(setting, 'web_processing_service') and setting.web_processing_service) \
or (hasattr(setting, 'value') and setting.value != ''):
linked_settings.append(setting)
else:
unlinked_settings.append(setting)
return {
'linked_settings': linked_settings,
'unlinked_settings': unlinked_settings
}
except ObjectDoesNotExist:
write_error('The app you specified ("{0}") does not exist. Command aborted.'.format(app))
except Exception as e:
write_error(str(e))
write_error('Something went wrong. Please try again.')
def test_command(args):
args.manage = False
# Get the path to manage.py
manage_path = get_manage_path(args)
tests_path = os.path.join(TETHYS_SRC_DIRECTORY, 'tests')
try:
check_and_install_prereqs(tests_path)
except FileNotFoundError:
write_error(
'The "tethys test" command will not work because the tests are not installed. '
'To run tests you must download the tethys source code.')
exit(1)
# Define the process to be run
primary_process = ['python', manage_path, 'test']
# Tag to later check if tests are being run on a specific app or extension
app_package_tag = 'tethys_apps.tethysapp.'
extension_package_tag = 'tethysext.'
if args.coverage or args.coverage_html:
os.environ['TETHYS_TEST_DIR'] = tests_path
if args.file and app_package_tag in args.file:
app_package_parts = args.file.split(app_package_tag)
app_name = app_package_parts[1].split('.')[0]
core_app_package = '{}{}'.format(app_package_tag, app_name)
app_package = 'tethysapp.{}'.format(app_name)
config_opt = '--source={},{}'.format(core_app_package, app_package)
elif args.file and extension_package_tag in args.file:
extension_package_parts = args.file.split(extension_package_tag)
extension_name = extension_package_parts[1].split('.')[0]
core_extension_package = '{}{}'.format(extension_package_tag,
extension_name)
extension_package = 'tethysext.{}'.format(extension_name)
config_opt = '--source={},{}'.format(core_extension_package,
extension_package)
else:
config_opt = '--rcfile={0}'.format(
os.path.join(tests_path, 'coverage.cfg'))
primary_process = ['coverage', 'run', config_opt, manage_path, 'test']
if args.file:
primary_process.append(args.file)
elif args.unit:
primary_process.append(os.path.join(tests_path, 'unit_tests'))
elif args.gui:
primary_process.append(os.path.join(tests_path, 'gui_tests'))
if args.verbosity:
primary_process.extend(['-v', args.verbosity])
test_status = run_process(primary_process)
if args.coverage:
if args.file and (app_package_tag in args.file
or extension_package_tag in args.file):
run_process(['coverage', 'report'])
else:
run_process(['coverage', 'report', config_opt])
if args.coverage_html:
report_dirname = 'coverage_html_report'
index_fname = 'index.html'
if args.file and (app_package_tag in args.file
or extension_package_tag in args.file):
run_process([
'coverage', 'html', '--directory={0}'.format(
os.path.join(tests_path, report_dirname))
])
else:
run_process(['coverage', 'html', config_opt])
try:
status = run_process([
'open',
os.path.join(tests_path, report_dirname, index_fname)
])
if status != 0:
raise Exception
except Exception:
webbrowser.open_new_tab(
os.path.join(tests_path, report_dirname, index_fname))
# Removing Test App
# try:
# subprocess.call(['tethys', 'uninstall', 'test_app', '-f'], stdout=FNULL)
# except Exception:
# pass
# try:
# subprocess.call(['tethys', 'uninstall', 'test_extension', '-ef'], stdout=FNULL)
# except Exception:
# pass
exit(test_status)
def _run_process(args, msg, err_msg='ERROR!!!'):
write_info(msg)
err_code = run_process(args)
if err_code:
write_error(err_msg)
exit(err_code)
def configure_services_from_file(services, app_name):
from tethys_apps.models import CustomSetting
if services['version']:
del services['version']
for service_type in services:
if services[service_type] is not None:
current_services = services[service_type]
for setting_name in current_services:
if service_type == 'custom_setting':
try:
custom_setting = CustomSetting.objects.get(
name=setting_name)
except ObjectDoesNotExist:
write_warning(
f'Custom setting named "{setting_name}" could not be found in app "{app_name}". '
f'Skipping...')
continue
try:
custom_setting.value = current_services[setting_name]
custom_setting.clean()
custom_setting.save()
write_success(
f'CustomSetting: "{setting_name}" was assigned the value: '
f'"{current_services[setting_name]}"')
except ValidationError:
write_error(
"Incorrect value type given for custom setting '{}'. Please adjust "
"services.yml or set the value in the app's settings page."
.format(setting_name))
else:
app_settings = get_app_settings(app_name)
# In the case the app isn't installed, has no settings, or it is an extension,
# skip configuring services/settings
if not app_settings:
write_msg(
f'No settings found for app "{app_name}". Skipping automated configuration...'
)
return
unlinked_settings = app_settings['unlinked_settings']
setting_found = False
for setting in unlinked_settings:
if setting.name != setting_name:
continue
setting_found = True
service_id = current_services[setting_name]
if not service_id:
write_warning(
f'No service given for setting "{setting_name}". Skipping...'
)
continue
find_and_link(service_type, setting_name, service_id,
app_name, setting)
if not setting_found:
write_warning(
f'Service setting "{setting_name}" already configured or does not exist in app '
f'"{app_name}". Skipping...')
def run_interactive_services(app_name):
write_msg(
'Running Interactive Service Mode. '
'Any configuration options in services.yml or portal_config.yml will be ignored...'
)
write_msg('Hit return at any time to skip a step.')
app_settings = get_app_settings(app_name)
# In the case the app isn't installed, has no settings, or it is an extension,
# skip configuring services/settings
if not app_settings:
write_msg(
f'No settings found for app "{app_name}". Skipping interactive configuration...'
)
return
unlinked_settings = app_settings['unlinked_settings']
for setting in unlinked_settings:
valid = False
configure_text = "Configuring {}".format(setting.name)
star_out = '*' * len(configure_text)
write_msg(f"\n{star_out}\n{configure_text}\n{star_out}")
write_msg(f"Type: {setting.__class__.__name__}\n"
f"Description: {setting.description}\n"
f"Required: {setting.required}")
if hasattr(setting, 'value'):
while not valid:
write_msg(
'\nEnter the desired value for the current custom setting: {}'
.format(setting.name))
try:
value = get_interactive_input()
if value != "":
try:
setting.value = value
setting.clean()
setting.save()
valid = True
write_success(
"{} successfully set with value: {}.".format(
setting.name, value))
except ValidationError:
write_error(
"Incorrect value type given for custom setting '{}'. Please try again"
.format(setting.name))
else:
write_msg("Skipping setup of {}".format(setting.name))
valid = True
except (KeyboardInterrupt, SystemExit):
write_msg('\nInstall Command cancelled.')
exit(0)
else:
# List existing services
args = Namespace()
for conf in ['spatial', 'persistent', 'wps', 'dataset']:
setattr(args, conf, False)
setattr(args, get_setting_type(setting), True)
services = services_list_command(args)[0]
if len(services) <= 0:
write_warning(
'No compatible services found. See:\n\n tethys services create {} -h\n'
.format(get_setting_type(setting)))
continue
while not valid:
write_msg(
'\nEnter the service ID/Name to link to the current service setting: {}.'
.format(setting.name))
try:
service_id = get_interactive_input()
if service_id != "":
try:
setting_type = get_setting_type_from_setting(
setting)
service_type = get_service_type_from_setting(
setting)
except RuntimeError as e:
write_error(str(e) + ' Skipping...')
break
# Validate the given service id
valid_service = validate_service_id(
service_type, service_id)
if valid_service:
link_service_to_app_setting(
service_type, service_id, app_name,
setting_type, setting.name)
valid = True
else:
write_error(
'Incorrect service ID/Name. Please try again.')
else:
write_msg("Skipping setup of {}".format(setting.name))
valid = True
except (KeyboardInterrupt, SystemExit):
write_msg('\nInstall Command cancelled.')
exit(0)
