def install_packages(conda_config, update_installed=False): # Compile channels arguments install_args = [] if validate_schema('channels', conda_config): channels = conda_config['channels'] for channel in channels: install_args.extend(['-c', channel]) # Install all Packages if validate_schema('packages', conda_config): if not update_installed: install_args.extend(['--freeze-installed']) else: write_warning( 'Warning: Updating previously installed packages. This could break your Tethys environment.' ) install_args.extend(conda_config['packages']) write_msg("Running conda installation tasks...") [resp, err, code] = conda_run(Commands.INSTALL, *install_args, use_exception_handler=False, stdout=None, stderr=None) if code != 0: write_error( 'Warning: Packages installation ran into an error. Please try again or a manual install' )
def get_destination_path(args): # Determine destination file name (defaults to type) destination_file = FILE_NAMES[args.type] # Default destination path is the tethys_portal source dir destination_dir = TETHYS_HOME # Make the Tethys Home directory if it doesn't exist yet. if not os.path.isdir(destination_dir): os.makedirs(destination_dir, exist_ok=True) if args.type in [GEN_SERVICES_OPTION, GEN_INSTALL_OPTION]: destination_dir = os.getcwd() elif args.type == GEN_META_YAML_OPTION: destination_dir = os.path.join(TETHYS_SRC, 'conda.recipe') if args.directory: destination_dir = os.path.abspath(args.directory) if not os.path.isdir(destination_dir): write_error('ERROR: "{0}" is not a valid directory.'.format(destination_dir)) exit(1) destination_path = os.path.join(destination_dir, destination_file) check_for_existing_file(destination_path, destination_file, args.overwrite) return destination_path
def configure_services(services, app_name): from tethys_apps.models import CustomSetting if services['version']: del services['version'] for service_type in services: if services[service_type] is not None: current_services = services[service_type] for service_setting_name in current_services: if service_type == 'custom_setting': custom_setting = CustomSetting.objects.get( name=service_setting_name) try: custom_setting.value = current_services[ service_setting_name] custom_setting.clean() custom_setting.save() except ValidationError: write_error( "Incorrect value type given for custom setting '{}'. Please adjust " "services.yml or set the value in the app's settings page." .format(service_setting_name)) else: find_and_link(service_type, service_setting_name, current_services[service_setting_name], app_name)
def open_file(file_path): try: with file_path.open() as f: return yaml.safe_load(f) except Exception as e: write_error(str(e)) write_error( 'An unexpected error occurred reading the file. Please try again.') exit(1)
def find_and_link(service_type, setting_name, service_id, app_name, setting): valid_service = validate_service_id(service_type, service_id) setting_type = get_setting_type_from_setting(setting) if valid_service: link_service_to_app_setting(service_type, service_id, app_name, setting_type, setting_name) else: write_error( f'Warning: Could not find service of type: "{service_type}" with the Name/ID: "{service_id}"' )
def find_and_link(service_type, setting_name, service_name, app_name): service = get_service_from_name(service_name) if service: link_service_to_app_setting(service['service_type'], service_name, app_name, service['linkParam'], setting_name) else: write_error( 'Warning: Could not find service of type: {} with the name/id: {}'. format(service_type, service_name))
def get_docker_client(cls): """ Configure DockerClient """ if cls._docker_client is None: try: cls._docker_client = docker.from_env() except docker.errors.DockerException: write_error( 'The Docker daemon must be running to use the tethys docker command.' ) exit(1) return cls._docker_client
def purge_db_server(**kwargs): """Stop Tethys db server (if running) and then remove the db directory. Args: **kwargs: processed key word arguments from commandline """ kwargs['exit_on_error'] = False stop_db_server(**kwargs) if kwargs['db_dir']: write_error( "This action will permanently delete the database. DATA WILL BE LOST!" ) response = 'y' if kwargs['no_confirmation'] else input( f"Are you sure you want to continue? [y/N]: ") if response.lower() in ['y', 'yes']: shutil.rmtree(kwargs['db_dir'])
def _get_dict_key_handle(d, key, not_exists_okay=False): keys = key.split('.') values = [d] for k in keys: try: values.append(values[-1][k]) except Exception: if not_exists_okay: if len(keys) > len(values): values[-1][k] = {} values.append(values[-1][k]) else: return values[-1], k else: write_error(f'The setting {key} does not exists.') return return values[-2], k
def _run_process(args, msg, err_msg='ERROR!!!', exit_on_error=True, **kwargs): """Run a process while outputting messages. If error then either exit or return error code. Args: args: args for process msg: Message to output before running process err_msg: Message to output if there is an error exit_on_error: If True then exit if process returns and error code **kwargs: processed key word arguments from commandline Returns: error code """ write_info(msg) err_code = run_process(args) if err_code: write_error(err_msg) if exit_on_error: exit(err_code) return err_code
def install_packages(conda_config): # Compile channels arguments install_args = [] if validate_schema('channels', conda_config): channels = conda_config['channels'] for channel in channels: install_args.extend(['-c', channel]) # Install all Packages if validate_schema('packages', conda_config): install_args.extend(conda_config['packages']) write_msg("Running conda installation tasks...") [resp, err, code] = conda_run(Commands.INSTALL, *install_args, use_exception_handler=False, stdout=None, stderr=None) if code != 0: write_error( 'Warning: Packages installation ran into an error. Please try again or a manual install' )
def create_portal_superuser(portal_superuser_name='admin', portal_superuser_email='', portal_superuser_password='******', **kwargs): """Create a superuser account for Tethys Portal Args: portal_superuser_name: username for the Tethys Portal superuser account portal_superuser_email: email for the Tethys Portal superuser account portal_superuser_password: password for the Tethys Portal superuser **kwargs: processed key word arguments from commandline """ write_info( f'Creating Tethys Portal superuser "{portal_superuser_name}"...') load_apps() from django.contrib.auth.models import User # noqa: E402 try: User.objects.create_superuser(portal_superuser_name, portal_superuser_email, portal_superuser_password) except IntegrityError: write_error( f'Tethys Portal Superuser "{portal_superuser_name}" already exists.' )
def get_app_settings(app): """ Get settings related to app Args: app(str): name of app Returns: dict (linked_settings, unlinked_settings): Dictionary with two keys: linked_settings(list) - list of linked settings, unlinked_settings(list) - list of unlinked settings # noqa: E501 """ from tethys_cli.cli_colors import write_error from tethys_apps.models import (TethysApp, PersistentStoreConnectionSetting, PersistentStoreDatabaseSetting, SpatialDatasetServiceSetting, DatasetServiceSetting, WebProcessingServiceSetting, CustomSetting) try: app = TethysApp.objects.get(package=app) app_settings = [] for setting in PersistentStoreConnectionSetting.objects.filter(tethys_app=app): app_settings.append(setting) for setting in PersistentStoreDatabaseSetting.objects.filter(tethys_app=app): app_settings.append(setting) for setting in SpatialDatasetServiceSetting.objects.filter(tethys_app=app): app_settings.append(setting) for setting in DatasetServiceSetting.objects.filter(tethys_app=app): app_settings.append(setting) for setting in WebProcessingServiceSetting.objects.filter(tethys_app=app): app_settings.append(setting) for setting in CustomSetting.objects.filter(tethys_app=app): app_settings.append(setting) unlinked_settings = [] linked_settings = [] for setting in app_settings: if (hasattr(setting, 'spatial_dataset_service') and setting.spatial_dataset_service) \ or (hasattr(setting, 'persistent_store_service') and setting.persistent_store_service) \ or (hasattr(setting, 'dataset_service') and setting.dataset_service) \ or (hasattr(setting, 'web_processing_service') and setting.web_processing_service) \ or (hasattr(setting, 'value') and setting.value != ''): linked_settings.append(setting) else: unlinked_settings.append(setting) return { 'linked_settings': linked_settings, 'unlinked_settings': unlinked_settings } except ObjectDoesNotExist: write_error('The app you specified ("{0}") does not exist. Command aborted.'.format(app)) except Exception as e: write_error(str(e)) write_error('Something went wrong. Please try again.')
def test_command(args): args.manage = False # Get the path to manage.py manage_path = get_manage_path(args) tests_path = os.path.join(TETHYS_SRC_DIRECTORY, 'tests') try: check_and_install_prereqs(tests_path) except FileNotFoundError: write_error( 'The "tethys test" command will not work because the tests are not installed. ' 'To run tests you must download the tethys source code.') exit(1) # Define the process to be run primary_process = ['python', manage_path, 'test'] # Tag to later check if tests are being run on a specific app or extension app_package_tag = 'tethys_apps.tethysapp.' extension_package_tag = 'tethysext.' if args.coverage or args.coverage_html: os.environ['TETHYS_TEST_DIR'] = tests_path if args.file and app_package_tag in args.file: app_package_parts = args.file.split(app_package_tag) app_name = app_package_parts[1].split('.')[0] core_app_package = '{}{}'.format(app_package_tag, app_name) app_package = 'tethysapp.{}'.format(app_name) config_opt = '--source={},{}'.format(core_app_package, app_package) elif args.file and extension_package_tag in args.file: extension_package_parts = args.file.split(extension_package_tag) extension_name = extension_package_parts[1].split('.')[0] core_extension_package = '{}{}'.format(extension_package_tag, extension_name) extension_package = 'tethysext.{}'.format(extension_name) config_opt = '--source={},{}'.format(core_extension_package, extension_package) else: config_opt = '--rcfile={0}'.format( os.path.join(tests_path, 'coverage.cfg')) primary_process = ['coverage', 'run', config_opt, manage_path, 'test'] if args.file: primary_process.append(args.file) elif args.unit: primary_process.append(os.path.join(tests_path, 'unit_tests')) elif args.gui: primary_process.append(os.path.join(tests_path, 'gui_tests')) if args.verbosity: primary_process.extend(['-v', args.verbosity]) test_status = run_process(primary_process) if args.coverage: if args.file and (app_package_tag in args.file or extension_package_tag in args.file): run_process(['coverage', 'report']) else: run_process(['coverage', 'report', config_opt]) if args.coverage_html: report_dirname = 'coverage_html_report' index_fname = 'index.html' if args.file and (app_package_tag in args.file or extension_package_tag in args.file): run_process([ 'coverage', 'html', '--directory={0}'.format( os.path.join(tests_path, report_dirname)) ]) else: run_process(['coverage', 'html', config_opt]) try: status = run_process([ 'open', os.path.join(tests_path, report_dirname, index_fname) ]) if status != 0: raise Exception except Exception: webbrowser.open_new_tab( os.path.join(tests_path, report_dirname, index_fname)) # Removing Test App # try: # subprocess.call(['tethys', 'uninstall', 'test_app', '-f'], stdout=FNULL) # except Exception: # pass # try: # subprocess.call(['tethys', 'uninstall', 'test_extension', '-ef'], stdout=FNULL) # except Exception: # pass exit(test_status)
def _run_process(args, msg, err_msg='ERROR!!!'): write_info(msg) err_code = run_process(args) if err_code: write_error(err_msg) exit(err_code)
def configure_services_from_file(services, app_name): from tethys_apps.models import CustomSetting if services['version']: del services['version'] for service_type in services: if services[service_type] is not None: current_services = services[service_type] for setting_name in current_services: if service_type == 'custom_setting': try: custom_setting = CustomSetting.objects.get( name=setting_name) except ObjectDoesNotExist: write_warning( f'Custom setting named "{setting_name}" could not be found in app "{app_name}". ' f'Skipping...') continue try: custom_setting.value = current_services[setting_name] custom_setting.clean() custom_setting.save() write_success( f'CustomSetting: "{setting_name}" was assigned the value: ' f'"{current_services[setting_name]}"') except ValidationError: write_error( "Incorrect value type given for custom setting '{}'. Please adjust " "services.yml or set the value in the app's settings page." .format(setting_name)) else: app_settings = get_app_settings(app_name) # In the case the app isn't installed, has no settings, or it is an extension, # skip configuring services/settings if not app_settings: write_msg( f'No settings found for app "{app_name}". Skipping automated configuration...' ) return unlinked_settings = app_settings['unlinked_settings'] setting_found = False for setting in unlinked_settings: if setting.name != setting_name: continue setting_found = True service_id = current_services[setting_name] if not service_id: write_warning( f'No service given for setting "{setting_name}". Skipping...' ) continue find_and_link(service_type, setting_name, service_id, app_name, setting) if not setting_found: write_warning( f'Service setting "{setting_name}" already configured or does not exist in app ' f'"{app_name}". Skipping...')
def run_interactive_services(app_name): write_msg( 'Running Interactive Service Mode. ' 'Any configuration options in services.yml or portal_config.yml will be ignored...' ) write_msg('Hit return at any time to skip a step.') app_settings = get_app_settings(app_name) # In the case the app isn't installed, has no settings, or it is an extension, # skip configuring services/settings if not app_settings: write_msg( f'No settings found for app "{app_name}". Skipping interactive configuration...' ) return unlinked_settings = app_settings['unlinked_settings'] for setting in unlinked_settings: valid = False configure_text = "Configuring {}".format(setting.name) star_out = '*' * len(configure_text) write_msg(f"\n{star_out}\n{configure_text}\n{star_out}") write_msg(f"Type: {setting.__class__.__name__}\n" f"Description: {setting.description}\n" f"Required: {setting.required}") if hasattr(setting, 'value'): while not valid: write_msg( '\nEnter the desired value for the current custom setting: {}' .format(setting.name)) try: value = get_interactive_input() if value != "": try: setting.value = value setting.clean() setting.save() valid = True write_success( "{} successfully set with value: {}.".format( setting.name, value)) except ValidationError: write_error( "Incorrect value type given for custom setting '{}'. Please try again" .format(setting.name)) else: write_msg("Skipping setup of {}".format(setting.name)) valid = True except (KeyboardInterrupt, SystemExit): write_msg('\nInstall Command cancelled.') exit(0) else: # List existing services args = Namespace() for conf in ['spatial', 'persistent', 'wps', 'dataset']: setattr(args, conf, False) setattr(args, get_setting_type(setting), True) services = services_list_command(args)[0] if len(services) <= 0: write_warning( 'No compatible services found. See:\n\n tethys services create {} -h\n' .format(get_setting_type(setting))) continue while not valid: write_msg( '\nEnter the service ID/Name to link to the current service setting: {}.' .format(setting.name)) try: service_id = get_interactive_input() if service_id != "": try: setting_type = get_setting_type_from_setting( setting) service_type = get_service_type_from_setting( setting) except RuntimeError as e: write_error(str(e) + ' Skipping...') break # Validate the given service id valid_service = validate_service_id( service_type, service_id) if valid_service: link_service_to_app_setting( service_type, service_id, app_name, setting_type, setting.name) valid = True else: write_error( 'Incorrect service ID/Name. Please try again.') else: write_msg("Skipping setup of {}".format(setting.name)) valid = True except (KeyboardInterrupt, SystemExit): write_msg('\nInstall Command cancelled.') exit(0)