def test_merge(self): # Config files are loaded and merged unlink(self.temp) conf = ChainConfig([ ('a', PathConfig(self.a)), ('b', PathConfig(self.b))]) eq_(+conf, PathConfig(self.final))
def test_default(self): # Missing, empty or malformed config files return an empty AttrDict conf = ChainConfig([ ('missing', PathConfig(self.missing)), ('error', PathConfig(self.error)), ('empty', PathConfig(self.empty)), ('string', PathConfig(self.string)), ]) eq_(+conf, AttrDict())
def mail(args, kwargs): # Get config file location default_dir = os.path.join(variables['GRAMEXDATA'], 'mail') _mkdir(default_dir) if 'conf' in kwargs: confpath = kwargs.conf elif os.path.exists('gramex.yaml'): confpath = os.path.abspath('gramex.yaml') else: confpath = os.path.join(default_dir, 'gramexmail.yaml') if not os.path.exists(confpath): if 'init' in kwargs: with io.open(confpath, 'w', encoding='utf-8') as handle: handle.write(default_mail_config.format(confpath=confpath)) app_log.info('Initialized %s', confpath) elif not args and not kwargs: app_log.error(show_usage('mail')) else: app_log.error('Missing config %s. Use --init to generate skeleton', confpath) return conf = PathConfig(confpath) if 'list' in kwargs: for key, alert in conf.get('alert', {}).items(): to = alert.get('to', '') if isinstance(to, list): to = ', '.join(to) gramex.console('{:15}\t"{}" to {}'.format(key, alert.get('subject'), to)) return if 'init' in kwargs: app_log.error('Config already exists at %s', confpath) return if len(args) < 1: app_log.error(show_usage('mail')) return from gramex.services import email as setup_email, create_alert alert_conf = conf.get('alert', {}) email_conf = conf.get('email', {}) setup_email(email_conf) sys.path += os.path.dirname(confpath) for key in args: if key not in alert_conf: app_log.error('Missing key %s in %s', key, confpath) continue alert = create_alert(key, alert_conf[key]) alert()
def callback_commandline(commands): ''' Find what method should be run based on the command line programs. This refactoring allows us to test gramex.commandline() to see if it processes the command line correctly, without actually running the commands. Returns a callback method and kwargs for the callback method. ''' # Set logging config at startup. (Services may override this.) log_config = (+PathConfig(paths['source'] / 'gramex.yaml')).get( 'log', AttrDict()) log_config.root.level = logging.INFO from . import services services.log(log_config) # kwargs has all optional command line args as a dict of values / lists. # args has all positional arguments as a list. kwargs = parse_command_line(commands) args = kwargs.pop('_') # If --help or -V --version is specified, print a message and end if kwargs.get('V') is True or kwargs.get('version') is True: return console, {'msg': 'Gramex %s' % __version__} # Any positional argument is treated as a gramex command if len(args) > 0: base_command = args.pop(0).lower() method = 'install' if base_command == 'update' else base_command if method in { 'install', 'uninstall', 'setup', 'run', 'service', 'init', 'mail', 'license', }: import gramex.install if 'help' in kwargs: return console, {'msg': gramex.install.show_usage(method)} return getattr(gramex.install, method), { 'args': args, 'kwargs': kwargs } raise NotImplementedError('Unknown gramex command: %s' % base_command) elif kwargs.get('help') is True: return console, {'msg': __doc__.strip().format(**globals())} # Use current dir as base (where gramex is run from) if there's a gramex.yaml. if not os.path.isfile('gramex.yaml'): return console, { 'msg': 'No gramex.yaml. See https://learn.gramener.com/guide/' } # Run gramex.init(cmd={command line arguments like YAML variables}) app_log.info('Gramex %s | %s | Python %s', __version__, os.getcwd(), sys.version.replace('\n', ' ')) return init, {'cmd': AttrDict(app=kwargs)}
def test_chain_update(self): # Chained config files are changed on update # Set up a configuration with 2 files -- conf1.test and conf2.test. with self.conf1.open(mode='w', encoding='utf-8') as handle: yaml.dump({'url': {}}, handle) with self.conf2.open(mode='w', encoding='utf-8') as handle: yaml.dump({'url': {'a': 1}}, handle) conf = ChainConfig() conf.conf1 = PathConfig(self.conf1) conf.conf2 = PathConfig(self.conf2) eq_(+conf, {'url': {'a': 1}}) # Change conf2.test and ensure that its original contents are replaced, # not just merged with previous value with self.conf2.open(mode='w', encoding='utf-8') as handle: yaml.dump({'url': {'b': 10}}, handle) eq_(+conf, {'url': {'b': 10}})
def test_import(self): # Check if config files are imported conf_imp = ChainConfig(conf=PathConfig(self.imp)) conf_b = ChainConfig(conf=PathConfig(self.b)) # When temp is missing, config matches b unlink(self.temp) eq_(+conf_imp, +conf_b) # Once temp file is created, it is automatically imported data = AttrDict(a=1, b=2) with self.temp.open('w') as out: yaml.dump(dict(data), out) result = +conf_b result.update(data) eq_(+conf_imp, result) # Once removed, it no longer used unlink(self.temp) eq_(+conf_imp, +conf_b)
def check_files(self, appname, expected_files): '''app/ directory should have expected files''' folder = self.appdir(appname) actual = set() for root, dirs, files in os.walk(folder): for filename in files: if '.git' not in root: actual.add(os.path.join(root, filename)) expected = {os.path.abspath(os.path.join(folder, filename)) for filename in expected_files} self.assertEqual(actual, expected) conf = +PathConfig(Path(self.appdir('apps.yaml'))) self.assertTrue(appname in conf) self.assertTrue('target' in conf[appname]) self.assertTrue('cmd' in conf[appname] or 'url' in conf[appname]) self.assertTrue('installed' in conf[appname]) self.assertTrue('time' in conf[appname].installed)
def test_update(self): # Config files are updated on change conf = ChainConfig(temp=PathConfig(self.temp)) # When the file is missing, config is empty unlink(self.temp) eq_(+conf, {}) # When the file is blank, config is empty with self.temp.open('w') as out: out.write('') eq_(+conf, {}) # Once created, it is automatically reloaded data = AttrDict(a=1, b=2) with self.temp.open('w') as out: yaml.dump(dict(data), out) eq_(+conf, data) # Deleted file is detected self.temp.unlink() eq_(+conf, {})
if exe_path is not None: cmd = cmd.format(FILE=setup_file, EXE=exe_path) app_log.info('Running %s', cmd) _run_console(cmd, cwd=target) break else: app_log.warning('Skipping %s. No %s found', setup_file, exe) app_dir = Path(variables.get('GRAMEXDATA')) / 'apps' if not app_dir.exists(): app_dir.mkdir(parents=True) # Get app configuration by chaining apps.yaml in gramex + app_dir + command line apps_config = ChainConfig() apps_config['base'] = PathConfig(gramex.paths['source'] / 'apps.yaml') user_conf_file = app_dir / 'apps.yaml' apps_config['user'] = PathConfig( user_conf_file) if user_conf_file.exists() else AttrDict() app_keys = { 'url': 'URL / filename of a ZIP file to install', 'cmd': 'Command used to install file', 'dir': 'Sub-directory under "url" to run from (optional)', 'contentdir': 'Strip root directory with a single child (optional, default=True)', 'target': 'Local directory where the app is installed', 'installed': 'Additional installation information about the app', 'run': 'Runtime keyword arguments for the app', }
def init(force_reload=False, **kwargs): ''' Update Gramex configurations and start / restart the instance. ``gramex.init()`` can be called any time to refresh configuration files. ``gramex.init(key=val)`` adds ``val`` as a configuration layer named ``key``. If ``val`` is a Path, it is converted into a PathConfig. (If it is Path directory, use ``gramex.yaml``.) Services are re-initialised if their configurations have changed. Service callbacks are always re-run (even if the configuration hasn't changed.) ''' try: setup_secrets(paths['base'] / '.secrets.yaml') except Exception as e: app_log.exception(e) # Reset variables variables.clear() variables.update(setup_variables()) # Initialise configuration layers with provided configurations # AttrDicts are updated as-is. Paths are converted to PathConfig paths.update(kwargs) for key, val in paths.items(): if isinstance(val, Path): if val.is_dir(): val = val / 'gramex.yaml' val = PathConfig(val) config_layers[key] = val # Locate all config files config_files = set() for path_config in config_layers.values(): if hasattr(path_config, '__info__'): for pathinfo in path_config.__info__.imports: config_files.add(pathinfo.path) config_files = list(config_files) # Add config file folders to sys.path sys.path[:] = _sys_path + [ str(path.absolute().parent) for path in config_files ] from . import services globals( )['service'] = services.info # gramex.service = gramex.services.info # Override final configurations appconfig.clear() appconfig.update(+config_layers) # --settings.debug => log.root.level = True if appconfig.app.get('settings', {}).get('debug', False): appconfig.log.root.level = logging.DEBUG # Set up a watch on config files (including imported files) if appconfig.app.get('watch', True): from services import watcher watcher.watch('gramex-reconfig', paths=config_files, on_modified=lambda event: init()) # Run all valid services. (The "+" before config_chain merges the chain) # Services may return callbacks to be run at the end for key, val in appconfig.items(): if key not in conf or conf[key] != val or force_reload: if hasattr(services, key): app_log.debug('Loading service: %s', key) conf[key] = prune_keys(val, {'comment'}) callback = getattr(services, key)(conf[key]) if callable(callback): callbacks[key] = callback else: app_log.error('No service named %s', key) # Run the callbacks. Specifically, the app service starts the Tornado ioloop for key in (+config_layers).keys(): if key in callbacks: app_log.debug('Running callback: %s', key) callbacks[key]()
def test_random(self): # * in keys is replaced with a random 5-char alphanumeric conf = PathConfig(self.random) for key, val in conf.random.items(): regex = re.compile(val.replace('*', '[A-Za-z0-9]{5}')) ok_(regex.match(key))
def test_import_merge(self): conf = PathConfig(self.importmerge) for key, val in conf.items(): eq_(val['expected'], val['actual'])
def test_if(self): conf = PathConfig(self.condition) for key, val in conf.items(): eq_(val['expected'], val['actual'])
def test_imports(self): conf = PathConfig(self.imports) ok_(conf) for key, result in conf.items(): eq_(result.source, result.target, key + ': %r != %r' % (result.source, result.target))
import pytest import re import requests import time import gramex.cache from fnmatch import fnmatch from lxml.html import document_fromstring from selenium import webdriver from selenium.common.exceptions import NoSuchElementException from six import string_types from tornado.web import create_signed_value from gramex.config import ChainConfig, PathConfig, objectpath, variables # Get Gramex conf from current directory gramex_conf = ChainConfig() gramex_conf['source'] = PathConfig(os.path.join(variables['GRAMEXPATH'], 'gramex.yaml')) gramex_conf['base'] = PathConfig('gramex.yaml') secret = objectpath(+gramex_conf, 'app.settings.cookie_secret') drivers = {} default = object() class ChromeConf(dict): def __init__(self, **conf): self['goog:chromeOptions'] = {'args': ['--no-sandbox']} for key, val in conf.items(): getattr(self, key)(val) def headless(self, val): if val: self['goog:chromeOptions']['args'].append('--headless')
def test_variables(self): # Templates interpolate string variables # Create configuration with 2 layers and a subdirectory import conf = +ChainConfig( base=PathConfig(self.chain.base), child=PathConfig(self.chain.child), ) # Custom variables are deleted after use ok_('variables' not in conf) for key in ['base', 'child', 'subdir']: # {.} maps to YAML file's directory eq_(conf['%s_DOT' % key], str(self.chain[key].parent)) # $YAMLPATH maps to YAML file's directory eq_(conf['%s_YAMLPATH' % key], str(self.chain[key].parent)) # $YAMLURL is the relative path to YAML file's directory eq_(conf['%s_YAMLURL' % key], conf['%s_YAMLURL_EXPECTED' % key]) # Environment variables are present by default eq_(conf['%s_HOME' % key], os.environ.get('HOME', '')) # Non-existent variables map to '' eq_(conf['%s_NONEXISTENT' % key], os.environ.get('NONEXISTENT', '')) # Custom variables are applied eq_(conf['%s_THIS' % key], key) # Custom variables are inherited. Defaults do not override eq_(conf['%s_ROOT' % key], conf.base_ROOT) # Default variables are set eq_(conf['%s_DEFAULT' % key], key) # Functions run and override values eq_(conf['%s_FUNCTION' % key], key) # Default functions "underride" values eq_(conf['%s_DEFAULT_FUNCTION' % key], 'base') # Functions can use variables using gramex.config.variables eq_(conf['%s_FUNCTION_VAR' % key], conf.base_ROOT + key) # Derived variables eq_(conf['%s_DERIVED' % key], '%s/derived' % key) # $URLROOT is the frozen to base $YAMLURL eq_(conf['%s_YAMLURL_VAR' % key], conf['%s_YAMLURL_VAR_EXPECTED' % key]) # $GRAMEXPATH is the gramex path gramex_path = os.path.dirname(inspect.getfile(gramex)) eq_(conf['%s_GRAMEXPATH' % key], gramex_path) # $GRAMEXAPPS is the gramex apps path eq_(conf['%s_GRAMEXAPPS' % key], os.path.join(gramex_path, 'apps')) # $GRAMEXHOST is the socket.gethostname eq_(conf['%s_GRAMEXHOST' % key], socket.gethostname()) # Imports do not override, but do setdefault eq_(conf['path'], str(self.chain['base'].parent)) eq_(conf['subpath'], str(self.chain['subdir'].parent)) # Check if variable types are preserved eq_(conf['numeric'], 1) eq_(conf['boolean'], True) eq_(conf['object'], {'x': 1}) eq_(conf['list'], [1, 2]) # Check if variables of different types are string substituted eq_(conf['numeric_subst'], '/1') eq_(conf['boolean_subst'], '/True') # Actually, conf['object_subst'] is "/AttrDict([('x', 1)])". Let's not test that. # eq_(conf['object_subst'], "/{'x': 1}") eq_(conf['list_subst'], '/[1, 2]') # Check condition variables for key, val in conf['conditions'].items(): eq_('is-' + key, val)
def callback_commandline(commands): ''' Find what method should be run based on the command line programs. This refactoring allows us to test gramex.commandline() to see if it processes the command line correctly, without actually running the commands. Returns a callback method and kwargs for the callback method. ''' # Ensure that log colours are printed properly on cygwin. if sys.platform == 'cygwin': # colorama.init() gets it wrong on Cygwin import colorlog.escape_codes # noqa: Let colorlog call colorama.init() first import colorama colorama.init(convert=True) # Now we'll override with convert=True # Set logging config at startup. (Services may override this.) log_config = (+PathConfig(paths['source'] / 'gramex.yaml')).get( 'log', AttrDict()) log_config.root.level = logging.INFO from . import services services.log(log_config) # args has all optional command line args as a dict of values / lists. # cmd has all positional arguments as a list. args = parse_command_line(commands) cmd = args.pop('_') # If --help or -V --version is specified, print a message and end if args.get('V') is True or args.get('version') is True: return console, {'msg': 'Gramex %s' % __version__} if args.get('help') is True: return console, {'msg': __doc__.strip().format(**globals())} # Any positional argument is treated as a gramex command if len(cmd) > 0: kwargs = {'cmd': cmd, 'args': args} base_command = cmd.pop(0).lower() method = 'install' if base_command == 'update' else base_command if method in { 'install', 'uninstall', 'setup', 'run', 'service', 'init', 'mail', 'license', }: import gramex.install return getattr(gramex.install, method), kwargs raise NotImplementedError('Unknown gramex command: %s' % base_command) # Use current dir as base (where gramex is run from) if there's a gramex.yaml. # Else use source/guide, and point the user to the welcome screen if not os.path.isfile('gramex.yaml'): from gramex.install import run args.setdefault('browser', '/welcome') return run, {'cmd': ['guide'], 'args': args} app_log.info('Gramex %s | %s | Python %s', __version__, os.getcwd(), sys.version.replace('\n', ' ')) return init, {'cmd': AttrDict(app=args)}