def test_merge(self):
# Config files are loaded and merged
unlink(self.temp)
conf = ChainConfig([
('a', PathConfig(self.a)),
('b', PathConfig(self.b))])
eq_(+conf, PathConfig(self.final))
def test_default(self):
# Missing, empty or malformed config files return an empty AttrDict
conf = ChainConfig([
('missing', PathConfig(self.missing)),
('error', PathConfig(self.error)),
('empty', PathConfig(self.empty)),
('string', PathConfig(self.string)),
])
eq_(+conf, AttrDict())
def mail(args, kwargs):
# Get config file location
default_dir = os.path.join(variables['GRAMEXDATA'], 'mail')
_mkdir(default_dir)
if 'conf' in kwargs:
confpath = kwargs.conf
elif os.path.exists('gramex.yaml'):
confpath = os.path.abspath('gramex.yaml')
else:
confpath = os.path.join(default_dir, 'gramexmail.yaml')
if not os.path.exists(confpath):
if 'init' in kwargs:
with io.open(confpath, 'w', encoding='utf-8') as handle:
handle.write(default_mail_config.format(confpath=confpath))
app_log.info('Initialized %s', confpath)
elif not args and not kwargs:
app_log.error(show_usage('mail'))
else:
app_log.error('Missing config %s. Use --init to generate skeleton',
confpath)
return
conf = PathConfig(confpath)
if 'list' in kwargs:
for key, alert in conf.get('alert', {}).items():
to = alert.get('to', '')
if isinstance(to, list):
to = ', '.join(to)
gramex.console('{:15}\t"{}" to {}'.format(key,
alert.get('subject'),
to))
return
if 'init' in kwargs:
app_log.error('Config already exists at %s', confpath)
return
if len(args) < 1:
app_log.error(show_usage('mail'))
return
from gramex.services import email as setup_email, create_alert
alert_conf = conf.get('alert', {})
email_conf = conf.get('email', {})
setup_email(email_conf)
sys.path += os.path.dirname(confpath)
for key in args:
if key not in alert_conf:
app_log.error('Missing key %s in %s', key, confpath)
continue
alert = create_alert(key, alert_conf[key])
alert()
def callback_commandline(commands):
'''
Find what method should be run based on the command line programs. This
refactoring allows us to test gramex.commandline() to see if it processes
the command line correctly, without actually running the commands.
Returns a callback method and kwargs for the callback method.
'''
# Set logging config at startup. (Services may override this.)
log_config = (+PathConfig(paths['source'] / 'gramex.yaml')).get(
'log', AttrDict())
log_config.root.level = logging.INFO
from . import services
services.log(log_config)
# kwargs has all optional command line args as a dict of values / lists.
# args has all positional arguments as a list.
kwargs = parse_command_line(commands)
args = kwargs.pop('_')
# If --help or -V --version is specified, print a message and end
if kwargs.get('V') is True or kwargs.get('version') is True:
return console, {'msg': 'Gramex %s' % __version__}
# Any positional argument is treated as a gramex command
if len(args) > 0:
base_command = args.pop(0).lower()
method = 'install' if base_command == 'update' else base_command
if method in {
'install',
'uninstall',
'setup',
'run',
'service',
'init',
'mail',
'license',
}:
import gramex.install
if 'help' in kwargs:
return console, {'msg': gramex.install.show_usage(method)}
return getattr(gramex.install, method), {
'args': args,
'kwargs': kwargs
}
raise NotImplementedError('Unknown gramex command: %s' % base_command)
elif kwargs.get('help') is True:
return console, {'msg': __doc__.strip().format(**globals())}
# Use current dir as base (where gramex is run from) if there's a gramex.yaml.
if not os.path.isfile('gramex.yaml'):
return console, {
'msg': 'No gramex.yaml. See https://learn.gramener.com/guide/'
}
# Run gramex.init(cmd={command line arguments like YAML variables})
app_log.info('Gramex %s | %s | Python %s', __version__, os.getcwd(),
sys.version.replace('\n', ' '))
return init, {'cmd': AttrDict(app=kwargs)}
def test_chain_update(self):
# Chained config files are changed on update
# Set up a configuration with 2 files -- conf1.test and conf2.test.
with self.conf1.open(mode='w', encoding='utf-8') as handle:
yaml.dump({'url': {}}, handle)
with self.conf2.open(mode='w', encoding='utf-8') as handle:
yaml.dump({'url': {'a': 1}}, handle)
conf = ChainConfig()
conf.conf1 = PathConfig(self.conf1)
conf.conf2 = PathConfig(self.conf2)
eq_(+conf, {'url': {'a': 1}})
# Change conf2.test and ensure that its original contents are replaced,
# not just merged with previous value
with self.conf2.open(mode='w', encoding='utf-8') as handle:
yaml.dump({'url': {'b': 10}}, handle)
eq_(+conf, {'url': {'b': 10}})
def test_import(self):
# Check if config files are imported
conf_imp = ChainConfig(conf=PathConfig(self.imp))
conf_b = ChainConfig(conf=PathConfig(self.b))
# When temp is missing, config matches b
unlink(self.temp)
eq_(+conf_imp, +conf_b)
# Once temp file is created, it is automatically imported
data = AttrDict(a=1, b=2)
with self.temp.open('w') as out:
yaml.dump(dict(data), out)
result = +conf_b
result.update(data)
eq_(+conf_imp, result)
# Once removed, it no longer used
unlink(self.temp)
eq_(+conf_imp, +conf_b)
def check_files(self, appname, expected_files):
'''app/ directory should have expected files'''
folder = self.appdir(appname)
actual = set()
for root, dirs, files in os.walk(folder):
for filename in files:
if '.git' not in root:
actual.add(os.path.join(root, filename))
expected = {os.path.abspath(os.path.join(folder, filename))
for filename in expected_files}
self.assertEqual(actual, expected)
conf = +PathConfig(Path(self.appdir('apps.yaml')))
self.assertTrue(appname in conf)
self.assertTrue('target' in conf[appname])
self.assertTrue('cmd' in conf[appname] or 'url' in conf[appname])
self.assertTrue('installed' in conf[appname])
self.assertTrue('time' in conf[appname].installed)
def test_update(self):
# Config files are updated on change
conf = ChainConfig(temp=PathConfig(self.temp))
# When the file is missing, config is empty
unlink(self.temp)
eq_(+conf, {})
# When the file is blank, config is empty
with self.temp.open('w') as out:
out.write('')
eq_(+conf, {})
# Once created, it is automatically reloaded
data = AttrDict(a=1, b=2)
with self.temp.open('w') as out:
yaml.dump(dict(data), out)
eq_(+conf, data)
# Deleted file is detected
self.temp.unlink()
eq_(+conf, {})
if exe_path is not None:
cmd = cmd.format(FILE=setup_file, EXE=exe_path)
app_log.info('Running %s', cmd)
_run_console(cmd, cwd=target)
break
else:
app_log.warning('Skipping %s. No %s found', setup_file, exe)
app_dir = Path(variables.get('GRAMEXDATA')) / 'apps'
if not app_dir.exists():
app_dir.mkdir(parents=True)
# Get app configuration by chaining apps.yaml in gramex + app_dir + command line
apps_config = ChainConfig()
apps_config['base'] = PathConfig(gramex.paths['source'] / 'apps.yaml')
user_conf_file = app_dir / 'apps.yaml'
apps_config['user'] = PathConfig(
user_conf_file) if user_conf_file.exists() else AttrDict()
app_keys = {
'url': 'URL / filename of a ZIP file to install',
'cmd': 'Command used to install file',
'dir': 'Sub-directory under "url" to run from (optional)',
'contentdir':
'Strip root directory with a single child (optional, default=True)',
'target': 'Local directory where the app is installed',
'installed': 'Additional installation information about the app',
'run': 'Runtime keyword arguments for the app',
}
def init(force_reload=False, **kwargs):
'''
Update Gramex configurations and start / restart the instance.
``gramex.init()`` can be called any time to refresh configuration files.
``gramex.init(key=val)`` adds ``val`` as a configuration layer named
``key``. If ``val`` is a Path, it is converted into a PathConfig. (If it is
Path directory, use ``gramex.yaml``.)
Services are re-initialised if their configurations have changed. Service
callbacks are always re-run (even if the configuration hasn't changed.)
'''
try:
setup_secrets(paths['base'] / '.secrets.yaml')
except Exception as e:
app_log.exception(e)
# Reset variables
variables.clear()
variables.update(setup_variables())
# Initialise configuration layers with provided configurations
# AttrDicts are updated as-is. Paths are converted to PathConfig
paths.update(kwargs)
for key, val in paths.items():
if isinstance(val, Path):
if val.is_dir():
val = val / 'gramex.yaml'
val = PathConfig(val)
config_layers[key] = val
# Locate all config files
config_files = set()
for path_config in config_layers.values():
if hasattr(path_config, '__info__'):
for pathinfo in path_config.__info__.imports:
config_files.add(pathinfo.path)
config_files = list(config_files)
# Add config file folders to sys.path
sys.path[:] = _sys_path + [
str(path.absolute().parent) for path in config_files
]
from . import services
globals(
)['service'] = services.info # gramex.service = gramex.services.info
# Override final configurations
appconfig.clear()
appconfig.update(+config_layers)
# --settings.debug => log.root.level = True
if appconfig.app.get('settings', {}).get('debug', False):
appconfig.log.root.level = logging.DEBUG
# Set up a watch on config files (including imported files)
if appconfig.app.get('watch', True):
from services import watcher
watcher.watch('gramex-reconfig',
paths=config_files,
on_modified=lambda event: init())
# Run all valid services. (The "+" before config_chain merges the chain)
# Services may return callbacks to be run at the end
for key, val in appconfig.items():
if key not in conf or conf[key] != val or force_reload:
if hasattr(services, key):
app_log.debug('Loading service: %s', key)
conf[key] = prune_keys(val, {'comment'})
callback = getattr(services, key)(conf[key])
if callable(callback):
callbacks[key] = callback
else:
app_log.error('No service named %s', key)
# Run the callbacks. Specifically, the app service starts the Tornado ioloop
for key in (+config_layers).keys():
if key in callbacks:
app_log.debug('Running callback: %s', key)
callbacks[key]()
def test_random(self):
# * in keys is replaced with a random 5-char alphanumeric
conf = PathConfig(self.random)
for key, val in conf.random.items():
regex = re.compile(val.replace('*', '[A-Za-z0-9]{5}'))
ok_(regex.match(key))
def test_import_merge(self):
conf = PathConfig(self.importmerge)
for key, val in conf.items():
eq_(val['expected'], val['actual'])
def test_if(self):
conf = PathConfig(self.condition)
for key, val in conf.items():
eq_(val['expected'], val['actual'])
def test_imports(self):
conf = PathConfig(self.imports)
ok_(conf)
for key, result in conf.items():
eq_(result.source, result.target,
key + ': %r != %r' % (result.source, result.target))
import pytest
import re
import requests
import time
import gramex.cache
from fnmatch import fnmatch
from lxml.html import document_fromstring
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from six import string_types
from tornado.web import create_signed_value
from gramex.config import ChainConfig, PathConfig, objectpath, variables
# Get Gramex conf from current directory
gramex_conf = ChainConfig()
gramex_conf['source'] = PathConfig(os.path.join(variables['GRAMEXPATH'], 'gramex.yaml'))
gramex_conf['base'] = PathConfig('gramex.yaml')
secret = objectpath(+gramex_conf, 'app.settings.cookie_secret')
drivers = {}
default = object()
class ChromeConf(dict):
def __init__(self, **conf):
self['goog:chromeOptions'] = {'args': ['--no-sandbox']}
for key, val in conf.items():
getattr(self, key)(val)
def headless(self, val):
if val:
self['goog:chromeOptions']['args'].append('--headless')
def test_variables(self):
# Templates interpolate string variables
# Create configuration with 2 layers and a subdirectory import
conf = +ChainConfig(
base=PathConfig(self.chain.base),
child=PathConfig(self.chain.child),
)
# Custom variables are deleted after use
ok_('variables' not in conf)
for key in ['base', 'child', 'subdir']:
# {.} maps to YAML file's directory
eq_(conf['%s_DOT' % key], str(self.chain[key].parent))
# $YAMLPATH maps to YAML file's directory
eq_(conf['%s_YAMLPATH' % key], str(self.chain[key].parent))
# $YAMLURL is the relative path to YAML file's directory
eq_(conf['%s_YAMLURL' % key], conf['%s_YAMLURL_EXPECTED' % key])
# Environment variables are present by default
eq_(conf['%s_HOME' % key], os.environ.get('HOME', ''))
# Non-existent variables map to ''
eq_(conf['%s_NONEXISTENT' % key],
os.environ.get('NONEXISTENT', ''))
# Custom variables are applied
eq_(conf['%s_THIS' % key], key)
# Custom variables are inherited. Defaults do not override
eq_(conf['%s_ROOT' % key], conf.base_ROOT)
# Default variables are set
eq_(conf['%s_DEFAULT' % key], key)
# Functions run and override values
eq_(conf['%s_FUNCTION' % key], key)
# Default functions "underride" values
eq_(conf['%s_DEFAULT_FUNCTION' % key], 'base')
# Functions can use variables using gramex.config.variables
eq_(conf['%s_FUNCTION_VAR' % key], conf.base_ROOT + key)
# Derived variables
eq_(conf['%s_DERIVED' % key], '%s/derived' % key)
# $URLROOT is the frozen to base $YAMLURL
eq_(conf['%s_YAMLURL_VAR' % key],
conf['%s_YAMLURL_VAR_EXPECTED' % key])
# $GRAMEXPATH is the gramex path
gramex_path = os.path.dirname(inspect.getfile(gramex))
eq_(conf['%s_GRAMEXPATH' % key], gramex_path)
# $GRAMEXAPPS is the gramex apps path
eq_(conf['%s_GRAMEXAPPS' % key], os.path.join(gramex_path, 'apps'))
# $GRAMEXHOST is the socket.gethostname
eq_(conf['%s_GRAMEXHOST' % key], socket.gethostname())
# Imports do not override, but do setdefault
eq_(conf['path'], str(self.chain['base'].parent))
eq_(conf['subpath'], str(self.chain['subdir'].parent))
# Check if variable types are preserved
eq_(conf['numeric'], 1)
eq_(conf['boolean'], True)
eq_(conf['object'], {'x': 1})
eq_(conf['list'], [1, 2])
# Check if variables of different types are string substituted
eq_(conf['numeric_subst'], '/1')
eq_(conf['boolean_subst'], '/True')
# Actually, conf['object_subst'] is "/AttrDict([('x', 1)])". Let's not test that.
# eq_(conf['object_subst'], "/{'x': 1}")
eq_(conf['list_subst'], '/[1, 2]')
# Check condition variables
for key, val in conf['conditions'].items():
eq_('is-' + key, val)
def callback_commandline(commands):
'''
Find what method should be run based on the command line programs. This
refactoring allows us to test gramex.commandline() to see if it processes
the command line correctly, without actually running the commands.
Returns a callback method and kwargs for the callback method.
'''
# Ensure that log colours are printed properly on cygwin.
if sys.platform == 'cygwin': # colorama.init() gets it wrong on Cygwin
import colorlog.escape_codes # noqa: Let colorlog call colorama.init() first
import colorama
colorama.init(convert=True) # Now we'll override with convert=True
# Set logging config at startup. (Services may override this.)
log_config = (+PathConfig(paths['source'] / 'gramex.yaml')).get(
'log', AttrDict())
log_config.root.level = logging.INFO
from . import services
services.log(log_config)
# args has all optional command line args as a dict of values / lists.
# cmd has all positional arguments as a list.
args = parse_command_line(commands)
cmd = args.pop('_')
# If --help or -V --version is specified, print a message and end
if args.get('V') is True or args.get('version') is True:
return console, {'msg': 'Gramex %s' % __version__}
if args.get('help') is True:
return console, {'msg': __doc__.strip().format(**globals())}
# Any positional argument is treated as a gramex command
if len(cmd) > 0:
kwargs = {'cmd': cmd, 'args': args}
base_command = cmd.pop(0).lower()
method = 'install' if base_command == 'update' else base_command
if method in {
'install',
'uninstall',
'setup',
'run',
'service',
'init',
'mail',
'license',
}:
import gramex.install
return getattr(gramex.install, method), kwargs
raise NotImplementedError('Unknown gramex command: %s' % base_command)
# Use current dir as base (where gramex is run from) if there's a gramex.yaml.
# Else use source/guide, and point the user to the welcome screen
if not os.path.isfile('gramex.yaml'):
from gramex.install import run
args.setdefault('browser', '/welcome')
return run, {'cmd': ['guide'], 'args': args}
app_log.info('Gramex %s | %s | Python %s', __version__, os.getcwd(),
sys.version.replace('\n', ' '))
return init, {'cmd': AttrDict(app=args)}