def languagetool(handler, *args, **kwargs):
import gramex
merge(kwargs, _languagetool['defaults'], mode='setdefault')
yield gramex.service.threadpool.submit(languagetool_download)
if not handler:
lang = kwargs.get('lang', 'en-us')
q = kwargs.get('q', '')
else:
lang = handler.get_argument('lang', 'en-us')
q = handler.get_argument('q', '')
result = yield languagetoolrequest(q, lang, **kwargs)
errors = json.loads(result.decode('utf8'))['matches']
if errors:
result = {
"errors": errors,
}
corrected = list(q)
d_offset = 0 # difference in the offset caused by the correction
for error in errors:
# only accept the first replacement for an error
correction = error['replacements'][0]['value']
offset, limit = error['offset'], error['length']
offset += d_offset
del corrected[offset:(offset + limit)]
for i, char in enumerate(correction):
corrected.insert(offset + i, char)
d_offset += len(correction) - limit
result['correction'] = "".join(corrected)
result = json.dumps(result)
raise Return(result)
def test_edit_multidata_modify(self):
csv_path = os.path.join(folder, 'sales-edits.csv')
self.sales.to_csv(csv_path, index=False, encoding='utf-8')
tempfiles[csv_path] = csv_path
dbutils.mysql_create_db(variables.MYSQL_SERVER, 'test_formhandler', sales=self.sales)
try:
row = {'देश': 'भारत', 'city': 'X', 'product': 'Q', 'growth': None}
result = self.check('/formhandler/edits-multidata-modify', method='post', data={
'csv:देश': ['भारत'],
'csv:city': ['X'],
'csv:product': ['Q'],
'csv:sales': ['10'],
'sql:देश': ['भारत'],
'sql:city': ['X'],
'sql:product': ['Q'],
'sql:sales': ['20'],
}, headers={
'count-csv': '1',
'count-sql': '1',
}).json()
eq_(result['csv']['modify'], 8)
eq_(result['modify'], 8)
data = self.check('/formhandler/edits-multidata').json()
eq_(data['csv'][-1], merge(row, {'sales': 10}))
eq_(data['sql'][-1], merge(row, {'sales': 20}))
eq_(len(data['csv']), len(self.sales) + 1)
eq_(len(data['sql']), len(self.sales) + 1)
finally:
dbutils.mysql_drop_db(variables.MYSQL_SERVER, 'test_formhandler')
def get(self):
self.settings[self._OAUTH_SETTINGS_KEY] = {
'key': self.kwargs['key'],
'secret': self.kwargs['secret']
}
code = self.get_arg('code', '')
if code:
access = yield self.get_authenticated_user(
redirect_uri=self.xredirect_uri, code=code)
user = yield self.oauth2_request(
'https://www.googleapis.com/oauth2/v1/userinfo',
access_token=access['access_token'])
merge(user, access, mode='setdefault')
yield self.set_user(user, id='email')
self.session['google_access_token'] = access['access_token']
self.redirect_next()
else:
self.save_redirect_page()
# Ensure user-specified scope has 'profile' and 'email'
scope = self.kwargs.get('scope', [])
scope = scope if isinstance(scope, list) else [scope]
scope = list(set(scope) | {'profile', 'email'})
# Return the list
yield self.authorize_redirect(redirect_uri=self.xredirect_uri,
client_id=self.kwargs['key'],
scope=scope,
response_type='code',
extra_params=self.kwargs.get(
'extra_params', {}))
def sass(handler, template=join(ui_dir, 'bootstrap-theme.scss')):
'''
Return a bootstrap theme based on the custom SASS variables provided.
'''
args = dict(variables.get('ui-bootstrap', {}))
args.update({key: handler.get_arg(key) for key in handler.args})
args = {key: val for key, val in args.items() if val}
# Set default args
config = gramex.cache.open(config_file)
merge(args, config.get('defaults'), mode='setdefault')
cache_key = get_cache_key({'template': template, 'args': args})
# Replace fonts from config file, if available
google_fonts = set()
for key in ('font-family-base', 'headings-font-family'):
if key in args and args[key] in config['fonts']:
fontinfo = config['fonts'][args[key]]
args[key] = fontinfo['stack']
if 'google' in fontinfo:
google_fonts.add(fontinfo['google'])
# Cache based on the dict and config as template.<cache-key>.css
base = os.path.splitext(os.path.basename(template))[0] + '.' + cache_key
cache_path = join(cache_dir, base + '.css')
if not os.path.exists(cache_path) or os.stat(template).st_mtime > os.stat(
cache_path).st_mtime:
# Create a SCSS file based on the args
scss_path = join(cache_dir, base + '.scss')
with io.open(scss_path, 'wb') as handle:
result = gramex.cache.open(template, 'template').generate(
variables=args,
google_fonts=google_fonts,
)
handle.write(result)
# Run sass to generate the output
proc = gramex.cache.Subprocess([
'node',
sass_path,
scss_path,
cache_path,
'--output-style',
'compressed',
# Allow importing path from these paths
'--include-path',
os.path.dirname(template),
'--include-path',
ui_dir,
'--include-path',
bootstrap_dir,
])
out, err = yield proc.wait_for_exit()
if proc.proc.returncode:
raise RuntimeError('node-sass compilation failure',
err.decode('utf-8'))
handler.set_header('Content-Type', 'text/css')
raise Return(gramex.cache.open(cache_path, 'bin', mode='rb'))
def setup_default_kwargs(cls):
'''
Use default config from handlers.<Class>.* and handlers.BaseHandler.
Called by gramex.services.url().
'''
c = cls.conf.setdefault('kwargs', {})
merge(c, objectpath(conf, 'handlers.' + cls.conf.handler, {}), mode='setdefault')
merge(c, objectpath(conf, 'handlers.BaseHandler', {}), mode='setdefault')
def setup(cls, **kwargs):
super(FormHandler, cls).setup(**kwargs)
conf_kwargs = merge(AttrDict(cls.conf.kwargs),
objectpath(gramex_conf, 'handlers.FormHandler', {}),
'setdefault')
cls.headers = conf_kwargs.pop('headers', {})
# Top level formats: key is special. Don't treat it as data
cls.formats = conf_kwargs.pop('formats', {})
default_config = conf_kwargs.pop('default', None)
# Remove other known special keys from dataset configuration
cls.clear_special_keys(conf_kwargs)
# If top level has url: then data spec is at top level. Else it's a set of sub-keys
if 'url' in conf_kwargs:
cls.datasets = {'data': conf_kwargs}
cls.single = True
else:
if 'modify' in conf_kwargs:
cls.modify_all = staticmethod(build_transform(
conf={'function': conf_kwargs.pop('modify', None)},
vars=cls.function_vars['modify'],
filename='%s.%s' % (cls.name, 'modify'), iter=False))
cls.datasets = conf_kwargs
cls.single = False
# Apply defaults to each key
if isinstance(default_config, dict):
for key in cls.datasets:
config = cls.datasets[key].get('default', {})
cls.datasets[key]['default'] = merge(config, default_config, mode='setdefault')
# Ensure that each dataset is a dict with a url: key at least
for key, dataset in list(cls.datasets.items()):
if not isinstance(dataset, dict):
app_log.error('%s: %s: must be a dict, not %r' % (cls.name, key, dataset))
del cls.datasets[key]
elif 'url' not in dataset:
app_log.error('%s: %s: does not have a url: key' % (cls.name, key))
del cls.datasets[key]
# Ensure that id: is a list -- if it exists
if 'id' in dataset and not isinstance(dataset['id'], list):
dataset['id'] = [dataset['id']]
# Convert function: into a data = transform(data) function
conf = {
'function': dataset.pop('function', None),
'args': dataset.pop('args', None),
'kwargs': dataset.pop('kwargs', None)
}
if conf['function'] is not None:
fn_name = '%s.%s.transform' % (cls.name, key)
dataset['transform'] = build_transform(
conf, vars={'data': None, 'handler': None}, filename=fn_name, iter=False)
# Convert modify: and prepare: into a data = modify(data) function
for fn, fn_vars in cls.function_vars.items():
if fn in dataset:
dataset[fn] = build_transform(
conf={'function': dataset[fn]},
vars=fn_vars,
filename='%s.%s.%s' % (cls.name, key, fn), iter=False)
def opener(callback, read=False, **open_kwargs):
'''
Converts any function that accepts a string or handle as its parameter into
a function that takes the first parameter from a file path.
Here are a few examples::
jsonload = opener(json.load)
jsonload('x.json') # opens x.json and runs json.load(handle)
gramex.cache.open('x.json', jsonload) # Loads x.json, cached
# read=True parameter passes the contents (not handle) to the function
template = opener(string.Template, read=True)
template('abc.txt').substitute(x=val)
gramex.cache.open('abc.txt', template).substitute(x=val)
# If read=True, callback may be None. The result of .read() is passed as-is
text = opener(None, read=True)
gramex.cache.open('abc.txt', text)
Keyword arguments applicable for ``io.open`` are passed to ``io.open``. These
default to ``io.open(mode='r', buffering=-1, encoding='utf-8',
errors='strict', newline=None, closefd=True)``. All other arguments and
keyword arguments are passed to the callback (e.g. to ``json.load``).
When reading binary files, pass ``mode='rb', encoding=None, errors=None``.
'''
merge(open_kwargs, _opener_defaults, 'setdefault')
if read:
# Pass contents to callback
def method(path, **kwargs):
open_args = {
key: kwargs.pop(key, val)
for key, val in open_kwargs.items()
}
with io.open(path, **open_args) as handle:
result = handle.read()
return callback(result, **
kwargs) if callable(callback) else result
else:
if not callable(callback):
raise ValueError('opener callback %s not a function',
repr(callback))
# Pass handle to callback
def method(path, **kwargs):
open_args = {
key: kwargs.pop(key, val)
for key, val in open_kwargs.items()
}
with io.open(path, **open_args) as handle:
return callback(handle, **kwargs)
return method
def sass(handler, template=uicomponents_path):
'''
Return a bootstrap theme based on the custom SASS variables provided.
'''
args = dict(variables.get('ui-bootstrap', {}))
args.update({key: handler.get_arg(key) for key in handler.args})
args = {key: val for key, val in args.items() if val}
# Set default args
config = gramex.cache.open(config_file)
merge(args, config.get('defaults'), mode='setdefault')
cache_key = {'template': template, 'args': args}
cache_key = json.dumps(cache_key, sort_keys=True,
ensure_ascii=True).encode('utf-8')
cache_key = md5(cache_key).hexdigest()[:5]
# Replace fonts from config file, if available
google_fonts = set()
for key in ('font-family-base', 'headings-font-family'):
if key in args and args[key] in config['fonts']:
fontinfo = config['fonts'][args[key]]
args[key] = fontinfo['stack']
if 'google' in fontinfo:
google_fonts.add(fontinfo['google'])
# Cache based on the dict and config as template.<cache-key>.css
base = os.path.splitext(os.path.basename(template))[0] + '.' + cache_key
cache_path = join(cache_dir, base + '.css')
if not os.path.exists(cache_path) or os.stat(template).st_mtime > os.stat(
cache_path).st_mtime:
# Create a SCSS file based on the args
scss_path = join(cache_dir, base + '.scss')
with io.open(scss_path, 'wb') as handle:
result = gramex.cache.open(template, 'template').generate(
variables=args,
uicomponents_path=uicomponents_path.replace('\\', '/'),
bootstrap_path=bootstrap_path.replace('\\', '/'),
google_fonts=google_fonts,
)
handle.write(result)
# Run sass to generate the output
options = ['--output-style', 'compressed']
proc = gramex.cache.Subprocess(
['node', sass_path, scss_path, cache_path] + options)
out, err = yield proc.wait_for_exit()
if proc.proc.returncode:
app_log.error('node-sass error: %s', err)
raise RuntimeError('Compilation failure')
handler.set_header('Content-Type', 'text/css')
raise Return(gramex.cache.open(cache_path, 'bin', mode='rb'))
def set_user(self, user, id):
# Find session expiry time
expires_days = self.session_expiry
if isinstance(self.session_expiry, dict):
# If session_expiry (se) is a dict, use se.values[args[se.key]]
# Or else, default to se.default - or None
default = self.session_expiry.get('default', None)
key = self.session_expiry.get('key', None)
val = self.get_arg(key, None)
lookup = self.session_expiry.get('values', {})
expires_days = lookup.get(val, default)
# When user logs in, change session ID and invalidate old session
# https://www.owasp.org/index.php/Session_fixation
self.get_session(expires_days=expires_days, new=True)
# The unique ID for a user varies across logins. For example, Google and
# Facebook provide an "id", but for Twitter, it's "username". For LDAP,
# it's "dn". Allow auth handlers to decide their own ID attribute and
# store it as "id" for consistency. Logging depends on this, for example.
user['id'] = user[id]
self.session[self.session_user_key] = user
self.failed_logins[user[id]] = 0
# Extend user attributes looking up the user ID in a lookup table
if self.lookup is not None:
# Look up the user ID in the lookup table and fetch all matching rows
users = yield gramex.service.threadpool.submit(
gramex.data.filter,
args={self.lookup_id: [user['id']]},
**self.lookup)
if len(users) > 0 and self.lookup_id in users.columns:
# Update the user attributes with the non-null items in the looked up row
user.update({
key: val
for key, val in users.iloc[0].iteritems()
if not gramex.data.pd.isnull(val)
})
# Persist user attributes (e.g. refresh_token from Google auth.)
# If new user object doesn't have anything from previous login, restore it.
info = self.update_user(user[id], active='y', **user)
merge(self.session[self.session_user_key], info, mode='setdefault')
# If session_inactive: is specified, set expiry date on the session
if self.session_inactive is not None:
self.session['_i'] = self.session_inactive * 24 * 60 * 60
# Run post-login events (e.g. ensure_single_session) specified in config
for callback in self.actions:
callback(self)
self.log_user_event(event='login')
def cache(conf):
'''Set up caches'''
for name, config in conf.items():
cache_type = config['type']
if cache_type not in _cache_defaults:
app_log.warning('cache: %s has unknown type %s', name, config.type)
continue
config = merge(dict(config),
_cache_defaults[cache_type],
mode='setdefault')
if cache_type == 'memory':
info.cache[name] = urlcache.MemoryCache(
maxsize=config['size'], getsizeof=gramex.cache.sizeof)
elif cache_type == 'disk':
path = config.get('path', '.cache-' + name)
info.cache[name] = urlcache.DiskCache(
path,
size_limit=config['size'],
eviction_policy='least-recently-stored')
atexit.register(info.cache[name].close)
# if default: true, make this the default cache for gramex.cache.{open,query}
if config.get('default'):
for key in ['_OPEN_CACHE', '_QUERY_CACHE']:
val = gramex.cache.set_cache(info.cache[name],
getattr(gramex.cache, key))
setattr(gramex.cache, key, val)
def _options(self, dataset, args, path_args, path_kwargs, key):
"""For each dataset, prepare the arguments."""
if self.request.body:
content_type = self.request.headers.get('Content-Type', '')
if content_type == 'application/json':
args.update(json.loads(self.request.body))
filter_kwargs = AttrDict(dataset)
filter_kwargs.pop('modify', None)
prepare = filter_kwargs.pop('prepare', None)
queryfunction = filter_kwargs.pop('queryfunction', None)
filter_kwargs['transform_kwargs'] = {'handler': self}
# Use default arguments
defaults = {
k: v if isinstance(v, list) else [v]
for k, v in filter_kwargs.pop('default', {}).items()
}
# /(.*)/(.*) become 2 path arguments _0 and _1
defaults.update({'_%d' % k: [v] for k, v in enumerate(path_args)})
# /(?P<x>\d+)/(?P<y>\d+) become 2 keyword arguments x and y
defaults.update({k: [v] for k, v in path_kwargs.items()})
args = merge(namespaced_args(args, key), defaults, mode='setdefault')
if callable(prepare):
result = prepare(args=args, key=key, handler=self)
if result is not None:
args = result
if callable(queryfunction):
filter_kwargs['query'] = queryfunction(args=args, key=key, handler=self)
return AttrDict(
fmt=args.pop('_format', ['json'])[0],
download=args.pop('_download', [''])[0],
args=args,
meta_header=args.pop('_meta', [''])[0],
filter_kwargs=filter_kwargs,
)
def run_commands(commands, callback):
'''
For example::
run_commands(['a.yaml', 'b.yaml', '--x=1'], method)
will do the following:
- Load a.yaml into config
- Set config['a'] = 1
- Change to directory where a.yaml is
- Call method(config)
- Load b.yaml into config
- Set config['a'] = 1
- Change to directory where b.yaml is
- Call method(config)
Command line arguments are passed as ``commands``.
Callback is a function that is called for each config file.
'''
args = parse_command_line(commands)
original_path = os.getcwd()
for config_file in args.pop('_'):
config = gramex.cache.open(config_file, 'config')
config = merge(old=config, new=args, mode='overwrite')
os.chdir(os.path.dirname(os.path.abspath(config_file)))
try:
callback(**config)
finally:
os.chdir(original_path)
def check(a, b, c, mode='overwrite'):
'''Check if merge(a, b) is c. Parameters are in YAML'''
old = yaml.load(a, Loader=ConfigYAMLLoader)
new = yaml.load(b, Loader=ConfigYAMLLoader)
# merging a + b gives c
eq_(yaml.load(c, Loader=ConfigYAMLLoader), merge(old, new, mode))
# new is unchanged
# eq_(old, yaml.load(a, Loader=ConfigYAMLLoader))
eq_(new, yaml.load(b, Loader=ConfigYAMLLoader))
def test_merge(self):
# Test gramex.config.merge
def check(a, b, c, mode='overwrite'):
'''Check if merge(a, b) is c. Parameters are in YAML'''
old = yaml.load(a, Loader=ConfigYAMLLoader)
new = yaml.load(b, Loader=ConfigYAMLLoader)
# merging a + b gives c
eq_(
yaml.load(c, Loader=ConfigYAMLLoader),
merge(old, new, mode))
# new is unchanged
# eq_(old, yaml.load(a, Loader=ConfigYAMLLoader))
eq_(new, yaml.load(b, Loader=ConfigYAMLLoader))
check('x: 1', 'y: 2', 'x: 1\ny: 2')
check('x: {a: 1}', 'x: {a: 2}', 'x: {a: 2}')
check('x: {a: 1}', 'x: null', 'x: null')
check('x: {a: 1}', 'x: {b: 2}', 'x: {a: 1, b: 2}')
check('x: {a: {p: 1}}', 'x: {a: {q: 1}, b: 2}', 'x: {a: {p: 1, q: 1}, b: 2}')
check('x: {a: {p: 1}}', 'x: {a: null, b: null}', 'x: {a: null, b: null}')
check('x: 1', 'x: 2', 'x: 1', mode='underwrite')
check('x: {a: 1, c: 3}', 'x: {a: 2, b: 2}', 'x: {a: 1, c: 3, b: 2}', mode='underwrite')
# Check basic behaviour
eq_(merge({'a': 1}, {'a': 2}), {'a': 2})
eq_(merge({'a': 1}, {'a': 2}, mode='setdefault'), {'a': 1})
eq_(merge({'a': {'b': 1}}, {'a': {'b': 2}}), {'a': {'b': 2}})
eq_(merge({'a': {'b': 1}}, {'a': {'b': 2}}, mode='setdefault'), {'a': {'b': 1}})
# Ensure int keys will work
eq_(merge({1: {1: 1}}, {1: {1: 2}}), {1: {1: 2}})
eq_(merge({1: {1: 1}}, {1: {1: 2}}, mode='setdefault'), {1: {1: 1}})
