def setup_default_kwargs(cls):
'''
Use default config from handlers.<Class>.* and handlers.BaseHandler.
Called by gramex.services.url().
'''
c = cls.conf.setdefault('kwargs', {})
merge(c, objectpath(conf, 'handlers.' + cls.conf.handler, {}), mode='setdefault')
merge(c, objectpath(conf, 'handlers.BaseHandler', {}), mode='setdefault')
def call(self, url, args, method, headers):
r = self.check('/formhandler/edits-' + url, data=args, method=method, headers=headers)
meta = r.json()
# meta has 'ignored' with list of ignored columns
ok_(['x', args.get('x', [1])] in objectpath(meta, 'data.ignored'))
# meta has 'filters' for PUT and DELETE. It is empty for post
if method.lower() == 'post':
eq_(objectpath(meta, 'data.filters'), [])
else:
ok_(isinstance(objectpath(meta, 'data.filters'), list))
return r
def put(self, *path_args, **path_kwargs):
'''Update attributes and files'''
# PUT can update only 1 ID at a time. Use only the first upload, if any
uploads = self.request.files.get('file', [])[:1]
id = self.args.get('id', [-1])
# User cannot change the path, size, date or user attributes
for s in ('path', 'size', 'date'):
self.args.pop(s, None)
for s in self.user_fields:
self.args.pop('user_%s' % s, None)
# These are updated only when a file is uploaded
if len(uploads):
user = self.current_user or {}
self.args.setdefault('size', []).append(len(uploads[0]['body']))
self.args.setdefault('date', []).append(int(time.time()))
for s in self.user_fields:
self.args.setdefault('user_%s' % s.replace('.', '_'), []).append(
objectpath(user, s))
conf = self.datasets.data
files = gramex.data.filter(conf.url, table=conf.table, args={'id': id})
result = yield super().put(*path_args, **path_kwargs)
if len(uploads) and len(files):
path = os.path.join(self.path, files['path'].iloc[0])
with open(path, 'wb') as handle:
handle.write(uploads[0]['body'])
return result
def post(self, *path_args, **path_kwargs):
'''Saves uploaded files, then updates metadata DB'''
user = self.current_user or {}
uploads = self.request.files.get('file', [])
n = len(uploads)
# Initialize all DB columns (except ID) to have the same number of rows as uploads
for key, col in list(self._db_cols.items())[1:]:
self.args[key] = self.args.get(key, []) + [col.type.python_type()] * n
for key in self.args:
self.args[key] = self.args[key][:n]
for i, upload in enumerate(uploads):
file = os.path.basename(upload.get('filename', ''))
ext = os.path.splitext(file)[1]
path = re.sub(r'[^!#$%&()+,.0-9;<=>@A-Z\[\]^`a-z{}~]', '-', file)
while os.path.exists(os.path.join(self.path, path)):
path = os.path.splitext(path)[0] + choice(digits + ascii_lowercase) + ext
self.args['file'][i] = file
self.args['ext'][i] = ext.lower()
self.args['path'][i] = path
self.args['size'][i] = len(upload['body'])
self.args['date'][i] = int(time.time())
# Guess MIME type from filename if it's unknown
self.args['mime'][i] = upload['content_type']
if self.args['mime'][i] == 'application/unknown':
self.args['mime'][i] = guess_type(file, strict=False)[0]
# Append user attributes
for s in self.user_fields:
self.args['user_%s' % s.replace('.', '_')][i] = objectpath(user, s)
self.check_filelimits()
yield super().post(*path_args, **path_kwargs)
for upload, path in zip(uploads, self.args['path']):
with open(os.path.join(self.path, path), 'wb') as handle:
handle.write(upload['body'])
def set_default_headers(self):
# Only set BaseHandler headers.
# Don't set headers for the specific class. Those are overrides handled
# by the respective classes, not the default headers.
headers = [('Server', server_header)]
headers += list(objectpath(conf, 'handlers.BaseHandler.headers', {}).items())
self._write_headers(headers)
def setup_log(cls):
'''
Logs access requests to gramex.requests as a CSV file.
'''
logger = logging.getLogger('gramex.requests')
keys = objectpath(conf, 'log.handlers.requests.keys', [])
log_info = build_log_info(keys)
cls.log_request = lambda handler: logger.info(log_info(handler))
def setup(cls, **kwargs):
super(FormHandler, cls).setup(**kwargs)
conf_kwargs = merge(AttrDict(cls.conf.kwargs),
objectpath(gramex_conf, 'handlers.FormHandler', {}),
'setdefault')
cls.headers = conf_kwargs.pop('headers', {})
# Top level formats: key is special. Don't treat it as data
cls.formats = conf_kwargs.pop('formats', {})
default_config = conf_kwargs.pop('default', None)
# Remove other known special keys from dataset configuration
cls.clear_special_keys(conf_kwargs)
# If top level has url: then data spec is at top level. Else it's a set of sub-keys
if 'url' in conf_kwargs:
cls.datasets = {'data': conf_kwargs}
cls.single = True
else:
if 'modify' in conf_kwargs:
cls.modify_all = staticmethod(build_transform(
conf={'function': conf_kwargs.pop('modify', None)},
vars=cls.function_vars['modify'],
filename='%s.%s' % (cls.name, 'modify'), iter=False))
cls.datasets = conf_kwargs
cls.single = False
# Apply defaults to each key
if isinstance(default_config, dict):
for key in cls.datasets:
config = cls.datasets[key].get('default', {})
cls.datasets[key]['default'] = merge(config, default_config, mode='setdefault')
# Ensure that each dataset is a dict with a url: key at least
for key, dataset in list(cls.datasets.items()):
if not isinstance(dataset, dict):
app_log.error('%s: %s: must be a dict, not %r' % (cls.name, key, dataset))
del cls.datasets[key]
elif 'url' not in dataset:
app_log.error('%s: %s: does not have a url: key' % (cls.name, key))
del cls.datasets[key]
# Ensure that id: is a list -- if it exists
if 'id' in dataset and not isinstance(dataset['id'], list):
dataset['id'] = [dataset['id']]
# Convert function: into a data = transform(data) function
conf = {
'function': dataset.pop('function', None),
'args': dataset.pop('args', None),
'kwargs': dataset.pop('kwargs', None)
}
if conf['function'] is not None:
fn_name = '%s.%s.transform' % (cls.name, key)
dataset['transform'] = build_transform(
conf, vars={'data': None, 'handler': None}, filename=fn_name, iter=False)
# Convert modify: and prepare: into a data = modify(data) function
for fn, fn_vars in cls.function_vars.items():
if fn in dataset:
dataset[fn] = build_transform(
conf={'function': dataset[fn]},
vars=fn_vars,
filename='%s.%s.%s' % (cls.name, key, fn), iter=False)
def setup(cls,
path,
default_filename=None,
index=None,
index_template=None,
template=None,
headers={},
default={},
methods=['GET', 'HEAD', 'POST'],
**kwargs):
# Convert template: '*.html' into transform: {'*.html': {function: template}}
# Do this before BaseHandler setup so that it can invoke the transforms required
if template is not None:
if template is True:
template = '*'
kwargs.setdefault(
'transform',
AttrDict())[template] = AttrDict(function='template')
super(FileHandler, cls).setup(**kwargs)
cls.root, cls.pattern = None, None
if isinstance(path, dict):
cls.root = AttrDict([(re.compile(p + '$'), val)
for p, val in path.items()])
elif isinstance(path, list):
cls.root = [Path(path_item).absolute() for path_item in path]
elif '*' in path:
cls.pattern = path
else:
cls.root = Path(path).absolute()
cls.default_filename = default_filename
cls.index = index
cls.ignore = cls.set(cls.kwargs.ignore)
cls.allow = cls.set(cls.kwargs.allow)
cls.default = default
cls.index_template = read_template(
Path(index_template
) if index_template is not None else _default_index_template)
cls.headers = AttrDict(
objectpath(gramex_conf, 'handlers.FileHandler.headers', {}))
cls.headers.update(headers)
# Set supported methods
for method in (methods if isinstance(methods,
(tuple, list)) else [methods]):
method = method.lower()
setattr(cls, method, cls._head if method == 'head' else cls._get)
def setup(cls,
path,
default_filename=None,
index=None,
index_template=None,
headers={},
default={},
**kwargs):
# Convert template: '*.html' into transform: {'*.html': {function: template}}
# Convert sass: ['*.scss', '*.sass'] into transform: {'*.scss': {function: sass}}
# Do this before BaseHandler setup so that it can invoke the transforms required
for key in ('template', 'sass', 'scss'):
val = kwargs.pop(key, None)
if val:
# template/sass: true is the same as template: '*'
val = '*' if val is True else val if isinstance(
val, (list, tuple)) else [val]
kwargs.setdefault('transform', AttrDict()).update(
{v: AttrDict(function=key)
for v in val})
super(FileHandler, cls).setup(**kwargs)
cls.root, cls.pattern = None, None
if isinstance(path, dict):
cls.root = AttrDict([(re.compile(p + '$'), val)
for p, val in path.items()])
elif isinstance(path, list):
cls.root = [Path(path_item).absolute() for path_item in path]
elif '*' in path:
cls.pattern = path
else:
cls.root = Path(path).absolute()
cls.default_filename = default_filename
cls.index = index
cls.ignore = cls.set(cls.kwargs.ignore)
cls.allow = cls.set(cls.kwargs.allow)
cls.default = default
cls.index_template = read_template(
Path(index_template
) if index_template is not None else _default_index_template)
cls.headers = AttrDict(
objectpath(gramex_conf, 'handlers.FileHandler.headers', {}))
cls.headers.update(headers)
cls.post = cls.put = cls.delete = cls.patch = cls.options = cls.get
def _check_condition(condition, user):
'''
A condition is a dictionary of {keypath: values}. Extract the keypath from
the user. Check if the value is in the values list. If not, this condition
fails.
'''
for keypath, values in condition.items():
node = objectpath(user, keypath)
# If nothing exists at keypath, the check fails
if node is None:
return False
# If the value is a list, it must overlap with values
elif isinstance(node, list):
if not set(node) & values:
return False
# If the value is not a list, it must be present in values
elif node not in values:
return False
return True
def set_opacity(attr, shape, spec, data: dict):
'''Generator for fill-opacity, stroke-opacity commands'''
val = expr(spec, data)
if val is not None:
fill_opacity(objectpath(shape, attr), val)
def set_color(attr, shape, spec, data: dict):
'''Generator for fill, stroke commands'''
val = expr(spec, data)
if val is not None:
fill_color(objectpath(shape, attr), val)
def setup(cls,
prepare=None,
action=None,
delay=None,
session_expiry=None,
session_inactive=None,
user_key='user',
lookup=None,
recaptcha=None,
**kwargs):
# Switch SSL certificates if required to access Google, etc
gramex.service.threadpool.submit(check_old_certs)
# Set up default redirection based on ?next=...
if 'redirect' not in kwargs:
kwargs['redirect'] = AttrDict([('query', 'next'),
('header', 'Referer')])
super(AuthHandler, cls).setup(**kwargs)
# Set up logging for login/logout events
logger = logging.getLogger('gramex.user')
keys = objectpath(gramex.conf, 'log.handlers.user.keys', [])
log_info = build_log_info(keys, 'event')
cls.log_user_event = lambda handler, event: logger.info(
log_info(handler, event))
# Count failed logins
cls.failed_logins = Counter()
# Set delay for failed logins from the delay: parameter which can be a number or list
default_delay = [1, 1, 5]
cls.delay = delay
if isinstance(cls.delay, list) and not all(
isinstance(n, (int, float)) for n in cls.delay):
app_log.warning('%s: Ignoring invalid delay: %r', cls.name,
cls.delay)
cls.delay = default_delay
elif isinstance(cls.delay, (int, float)) or cls.delay is None:
cls.delay = default_delay
# Set up session user key, session expiry and inactive expiry
cls.session_user_key = user_key
cls.session_expiry = session_expiry
cls.session_inactive = session_inactive
# Set up lookup. Split a copy into self.lookup_id which has the ID, and
# self.lookup which has gramex.data keywords.
cls.lookup = None
if lookup is not None:
cls.lookup = lookup.copy()
if isinstance(lookup, dict):
cls.lookup_id = cls.lookup.pop('id', 'user')
else:
app_log.error('%s: lookup must be a dict, not %s', cls.name,
cls.lookup)
# Set up prepare
cls.auth_methods = {}
if prepare is not None:
cls.auth_methods['prepare'] = build_transform(
conf={'function': prepare},
vars={
'handler': None,
'args': None
},
filename='url:%s:prepare' % cls.name,
iter=False)
# Prepare recaptcha
if recaptcha is not None:
if 'key' not in recaptcha:
app_log.error('%s: recaptcha.key missing', cls.name)
elif 'key' not in recaptcha:
app_log.error('%s: recaptcha.secret missing', cls.name)
else:
recaptcha.setdefault('action', 'login')
cls.auth_methods['recaptcha'] = cls.check_recaptcha
# Set up post-login actions
cls.actions = []
if action is not None:
if not isinstance(action, list):
action = [action]
for conf in action:
cls.actions.append(
build_transform(conf,
vars=AttrDict(handler=None),
filename='url:%s:%s' %
(cls.name, conf.function)))
import requests
import time
import gramex.cache
from fnmatch import fnmatch
from lxml.html import document_fromstring
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from six import string_types
from tornado.web import create_signed_value
from gramex.config import ChainConfig, PathConfig, objectpath, variables
# Get Gramex conf from current directory
gramex_conf = ChainConfig()
gramex_conf['source'] = PathConfig(os.path.join(variables['GRAMEXPATH'], 'gramex.yaml'))
gramex_conf['base'] = PathConfig('gramex.yaml')
secret = objectpath(+gramex_conf, 'app.settings.cookie_secret')
drivers = {}
default = object()
class ChromeConf(dict):
def __init__(self, **conf):
self['goog:chromeOptions'] = {'args': ['--no-sandbox']}
for key, val in conf.items():
getattr(self, key)(val)
def headless(self, val):
if val:
self['goog:chromeOptions']['args'].append('--headless')
