def setup_default_kwargs(cls): ''' Use default config from handlers.<Class>.* and handlers.BaseHandler. Called by gramex.services.url(). ''' c = cls.conf.setdefault('kwargs', {}) merge(c, objectpath(conf, 'handlers.' + cls.conf.handler, {}), mode='setdefault') merge(c, objectpath(conf, 'handlers.BaseHandler', {}), mode='setdefault')
def call(self, url, args, method, headers): r = self.check('/formhandler/edits-' + url, data=args, method=method, headers=headers) meta = r.json() # meta has 'ignored' with list of ignored columns ok_(['x', args.get('x', [1])] in objectpath(meta, 'data.ignored')) # meta has 'filters' for PUT and DELETE. It is empty for post if method.lower() == 'post': eq_(objectpath(meta, 'data.filters'), []) else: ok_(isinstance(objectpath(meta, 'data.filters'), list)) return r
def put(self, *path_args, **path_kwargs): '''Update attributes and files''' # PUT can update only 1 ID at a time. Use only the first upload, if any uploads = self.request.files.get('file', [])[:1] id = self.args.get('id', [-1]) # User cannot change the path, size, date or user attributes for s in ('path', 'size', 'date'): self.args.pop(s, None) for s in self.user_fields: self.args.pop('user_%s' % s, None) # These are updated only when a file is uploaded if len(uploads): user = self.current_user or {} self.args.setdefault('size', []).append(len(uploads[0]['body'])) self.args.setdefault('date', []).append(int(time.time())) for s in self.user_fields: self.args.setdefault('user_%s' % s.replace('.', '_'), []).append( objectpath(user, s)) conf = self.datasets.data files = gramex.data.filter(conf.url, table=conf.table, args={'id': id}) result = yield super().put(*path_args, **path_kwargs) if len(uploads) and len(files): path = os.path.join(self.path, files['path'].iloc[0]) with open(path, 'wb') as handle: handle.write(uploads[0]['body']) return result
def post(self, *path_args, **path_kwargs): '''Saves uploaded files, then updates metadata DB''' user = self.current_user or {} uploads = self.request.files.get('file', []) n = len(uploads) # Initialize all DB columns (except ID) to have the same number of rows as uploads for key, col in list(self._db_cols.items())[1:]: self.args[key] = self.args.get(key, []) + [col.type.python_type()] * n for key in self.args: self.args[key] = self.args[key][:n] for i, upload in enumerate(uploads): file = os.path.basename(upload.get('filename', '')) ext = os.path.splitext(file)[1] path = re.sub(r'[^!#$%&()+,.0-9;<=>@A-Z\[\]^`a-z{}~]', '-', file) while os.path.exists(os.path.join(self.path, path)): path = os.path.splitext(path)[0] + choice(digits + ascii_lowercase) + ext self.args['file'][i] = file self.args['ext'][i] = ext.lower() self.args['path'][i] = path self.args['size'][i] = len(upload['body']) self.args['date'][i] = int(time.time()) # Guess MIME type from filename if it's unknown self.args['mime'][i] = upload['content_type'] if self.args['mime'][i] == 'application/unknown': self.args['mime'][i] = guess_type(file, strict=False)[0] # Append user attributes for s in self.user_fields: self.args['user_%s' % s.replace('.', '_')][i] = objectpath(user, s) self.check_filelimits() yield super().post(*path_args, **path_kwargs) for upload, path in zip(uploads, self.args['path']): with open(os.path.join(self.path, path), 'wb') as handle: handle.write(upload['body'])
def set_default_headers(self): # Only set BaseHandler headers. # Don't set headers for the specific class. Those are overrides handled # by the respective classes, not the default headers. headers = [('Server', server_header)] headers += list(objectpath(conf, 'handlers.BaseHandler.headers', {}).items()) self._write_headers(headers)
def setup_log(cls): ''' Logs access requests to gramex.requests as a CSV file. ''' logger = logging.getLogger('gramex.requests') keys = objectpath(conf, 'log.handlers.requests.keys', []) log_info = build_log_info(keys) cls.log_request = lambda handler: logger.info(log_info(handler))
def setup(cls, **kwargs): super(FormHandler, cls).setup(**kwargs) conf_kwargs = merge(AttrDict(cls.conf.kwargs), objectpath(gramex_conf, 'handlers.FormHandler', {}), 'setdefault') cls.headers = conf_kwargs.pop('headers', {}) # Top level formats: key is special. Don't treat it as data cls.formats = conf_kwargs.pop('formats', {}) default_config = conf_kwargs.pop('default', None) # Remove other known special keys from dataset configuration cls.clear_special_keys(conf_kwargs) # If top level has url: then data spec is at top level. Else it's a set of sub-keys if 'url' in conf_kwargs: cls.datasets = {'data': conf_kwargs} cls.single = True else: if 'modify' in conf_kwargs: cls.modify_all = staticmethod(build_transform( conf={'function': conf_kwargs.pop('modify', None)}, vars=cls.function_vars['modify'], filename='%s.%s' % (cls.name, 'modify'), iter=False)) cls.datasets = conf_kwargs cls.single = False # Apply defaults to each key if isinstance(default_config, dict): for key in cls.datasets: config = cls.datasets[key].get('default', {}) cls.datasets[key]['default'] = merge(config, default_config, mode='setdefault') # Ensure that each dataset is a dict with a url: key at least for key, dataset in list(cls.datasets.items()): if not isinstance(dataset, dict): app_log.error('%s: %s: must be a dict, not %r' % (cls.name, key, dataset)) del cls.datasets[key] elif 'url' not in dataset: app_log.error('%s: %s: does not have a url: key' % (cls.name, key)) del cls.datasets[key] # Ensure that id: is a list -- if it exists if 'id' in dataset and not isinstance(dataset['id'], list): dataset['id'] = [dataset['id']] # Convert function: into a data = transform(data) function conf = { 'function': dataset.pop('function', None), 'args': dataset.pop('args', None), 'kwargs': dataset.pop('kwargs', None) } if conf['function'] is not None: fn_name = '%s.%s.transform' % (cls.name, key) dataset['transform'] = build_transform( conf, vars={'data': None, 'handler': None}, filename=fn_name, iter=False) # Convert modify: and prepare: into a data = modify(data) function for fn, fn_vars in cls.function_vars.items(): if fn in dataset: dataset[fn] = build_transform( conf={'function': dataset[fn]}, vars=fn_vars, filename='%s.%s.%s' % (cls.name, key, fn), iter=False)
def setup(cls, path, default_filename=None, index=None, index_template=None, template=None, headers={}, default={}, methods=['GET', 'HEAD', 'POST'], **kwargs): # Convert template: '*.html' into transform: {'*.html': {function: template}} # Do this before BaseHandler setup so that it can invoke the transforms required if template is not None: if template is True: template = '*' kwargs.setdefault( 'transform', AttrDict())[template] = AttrDict(function='template') super(FileHandler, cls).setup(**kwargs) cls.root, cls.pattern = None, None if isinstance(path, dict): cls.root = AttrDict([(re.compile(p + '$'), val) for p, val in path.items()]) elif isinstance(path, list): cls.root = [Path(path_item).absolute() for path_item in path] elif '*' in path: cls.pattern = path else: cls.root = Path(path).absolute() cls.default_filename = default_filename cls.index = index cls.ignore = cls.set(cls.kwargs.ignore) cls.allow = cls.set(cls.kwargs.allow) cls.default = default cls.index_template = read_template( Path(index_template ) if index_template is not None else _default_index_template) cls.headers = AttrDict( objectpath(gramex_conf, 'handlers.FileHandler.headers', {})) cls.headers.update(headers) # Set supported methods for method in (methods if isinstance(methods, (tuple, list)) else [methods]): method = method.lower() setattr(cls, method, cls._head if method == 'head' else cls._get)
def setup(cls, path, default_filename=None, index=None, index_template=None, headers={}, default={}, **kwargs): # Convert template: '*.html' into transform: {'*.html': {function: template}} # Convert sass: ['*.scss', '*.sass'] into transform: {'*.scss': {function: sass}} # Do this before BaseHandler setup so that it can invoke the transforms required for key in ('template', 'sass', 'scss'): val = kwargs.pop(key, None) if val: # template/sass: true is the same as template: '*' val = '*' if val is True else val if isinstance( val, (list, tuple)) else [val] kwargs.setdefault('transform', AttrDict()).update( {v: AttrDict(function=key) for v in val}) super(FileHandler, cls).setup(**kwargs) cls.root, cls.pattern = None, None if isinstance(path, dict): cls.root = AttrDict([(re.compile(p + '$'), val) for p, val in path.items()]) elif isinstance(path, list): cls.root = [Path(path_item).absolute() for path_item in path] elif '*' in path: cls.pattern = path else: cls.root = Path(path).absolute() cls.default_filename = default_filename cls.index = index cls.ignore = cls.set(cls.kwargs.ignore) cls.allow = cls.set(cls.kwargs.allow) cls.default = default cls.index_template = read_template( Path(index_template ) if index_template is not None else _default_index_template) cls.headers = AttrDict( objectpath(gramex_conf, 'handlers.FileHandler.headers', {})) cls.headers.update(headers) cls.post = cls.put = cls.delete = cls.patch = cls.options = cls.get
def _check_condition(condition, user): ''' A condition is a dictionary of {keypath: values}. Extract the keypath from the user. Check if the value is in the values list. If not, this condition fails. ''' for keypath, values in condition.items(): node = objectpath(user, keypath) # If nothing exists at keypath, the check fails if node is None: return False # If the value is a list, it must overlap with values elif isinstance(node, list): if not set(node) & values: return False # If the value is not a list, it must be present in values elif node not in values: return False return True
def set_opacity(attr, shape, spec, data: dict): '''Generator for fill-opacity, stroke-opacity commands''' val = expr(spec, data) if val is not None: fill_opacity(objectpath(shape, attr), val)
def set_color(attr, shape, spec, data: dict): '''Generator for fill, stroke commands''' val = expr(spec, data) if val is not None: fill_color(objectpath(shape, attr), val)
def setup(cls, prepare=None, action=None, delay=None, session_expiry=None, session_inactive=None, user_key='user', lookup=None, recaptcha=None, **kwargs): # Switch SSL certificates if required to access Google, etc gramex.service.threadpool.submit(check_old_certs) # Set up default redirection based on ?next=... if 'redirect' not in kwargs: kwargs['redirect'] = AttrDict([('query', 'next'), ('header', 'Referer')]) super(AuthHandler, cls).setup(**kwargs) # Set up logging for login/logout events logger = logging.getLogger('gramex.user') keys = objectpath(gramex.conf, 'log.handlers.user.keys', []) log_info = build_log_info(keys, 'event') cls.log_user_event = lambda handler, event: logger.info( log_info(handler, event)) # Count failed logins cls.failed_logins = Counter() # Set delay for failed logins from the delay: parameter which can be a number or list default_delay = [1, 1, 5] cls.delay = delay if isinstance(cls.delay, list) and not all( isinstance(n, (int, float)) for n in cls.delay): app_log.warning('%s: Ignoring invalid delay: %r', cls.name, cls.delay) cls.delay = default_delay elif isinstance(cls.delay, (int, float)) or cls.delay is None: cls.delay = default_delay # Set up session user key, session expiry and inactive expiry cls.session_user_key = user_key cls.session_expiry = session_expiry cls.session_inactive = session_inactive # Set up lookup. Split a copy into self.lookup_id which has the ID, and # self.lookup which has gramex.data keywords. cls.lookup = None if lookup is not None: cls.lookup = lookup.copy() if isinstance(lookup, dict): cls.lookup_id = cls.lookup.pop('id', 'user') else: app_log.error('%s: lookup must be a dict, not %s', cls.name, cls.lookup) # Set up prepare cls.auth_methods = {} if prepare is not None: cls.auth_methods['prepare'] = build_transform( conf={'function': prepare}, vars={ 'handler': None, 'args': None }, filename='url:%s:prepare' % cls.name, iter=False) # Prepare recaptcha if recaptcha is not None: if 'key' not in recaptcha: app_log.error('%s: recaptcha.key missing', cls.name) elif 'key' not in recaptcha: app_log.error('%s: recaptcha.secret missing', cls.name) else: recaptcha.setdefault('action', 'login') cls.auth_methods['recaptcha'] = cls.check_recaptcha # Set up post-login actions cls.actions = [] if action is not None: if not isinstance(action, list): action = [action] for conf in action: cls.actions.append( build_transform(conf, vars=AttrDict(handler=None), filename='url:%s:%s' % (cls.name, conf.function)))
import requests import time import gramex.cache from fnmatch import fnmatch from lxml.html import document_fromstring from selenium import webdriver from selenium.common.exceptions import NoSuchElementException from six import string_types from tornado.web import create_signed_value from gramex.config import ChainConfig, PathConfig, objectpath, variables # Get Gramex conf from current directory gramex_conf = ChainConfig() gramex_conf['source'] = PathConfig(os.path.join(variables['GRAMEXPATH'], 'gramex.yaml')) gramex_conf['base'] = PathConfig('gramex.yaml') secret = objectpath(+gramex_conf, 'app.settings.cookie_secret') drivers = {} default = object() class ChromeConf(dict): def __init__(self, **conf): self['goog:chromeOptions'] = {'args': ['--no-sandbox']} for key, val in conf.items(): getattr(self, key)(val) def headless(self, val): if val: self['goog:chromeOptions']['args'].append('--headless')