def __init__(self, path, keys=None, **kwargs): if keys is None: keys = {} for cat in ('file', 'delete', 'save'): keys.setdefault(cat, [cat]) if not isinstance(keys[cat], list): if isinstance(keys[cat], (str, bytes)): keys[cat] = [keys[cat]] else: app_log.error('FileUpload: cat: %r must be a list or str', keys[cat]) self.keys = keys self.path = os.path.abspath(path) if not os.path.exists(self.path): os.makedirs(self.path) # store default: sqlite .meta.db store_kwargs = kwargs.get('store', { 'type': 'sqlite', 'path': os.path.join(self.path, '.meta.db') }) if self.path not in self.stores: self.stores[self.path] = get_store(**store_kwargs) self.store = self.stores[self.path] old_store_path = os.path.abspath(os.path.join(self.path, '.meta.h5')) store_path = os.path.abspath(getattr(self.store, 'path', None)) # migration: if type is not hdf5 but .meta.h5 exists, update store and remove if (os.path.exists(old_store_path) and store_path != old_store_path): self._migrate_h5(old_store_path) if 'file' not in keys: keys['file'] = ['file'] self.keys['file'] = keys['file'] if isinstance( keys['file'], list) else [keys['file']]
def setup(cls, path, keys=None, if_exists='unique', transform=None, methods=[], **kwargs): super(UploadHandler, cls).setup(**kwargs) cls.if_exists = if_exists # FileUpload uses the store= from **kwargs and ignores the rest cls.uploader = FileUpload(path, keys=keys, **kwargs) # methods=['get'] will show all file into as JSON on GET if not isinstance(methods, list): methods = [methods] methods = {method.lower() for method in methods} if 'get' in methods: cls.get = cls.fileinfo cls.transform = [] if transform is not None: if isinstance(transform, dict) and 'function' in transform: cls.transform.append( build_transform(transform, vars=AttrDict((('content', None), ('handler', None))), filename='url:%s' % cls.name)) else: app_log.error( 'UploadHandler %s: no function: in transform: %r', cls.name, transform)
def run_command(config): ''' Run config.cmd. If the command has a TARGET, replace it with config.target. Else append config.target as an argument. ''' appcmd = config.cmd # Split the command into an array of words if isinstance(appcmd, six.string_types): appcmd = shlex.split(appcmd) # If the app is a Cygwin app, TARGET should be a Cygwin path too. target = config.target cygcheck, cygpath, kwargs = which('cygcheck'), which('cygpath'), {'universal_newlines': True} if cygcheck is not None and cygpath is not None: app_path = check_output([cygpath, '-au', which(appcmd[0])], **kwargs).strip() # nosec is_cygwin_app = check_output([cygcheck, '-f', app_path], **kwargs).strip() # nosec if is_cygwin_app: target = check_output([cygpath, '-au', target], **kwargs).strip() # n osec # Replace TARGET with the actual target if 'TARGET' in appcmd: appcmd = [target if arg == 'TARGET' else arg for arg in appcmd] else: appcmd.append(target) app_log.info('Running %s', ' '.join(appcmd)) if not safe_rmtree(config.target): app_log.error('Cannot delete target %s. Aborting installation', config.target) return proc = Popen(appcmd, bufsize=-1, stdout=sys.stdout, stderr=sys.stderr, **kwargs) # nosec proc.communicate() return proc.returncode
def run(args, kwargs): if len(args) < 1: app_log.error(show_usage('run')) return if len(args) > 1: app_log.error('Can only run one app. Ignoring %s', ', '.join(args[1:])) appname = args.pop(0) app_config = get_app_config(appname, kwargs) target = app_config.target if 'dir' in app_config: target = os.path.join(target, app_config.dir) if os.path.isdir(target): os.chdir(target) gramex.paths['base'] = Path('.') # If we run with updated parameters, save for next run under the .run config run_config = app_config.setdefault('run', {}) for key, val in kwargs.items(): if key not in app_keys: run_config[key] = app_config.pop(key) save_user_config(appname, app_config) # Tell the user what configs are used cline = ' '.join('--%s=%s' % arg for arg in flatten_config(app_config.get('run', {}))) app_log.info('Gramex %s | %s %s | %s | Python %s', gramex.__version__, appname, cline, os.getcwd(), sys.version.replace('\n', ' ')) gramex.init(args=AttrDict(app=app_config['run'])) elif appname in apps_config['user']: # The user configuration has a wrong path. Inform user app_log.error('%s: no directory %s', appname, app_config.target) app_log.error('Run "gramex uninstall %s" and try again.', appname) else: app_log.error('%s: no directory %s', appname, app_config.target)
def setup_auth(cls, auth): # auth: if there's no auth: in handler, default to app.auth if auth is None: auth = conf.app.get('auth') # Treat True as an empty dict, i.e. auth: {} if auth is True: auth = AttrDict() # Set up the auth if isinstance(auth, dict): cls._auth = auth cls._on_init_methods.append(cls.authorize) cls.permissions = [] # Add check for condition if auth.get('condition'): cls.permissions.append( build_transform(auth['condition'], vars=AttrDict(handler=None), filename='url:%s.auth.permission' % cls.name)) # Add check for membership memberships = auth.get('membership', []) if not isinstance(memberships, list): memberships = [memberships] if len(memberships): cls.permissions.append(check_membership(memberships)) elif auth: app_log.error('url:%s.auth is not a dict', cls.name)
def _ensure_remove(function, path, exc_info): '''onerror callback for rmtree that tries hard to delete files''' if issubclass(exc_info[0], WindowsError): import winerror # Delete read-only files # https://bugs.python.org/issue19643 # https://bugs.python.org/msg218021 if exc_info[1].winerror == winerror.ERROR_ACCESS_DENIED: os.chmod(path, stat.S_IWRITE) return os.remove(path) # Delay delete a bit if directory is used by another process. # Typically happens on uninstall immediately after bower / npm / git # (e.g. during testing.) elif exc_info[1].winerror == winerror.ERROR_SHARING_VIOLATION: delays = [ 0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0 ] for delay in delays: time.sleep(delay) try: return os.remove(path) except WindowsError: pass # npm creates windows shortcuts that shutil.rmtree cannot delete. # os.listdir failes with a PATH_NOT_FOUND. Delete these and try again elif function == os.listdir and exc_info[ 1].winerror == winerror.ERROR_PATH_NOT_FOUND: app_log.error('Cannot delete %s', path) from win32com.shell import shell, shellcon options = shellcon.FOF_NOCONFIRMATION | shellcon.FOF_NOERRORUI code, err = shell.SHFileOperation( (0, shellcon.FO_DELETE, path, None, options)) if code == 0: raise TryAgainError() raise exc_info[1]
def r(code=None, path=None, rel=True, conda=True, convert=True, repo='https://cran.microsoft.com/', **kwargs): ''' Runs the R script and returns the result. :arg str code: R code to execute. :arg str path: R script path. Cannot be used if code is specified :arg bool rel: True treats path as relative to the caller function's file :arg bool conda: True overrides R_HOME to use the Conda R :arg bool convert: True converts R objects to Pandas and vice versa :arg str repo: CRAN repo URL All other keyword arguments as passed as parameters ''' # Use Conda R if possible if conda: r_home = _conda_r_home() if r_home: os.environ['R_HOME'] = r_home # Import the global R session try: from rpy2.robjects import r, pandas2ri, globalenv except ImportError: app_log.error('rpy2 not installed. Run "conda install rpy2"') raise except RuntimeError: app_log.error('Cannot find R. Set R_HOME env variable') raise # Set a repo so that install.packages() need not ask for one r('local({r <- getOption("repos"); r["CRAN"] <- "%s"; options(repos = r)})' % repo) # Activate or de-activate automatic conversion # https://pandas.pydata.org/pandas-docs/version/0.22.0/r_interface.html if convert: pandas2ri.activate() else: pandas2ri.deactivate() # Pass all other kwargs as global environment variables for key, val in kwargs.items(): globalenv[key] = val if code and path: raise RuntimeError('Use r(code=) or r(path=...), not both') if path: # if rel=True, load path relative to parent directory if rel: stack = inspect.getouterframes(inspect.currentframe(), 2) folder = os.path.dirname(os.path.abspath(stack[1][1])) path = os.path.join(folder, path) result = r.source(path, chdir=True) # source() returns a withVisible: $value and $visible. Use only the first result = result[0] else: result = r(code) return result
def _run_console(cmd, **kwargs): '''Run cmd and pipe output to console. Log and raise error if cmd is not found''' cmd = shlex.split(cmd) try: proc = Popen(cmd, bufsize=-1, universal_newlines=True, **kwargs) except OSError: app_log.error('Cannot find command: %s', cmd[0]) raise proc.communicate()
def setup(cls, **kwargs): super(FormHandler, cls).setup(**kwargs) conf_kwargs = merge(AttrDict(cls.conf.kwargs), objectpath(gramex_conf, 'handlers.FormHandler', {}), 'setdefault') cls.headers = conf_kwargs.pop('headers', {}) # Top level formats: key is special. Don't treat it as data cls.formats = conf_kwargs.pop('formats', {}) default_config = conf_kwargs.pop('default', None) # Remove other known special keys from dataset configuration cls.clear_special_keys(conf_kwargs) # If top level has url: then data spec is at top level. Else it's a set of sub-keys if 'url' in conf_kwargs: cls.datasets = {'data': conf_kwargs} cls.single = True else: if 'modify' in conf_kwargs: cls.modify_all = staticmethod(build_transform( conf={'function': conf_kwargs.pop('modify', None)}, vars=cls.function_vars['modify'], filename='%s.%s' % (cls.name, 'modify'), iter=False)) cls.datasets = conf_kwargs cls.single = False # Apply defaults to each key if isinstance(default_config, dict): for key in cls.datasets: config = cls.datasets[key].get('default', {}) cls.datasets[key]['default'] = merge(config, default_config, mode='setdefault') # Ensure that each dataset is a dict with a url: key at least for key, dataset in list(cls.datasets.items()): if not isinstance(dataset, dict): app_log.error('%s: %s: must be a dict, not %r' % (cls.name, key, dataset)) del cls.datasets[key] elif 'url' not in dataset: app_log.error('%s: %s: does not have a url: key' % (cls.name, key)) del cls.datasets[key] # Ensure that id: is a list -- if it exists if 'id' in dataset and not isinstance(dataset['id'], list): dataset['id'] = [dataset['id']] # Convert function: into a data = transform(data) function conf = { 'function': dataset.pop('function', None), 'args': dataset.pop('args', None), 'kwargs': dataset.pop('kwargs', None) } if conf['function'] is not None: fn_name = '%s.%s.transform' % (cls.name, key) dataset['transform'] = build_transform( conf, vars={'data': None, 'handler': None}, filename=fn_name, iter=False) # Convert modify: and prepare: into a data = modify(data) function for fn, fn_vars in cls.function_vars.items(): if fn in dataset: dataset[fn] = build_transform( conf={'function': dataset[fn]}, vars=fn_vars, filename='%s.%s.%s' % (cls.name, key, fn), iter=False)
def redirect_method(handler): next_uri = method(handler) if next_uri is not None: target = urlparse(next_uri) if not target.scheme and not target.netloc: return next_uri req = handler.request if req.protocol == target.scheme and req.host == target.netloc: return next_uri app_log.error('Not redirecting to external url: %s', next_uri)
def transforms(self, content): for transform in self.transform: for value in transform(content, self): if isinstance(value, dict): content = value elif value is not None: app_log.error( 'UploadHandler %s: transform returned %r, not dict', self.name, value) return content
def _run_console(cmd, **kwargs): '''Run cmd and pipe output to console (sys.stdout / sys.stderr)''' cmd = shlex.split(cmd) try: proc = Popen(cmd, bufsize=-1, stdout=sys.stdout, stderr=sys.stderr, universal_newlines=True, **kwargs) except OSError: app_log.error('Cannot find command: %s', cmd[0]) raise proc.communicate()
def service(args, kwargs): try: import gramex.winservice except ImportError: app_log.error('Unable to load winservice. Is this Windows?') raise if len(args) < 1: app_log.error(show_usage('service')) return gramex.winservice.GramexService.setup(args, **kwargs)
def init(args, kwargs): '''Create Gramex scaffolding files.''' if len(args) > 1: app_log.error(show_usage('init')) return kwargs.setdefault('target', os.getcwd()) app_log.info('Initializing Gramex project at %s', kwargs.target) data = { 'appname': os.path.basename(kwargs.target), 'author': _check_output('git config user.name', default='Author'), 'email': _check_output('git config user.email', default='*****@*****.**'), 'date': datetime.datetime.today().strftime('%Y-%m-%d'), 'version': gramex.__version__, } # Ensure that appname is a valid Python module name appname = slug.module(data['appname']) if appname[0] not in string.ascii_lowercase: appname = 'app' + appname data['appname'] = appname # Create a git repo. But if git fails, do not stop. Continue with the rest. try: _run_console('git init') except OSError: pass # Install Git LFS if available. Set git_lfs=None if it fails, so .gitignore ignores assets/** data['git_lfs'] = which('git-lfs') if data['git_lfs']: try: _run_console('git lfs install') _run_console('git lfs track "assets/**"') except OSError: data['git_lfs'] = None # Copy all directories & files (as templates) source_dir = os.path.join(variables['GRAMEXPATH'], 'apps', 'init') for root, dirs, files in os.walk(source_dir): for name in dirs + files: source = os.path.join(root, name) relpath = os.path.relpath(root, start=source_dir) target = os.path.join(kwargs.target, relpath, name.replace('appname', appname)) _copy(source, target, template_data=data) for empty_dir in ('img', 'data'): _mkdir(os.path.join(kwargs.target, 'assets', empty_dir)) # Copy error files as-is (not as templates) error_dir = os.path.join(kwargs.target, 'error') _mkdir(error_dir) for source in glob( os.path.join(variables['GRAMEXPATH'], 'handlers', '?0?.html')): target = os.path.join(error_dir, os.path.basename(source)) _copy(source, target) run_setup(kwargs.target)
def load(self, key, default=None): result = self.store.get(key) if result is None: return default try: return json.loads( result, object_pairs_hook=AttrDict, cls=CustomJSONDecoder) except ValueError: app_log.error('RedisStore("%s").load("%s") is not JSON ("%r..."")', self.store, key, result) return default
def create_mail(data): ''' Return kwargs that can be passed to a mailer.mail ''' mail = {} for key in ['bodyfile', 'htmlfile', 'markdownfile']: target = key.replace('file', '') if key in alert and target not in alert: path = _tmpl(alert[key]).generate(**data).decode('utf-8') tmpl = gramex.cache.open(path, 'template') mail[target] = tmpl.generate(**data).decode('utf-8') for key in addr_fields + ['subject', 'body', 'html', 'markdown']: if key not in alert: continue if isinstance(alert[key], list): mail[key] = [ _tmpl(v).generate(**data).decode('utf-8') for v in alert[key] ] else: mail[key] = _tmpl(alert[key]).generate(**data).decode('utf-8') headers = {} # user: {id: ...} creates an X-Gramex-User header to mimic the user if 'user' in alert: user = deepcopy(alert['user']) for key, val, node in walk(user): node[key] = _tmpl(val).generate(**data).decode('utf-8') user = json.dumps(user, ensure_ascii=True, separators=(',', ':')) headers['X-Gramex-User'] = tornado.web.create_signed_value( info.app.settings['cookie_secret'], 'user', user) if 'markdown' in mail: mail['html'] = _markdown_convert(mail.pop('markdown')) if 'images' in alert: mail['images'] = {} for cid, val in alert['images'].items(): urlpath = _tmpl(val).generate(**data).decode('utf-8') urldata = urlfetch(urlpath, info=True, headers=headers) if urldata['content_type'].startswith('image/'): mail['images'][cid] = urldata['name'] else: with io.open(urldata['name'], 'rb') as temp_file: bytestoread = 80 first_line = temp_file.read(bytestoread) # TODO: let admin know that the image was not processed app_log.error( 'alert: %s: %s: %d (%s) not an image: %s\n%r', name, cid, urldata['r'].status_code, urldata['content_type'], urlpath, first_line) if 'attachments' in alert: mail['attachments'] = [ urlfetch(_tmpl(v).generate(**data).decode('utf-8'), headers=headers) for v in alert['attachments'] ] return mail
def build_log_info(keys, *vars): ''' Creates a ``handler.method(vars)`` that returns a dictionary of computed values. ``keys`` defines what keys are returned in the dictionary. The values are computed using the formulas in the code. ''' # Define direct keys. These can be used as-is direct_vars = { 'name': 'handler.name', 'class': 'handler.__class__.__name__', 'time': 'round(time.time() * 1000, 0)', 'datetime': 'datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ")', 'method': 'handler.request.method', 'uri': 'handler.request.uri', 'ip': 'handler.request.remote_ip', 'status': 'handler.get_status()', 'duration': 'round(handler.request.request_time() * 1000, 0)', 'port': 'conf.app.listen.port', # TODO: get_content_size() is not available in RequestHandler # 'size': 'handler.get_content_size()', 'user': '******', 'session': 'handler.session.get("id", "")', 'error': 'getattr(handler, "_exception", "")', } # Define object keys for us as key.value. E.g. cookies.sid, user.email, etc object_vars = { 'args': 'handler.get_argument("{val}", "")', 'request': 'getattr(handler.request, "{val}", "")', 'headers': 'handler.request.headers.get("{val}", "")', 'cookies': 'handler.request.cookies["{val}"].value ' + 'if "{val}" in handler.request.cookies else ""', 'user': '******', 'env': 'os.environ.get("{val}", "")', } vals = [] for key in keys: if key in vars: vals.append('"{}": {},'.format(key, key)) continue if key in direct_vars: vals.append('"{}": {},'.format(key, direct_vars[key])) continue if '.' in key: prefix, value = key.split('.', 2) if prefix in object_vars: vals.append('"{}": {},'.format(key, object_vars[prefix].format(val=value))) continue app_log.error('Skipping unknown key %s', key) code = compile('def fn(handler, %s):\n\treturn {%s}' % (', '.join(vars), ' '.join(vals)), filename='log', mode='exec') context = {'os': os, 'time': time, 'datetime': datetime, 'conf': conf, 'AttrDict': AttrDict} # The code is constructed entirely by this function. Using exec is safe exec(code, context) # nosec return context['fn']
def mail(args, kwargs): # Get config file location default_dir = os.path.join(variables['GRAMEXDATA'], 'mail') _mkdir(default_dir) if 'conf' in kwargs: confpath = kwargs.conf elif os.path.exists('gramex.yaml'): confpath = os.path.abspath('gramex.yaml') else: confpath = os.path.join(default_dir, 'gramexmail.yaml') if not os.path.exists(confpath): if 'init' in kwargs: with io.open(confpath, 'w', encoding='utf-8') as handle: handle.write(default_mail_config.format(confpath=confpath)) app_log.info('Initialized %s', confpath) elif not args and not kwargs: app_log.error(show_usage('mail')) else: app_log.error('Missing config %s. Use --init to generate skeleton', confpath) return conf = PathConfig(confpath) if 'list' in kwargs: for key, alert in conf.get('alert', {}).items(): to = alert.get('to', '') if isinstance(to, list): to = ', '.join(to) gramex.console('{:15}\t"{}" to {}'.format(key, alert.get('subject'), to)) return if 'init' in kwargs: app_log.error('Config already exists at %s', confpath) return if len(args) < 1: app_log.error(show_usage('mail')) return from gramex.services import email as setup_email, create_alert alert_conf = conf.get('alert', {}) email_conf = conf.get('email', {}) setup_email(email_conf) sys.path += os.path.dirname(confpath) for key in args: if key not in alert_conf: app_log.error('Missing key %s in %s', key, confpath) continue alert = create_alert(key, alert_conf[key]) alert()
def _start(self): ''' Check if capture is already running at ``url``. If not, start ``cmd`` and check again. Print logs from ``cmd``. ''' self.started = False script = self.engine.script try: # Check if capture.js is at the url specified app_log.info('Pinging %s at %s', script, self.url) r = requests.get(self.url, timeout=self.timeout) self._validate_server(r) self.started = True except requests.ReadTimeout: # If capture.js doesn't respond immediately, we haven't started app_log.error('url: %s timed out', self.url) except requests.ConnectionError: # Try starting the process again app_log.info('Starting %s via %s', script, self.cmd) self.close() # self.cmd is taken from the YAML configuration. Safe to run self.proc = Popen(shlex.split(self.cmd), stdout=PIPE, stderr=STDOUT) # nosec self.proc.poll() atexit.register(self.close) # TODO: what if readline() does not return quickly? line = self.proc.stdout.readline().strip() if not self.first_line_re.search(line): return app_log.error('cmd: %s invalid. Returned "%s"', self.cmd, line) app_log.info('Pinging %s at %s', script, self.url) try: r = requests.get(self.url, timeout=self.timeout) self._validate_server(r) pid = self.proc.pid app_log.info(line.decode('utf-8') + ' live (pid=%s)', pid) self.started = True # Keep logging capture.js output until proc is killed by another thread while hasattr(self, 'proc'): line = self.proc.stdout.readline().strip() if len(line) == 0: app_log.info('%s terminated: pid=%d', script, pid) self.started = False break # Capture won't print anything, unless there's a problem, or if debug is on. # So log it at warning level not info. app_log.warning(line.decode('utf-8')) except Exception: app_log.exception('Ran %s. But %s not at %s', self.cmd, script, self.url) except Exception: app_log.exception('Cannot start Capture')
def sass(handler, template=uicomponents_path): ''' Return a bootstrap theme based on the custom SASS variables provided. ''' args = dict(variables.get('ui-bootstrap', {})) args.update({key: handler.get_arg(key) for key in handler.args}) args = {key: val for key, val in args.items() if val} # Set default args config = gramex.cache.open(config_file) merge(args, config.get('defaults'), mode='setdefault') cache_key = {'template': template, 'args': args} cache_key = json.dumps(cache_key, sort_keys=True, ensure_ascii=True).encode('utf-8') cache_key = md5(cache_key).hexdigest()[:5] # Replace fonts from config file, if available google_fonts = set() for key in ('font-family-base', 'headings-font-family'): if key in args and args[key] in config['fonts']: fontinfo = config['fonts'][args[key]] args[key] = fontinfo['stack'] if 'google' in fontinfo: google_fonts.add(fontinfo['google']) # Cache based on the dict and config as template.<cache-key>.css base = os.path.splitext(os.path.basename(template))[0] + '.' + cache_key cache_path = join(cache_dir, base + '.css') if not os.path.exists(cache_path) or os.stat(template).st_mtime > os.stat( cache_path).st_mtime: # Create a SCSS file based on the args scss_path = join(cache_dir, base + '.scss') with io.open(scss_path, 'wb') as handle: result = gramex.cache.open(template, 'template').generate( variables=args, uicomponents_path=uicomponents_path.replace('\\', '/'), bootstrap_path=bootstrap_path.replace('\\', '/'), google_fonts=google_fonts, ) handle.write(result) # Run sass to generate the output options = ['--output-style', 'compressed'] proc = gramex.cache.Subprocess( ['node', sass_path, scss_path, cache_path] + options) out, err = yield proc.wait_for_exit() if proc.proc.returncode: app_log.error('node-sass error: %s', err) raise RuntimeError('Compilation failure') handler.set_header('Content-Type', 'text/css') raise Return(gramex.cache.open(cache_path, 'bin', mode='rb'))
def load(self, key, default=None): # Keys cannot contain / in HDF5 store. Escape it key = self._escape(key).replace('/', '\t') result = self.store.get(key, None) if result is None: return default result = result[()] try: return json.loads(result, object_pairs_hook=AttrDict, cls=CustomJSONDecoder) except ValueError: app_log.error('HDF5Store("%s").load("%s") is not JSON ("%r..."")', self.path, key, result) return default
def license(cmd, args): if len(cmd) == 0: gramex.console(gramex.license.EULA) if gramex.license.is_accepted(): gramex.console('License already ACCEPTED. Run "gramex license reject" to reject') else: gramex.console('License NOT YET accepted. Run "gramex license accept" to accept') elif cmd[0] == 'accept': gramex.license.accept(force=True) elif cmd[0] == 'reject': gramex.license.reject() else: app_log.error('Invalid command license %s', cmd[0])
def setup(args, kwargs): for target in args: run_setup(target) return if 'all' in kwargs: root = os.path.join(variables['GRAMEXPATH'], 'apps') for filename in os.listdir(root): target = os.path.join(root, filename) # Only run setup on directories. Ignore __pycache__, etc if os.path.isdir(target) and not filename.startswith('_'): run_setup(target) return app_log.error(show_usage('setup'))
def commandline(args=None): ''' usage: slidesense [config.yaml] [url-name] [--source=...] [--target=...] [--data=...] Generates target PPTX from a source PPTX, applying rules in config file and opens it. If no config file is specified, uses `gramex.yaml` in the current directory. The config file can have a pptgen configuration like {source: ..., target: ..., rules: ...} or be a `gramex.yaml` with `url: {url-name: {handler: PPTXHandler, kwargs: {source: ...}}}` Rules are picked up from the first PPTXHandler URL that matches `url-name`, or the first PPTXHandler in `gramex.yaml`. --source=... overrides source PPTX path in config file --target=... overrides target PPTX path in config file (defaults to output.pptx) --data=... overrides data file in config path --no-open don't open target PPTX after generating it ''' args = gramex.parse_command_line(sys.argv[1:] if args is None else args) if 'help' in args or (not args['_'] and not os.path.exists('gramex.yaml')): return gramex.console(dedent(commandline.__doc__).strip()) config_file, *urls = args.pop('_') or ['gramex.yaml'] conf = gramex.cache.open(config_file, 'config') if 'url' in conf: for key, spec in conf.url.items(): if spec.handler == 'PPTXHandler': if not urls or any(url in key for url in urls): rules = spec.kwargs break else: return app_log.error( f'No PPTXHandler matched in file: {config_file}') elif any(key in conf for key in ('source', 'target', 'data', 'rules')): rules = conf else: return app_log.error(f'No rules found in file: {config_file}') gramex.config.merge(rules, args) rules.setdefault('target', 'output.pptx') rules.setdefault('mode', 'expr') # Allow importing python files in current directory sys.path.append('.') # Generate output gramex.pptgen2.pptgen(**rules) # If --no-open is specified, or the OS doesn't have startfile (e.g. Linux), stop here. # Otherwise, open the output PPTX created if not rules.get('no-open', False) and hasattr(os, 'startfile'): # os.startfile() is safe since the target is an explicit file we've created os.startfile(rules['target']) # nosec
def js(self, code=None, path=None, **kwargs): if self.conn is None: try: self.conn = yield websocket_connect( self.url, connect_timeout=self.timeout) except OSError as exc: import errno if exc.errno != errno.ECONNREFUSED: raise # TODO: node_path self.proc = yield daemon( [which('node'), self._path, '--port=%s' % self.port], first_line=re.compile('pynode: 1.\d+.\d+ port: %s' % self.port), cwd=self.cwd, ) self.conn = yield websocket_connect( self.url, connect_timeout=self.timeout) # code= takes preference over path= if code is not None: kwargs['code'] = code elif path is not None: kwargs['path'] = path # Send the commands. If node has died, clear the connection. try: yield self.conn.write_message(json.dumps(kwargs)) except WebSocketClosedError: self.conn = None raise # Receive the response. # Note: read_message() cannot be called again while a request is running. # (Yes, that's odd. Maybe Anand is missing something.) # So wait until the read_future is cleared. while self.conn.read_future is not None: yield sleep(self._delay) msg = yield self.conn.read_message() # If node has died, clear the connection to restart it. if msg is None: self.conn = None raise WebSocketClosedError() # Parse the result as JSON. Log errors if any result = json.loads(msg) if result['error']: app_log.error(result['error']['stack']) raise Return(result)
def _error_fn(cls, error_code, error_config): template_kwargs = {} if 'autoescape' in error_config: if not error_config['autoescape']: template_kwargs['autoescape'] = None else: app_log.error('url:%s.error.%d.autoescape can only be false', cls.name, error_code) if 'whitespace' in error_config: template_kwargs['whitespace'] = error_config['whitespace'] def error(*args, **kwargs): tmpl = gramex.cache.open(error_config['path'], 'template', **template_kwargs) return tmpl.generate(*args, **kwargs) return error
def fetch_tweets(self, tweet_params): oauth = oauth1.Client( client_key=self.params['key'], client_secret=self.params['secret'], resource_owner_key=self.params['access_key'], resource_owner_secret=self.params['access_secret']) headers = { 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'Gramex', } url, headers, data = oauth.sign( self.url, 'POST', body=urlencode(tweet_params), headers=headers) self.req = tornado.httpclient.HTTPRequest( method='POST', url=url, body=data, headers=headers, request_timeout=864000, # Keep request alive for 10 days streaming_callback=self._stream, header_callback=self.header_callback) try: self.headers = None self.client.fetch(self.req) self.delay = 0 except tornado.httpclient.HTTPError as e: # HTTPError is raised for non-200 HTTP status codes. # For rate limiting, start with 1 minute and double each attempt if e.code in {RATE_LIMITED, TOO_MANY_REQUESTS}: self.delay = self.delay * 2 if self.delay else 60 app_log.error('TwitterStream HTTP %d (rate limited): %s. Retry: %ss', e.code, e.response, self.delay) # For Tornado timeout errors, reconnect immediately elif e.code == CLIENT_TIMEOUT: self.delay = 0 app_log.error('TwitterStream HTTP %d (timeout): %s. Retry: %ss', e.code, e.response, self.delay) # For server errors, start with 5 seconds and double until 320 seconds elif INTERNAL_SERVER_ERROR <= e.code <= GATEWAY_TIMEOUT: self.delay = min(320, self.delay * 2 if self.delay else 1) # noqa: 320 seconds app_log.error('TwitterStream HTTP %d: %s. Retry: %ss', e.code, e.response, self.delay) # For client errors (e.g. wrong params), disable connection else: self.delay, self.enabled = 5, False app_log.error('TwitterStream HTTP %d: %s. Disabling', e.code, e.response) except Exception as e: # Other errors are possible, such as IOError. # Increase the delay in reconnects by 250ms each attempt, up to 16 seconds. self.delay = min(16, self.delay + 0.25) # noqa: 16 seconds, 0.25 seconds app_log.error('TwitterStream exception %s. Retry: %ss', e, self.delay)
def init(cmd, args): '''Create Gramex scaffolding files.''' if len(cmd) > 1: app_log.error(show_usage('init')) return args.setdefault('target', os.getcwd()) app_log.info('Initializing Gramex project at %s', args.target) data = { 'appname': os.path.basename(args.target), 'author': _check_output('git config user.name', default='Author'), 'email': _check_output('git config user.email', default='*****@*****.**'), 'date': datetime.datetime.today().strftime('%Y-%m-%d'), 'version': gramex.__version__, } # Ensure that appname is a valid Python module name appname = re.sub(r'[^a-z0-9_]+', '_', data['appname'].lower()) if appname[0] not in string.ascii_lowercase: appname = 'app' + appname data['appname'] = appname # Copy all directories & files (as templates) source_dir = os.path.join(variables['GRAMEXPATH'], 'apps', 'init') for root, dirs, files in os.walk(source_dir): for name in dirs + files: source = os.path.join(root, name) relpath = os.path.relpath(root, start=source_dir) target = os.path.join(args.target, relpath, name.replace('appname', appname)) _copy(source, target, template_data=data) for empty_dir in ('img', 'data'): _mkdir(os.path.join(args.target, 'assets', empty_dir)) # Copy error files as-is (not as templates) error_dir = os.path.join(args.target, 'error') _mkdir(error_dir) for source in glob( os.path.join(variables['GRAMEXPATH'], 'handlers', '?0?.html')): target = os.path.join(error_dir, os.path.basename(source)) _copy(source, target) # Create a git repo if none exists. # But if git is not installed, do not stop. Continue with the rest. if not os.path.exists(os.path.join(args.target, '.git')): try: _run_console('git init') except OSError: pass run_setup(args.target)
def _google_translate(q, source, target, key): import requests params = {'q': q, 'target': target, 'key': key} if source: params['source'] = source try: r = requests.post( 'https://translation.googleapis.com/language/translate/v2', data=params) except requests.RequestException: return app_log.exception('Cannot connect to Google Translate') response = r.json() if 'error' in response: return app_log.error('Google Translate API error: %s', response['error']) return { 'q': q, 't': [t['translatedText'] for t in response['data']['translations']], 'source': [ t.get('detectedSourceLanguage', params.get('source', None)) for t in response['data']['translations'] ], 'target': [target] * len(q), }
def __init__(self, flush=None, purge=None, purge_keys=None, **kwargs): '''Initialise the KeyStore at path''' self.store = {} if callable(purge_keys): self.purge_keys = purge_keys elif purge_keys is not None: app_log.error( 'KeyStore: purge_keys=%r invalid. Must be function(dict)', purge_keys) # Periodically flush and purge buffers if flush is not None: PeriodicCallback(self.flush, callback_time=flush * 1000).start() if purge is not None: PeriodicCallback(self.purge, callback_time=purge * 1000).start() # Call close() when Python gracefully exits atexit.register(self.close)