def __init__(self, path, keys=None, **kwargs):
if keys is None:
keys = {}
for cat in ('file', 'delete', 'save'):
keys.setdefault(cat, [cat])
if not isinstance(keys[cat], list):
if isinstance(keys[cat], (str, bytes)):
keys[cat] = [keys[cat]]
else:
app_log.error('FileUpload: cat: %r must be a list or str',
keys[cat])
self.keys = keys
self.path = os.path.abspath(path)
if not os.path.exists(self.path):
os.makedirs(self.path)
# store default: sqlite .meta.db
store_kwargs = kwargs.get('store', {
'type': 'sqlite',
'path': os.path.join(self.path, '.meta.db')
})
if self.path not in self.stores:
self.stores[self.path] = get_store(**store_kwargs)
self.store = self.stores[self.path]
old_store_path = os.path.abspath(os.path.join(self.path, '.meta.h5'))
store_path = os.path.abspath(getattr(self.store, 'path', None))
# migration: if type is not hdf5 but .meta.h5 exists, update store and remove
if (os.path.exists(old_store_path) and store_path != old_store_path):
self._migrate_h5(old_store_path)
if 'file' not in keys:
keys['file'] = ['file']
self.keys['file'] = keys['file'] if isinstance(
keys['file'], list) else [keys['file']]
def setup(cls,
path,
keys=None,
if_exists='unique',
transform=None,
methods=[],
**kwargs):
super(UploadHandler, cls).setup(**kwargs)
cls.if_exists = if_exists
# FileUpload uses the store= from **kwargs and ignores the rest
cls.uploader = FileUpload(path, keys=keys, **kwargs)
# methods=['get'] will show all file into as JSON on GET
if not isinstance(methods, list):
methods = [methods]
methods = {method.lower() for method in methods}
if 'get' in methods:
cls.get = cls.fileinfo
cls.transform = []
if transform is not None:
if isinstance(transform, dict) and 'function' in transform:
cls.transform.append(
build_transform(transform,
vars=AttrDict((('content', None),
('handler', None))),
filename='url:%s' % cls.name))
else:
app_log.error(
'UploadHandler %s: no function: in transform: %r',
cls.name, transform)
def run_command(config):
'''
Run config.cmd. If the command has a TARGET, replace it with config.target.
Else append config.target as an argument.
'''
appcmd = config.cmd
# Split the command into an array of words
if isinstance(appcmd, six.string_types):
appcmd = shlex.split(appcmd)
# If the app is a Cygwin app, TARGET should be a Cygwin path too.
target = config.target
cygcheck, cygpath, kwargs = which('cygcheck'), which('cygpath'), {'universal_newlines': True}
if cygcheck is not None and cygpath is not None:
app_path = check_output([cygpath, '-au', which(appcmd[0])], **kwargs).strip() # nosec
is_cygwin_app = check_output([cygcheck, '-f', app_path], **kwargs).strip() # nosec
if is_cygwin_app:
target = check_output([cygpath, '-au', target], **kwargs).strip() # n osec
# Replace TARGET with the actual target
if 'TARGET' in appcmd:
appcmd = [target if arg == 'TARGET' else arg for arg in appcmd]
else:
appcmd.append(target)
app_log.info('Running %s', ' '.join(appcmd))
if not safe_rmtree(config.target):
app_log.error('Cannot delete target %s. Aborting installation', config.target)
return
proc = Popen(appcmd, bufsize=-1, stdout=sys.stdout, stderr=sys.stderr, **kwargs) # nosec
proc.communicate()
return proc.returncode
def run(args, kwargs):
if len(args) < 1:
app_log.error(show_usage('run'))
return
if len(args) > 1:
app_log.error('Can only run one app. Ignoring %s', ', '.join(args[1:]))
appname = args.pop(0)
app_config = get_app_config(appname, kwargs)
target = app_config.target
if 'dir' in app_config:
target = os.path.join(target, app_config.dir)
if os.path.isdir(target):
os.chdir(target)
gramex.paths['base'] = Path('.')
# If we run with updated parameters, save for next run under the .run config
run_config = app_config.setdefault('run', {})
for key, val in kwargs.items():
if key not in app_keys:
run_config[key] = app_config.pop(key)
save_user_config(appname, app_config)
# Tell the user what configs are used
cline = ' '.join('--%s=%s' % arg
for arg in flatten_config(app_config.get('run', {})))
app_log.info('Gramex %s | %s %s | %s | Python %s',
gramex.__version__, appname, cline, os.getcwd(),
sys.version.replace('\n', ' '))
gramex.init(args=AttrDict(app=app_config['run']))
elif appname in apps_config['user']:
# The user configuration has a wrong path. Inform user
app_log.error('%s: no directory %s', appname, app_config.target)
app_log.error('Run "gramex uninstall %s" and try again.', appname)
else:
app_log.error('%s: no directory %s', appname, app_config.target)
def setup_auth(cls, auth):
# auth: if there's no auth: in handler, default to app.auth
if auth is None:
auth = conf.app.get('auth')
# Treat True as an empty dict, i.e. auth: {}
if auth is True:
auth = AttrDict()
# Set up the auth
if isinstance(auth, dict):
cls._auth = auth
cls._on_init_methods.append(cls.authorize)
cls.permissions = []
# Add check for condition
if auth.get('condition'):
cls.permissions.append(
build_transform(auth['condition'], vars=AttrDict(handler=None),
filename='url:%s.auth.permission' % cls.name))
# Add check for membership
memberships = auth.get('membership', [])
if not isinstance(memberships, list):
memberships = [memberships]
if len(memberships):
cls.permissions.append(check_membership(memberships))
elif auth:
app_log.error('url:%s.auth is not a dict', cls.name)
def _ensure_remove(function, path, exc_info):
'''onerror callback for rmtree that tries hard to delete files'''
if issubclass(exc_info[0], WindowsError):
import winerror
# Delete read-only files
# https://bugs.python.org/issue19643
# https://bugs.python.org/msg218021
if exc_info[1].winerror == winerror.ERROR_ACCESS_DENIED:
os.chmod(path, stat.S_IWRITE)
return os.remove(path)
# Delay delete a bit if directory is used by another process.
# Typically happens on uninstall immediately after bower / npm / git
# (e.g. during testing.)
elif exc_info[1].winerror == winerror.ERROR_SHARING_VIOLATION:
delays = [
0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0
]
for delay in delays:
time.sleep(delay)
try:
return os.remove(path)
except WindowsError:
pass
# npm creates windows shortcuts that shutil.rmtree cannot delete.
# os.listdir failes with a PATH_NOT_FOUND. Delete these and try again
elif function == os.listdir and exc_info[
1].winerror == winerror.ERROR_PATH_NOT_FOUND:
app_log.error('Cannot delete %s', path)
from win32com.shell import shell, shellcon
options = shellcon.FOF_NOCONFIRMATION | shellcon.FOF_NOERRORUI
code, err = shell.SHFileOperation(
(0, shellcon.FO_DELETE, path, None, options))
if code == 0:
raise TryAgainError()
raise exc_info[1]
def r(code=None, path=None, rel=True, conda=True, convert=True,
repo='https://cran.microsoft.com/', **kwargs):
'''
Runs the R script and returns the result.
:arg str code: R code to execute.
:arg str path: R script path. Cannot be used if code is specified
:arg bool rel: True treats path as relative to the caller function's file
:arg bool conda: True overrides R_HOME to use the Conda R
:arg bool convert: True converts R objects to Pandas and vice versa
:arg str repo: CRAN repo URL
All other keyword arguments as passed as parameters
'''
# Use Conda R if possible
if conda:
r_home = _conda_r_home()
if r_home:
os.environ['R_HOME'] = r_home
# Import the global R session
try:
from rpy2.robjects import r, pandas2ri, globalenv
except ImportError:
app_log.error('rpy2 not installed. Run "conda install rpy2"')
raise
except RuntimeError:
app_log.error('Cannot find R. Set R_HOME env variable')
raise
# Set a repo so that install.packages() need not ask for one
r('local({r <- getOption("repos"); r["CRAN"] <- "%s"; options(repos = r)})' % repo)
# Activate or de-activate automatic conversion
# https://pandas.pydata.org/pandas-docs/version/0.22.0/r_interface.html
if convert:
pandas2ri.activate()
else:
pandas2ri.deactivate()
# Pass all other kwargs as global environment variables
for key, val in kwargs.items():
globalenv[key] = val
if code and path:
raise RuntimeError('Use r(code=) or r(path=...), not both')
if path:
# if rel=True, load path relative to parent directory
if rel:
stack = inspect.getouterframes(inspect.currentframe(), 2)
folder = os.path.dirname(os.path.abspath(stack[1][1]))
path = os.path.join(folder, path)
result = r.source(path, chdir=True)
# source() returns a withVisible: $value and $visible. Use only the first
result = result[0]
else:
result = r(code)
return result
def _run_console(cmd, **kwargs):
'''Run cmd and pipe output to console. Log and raise error if cmd is not found'''
cmd = shlex.split(cmd)
try:
proc = Popen(cmd, bufsize=-1, universal_newlines=True, **kwargs)
except OSError:
app_log.error('Cannot find command: %s', cmd[0])
raise
proc.communicate()
def setup(cls, **kwargs):
super(FormHandler, cls).setup(**kwargs)
conf_kwargs = merge(AttrDict(cls.conf.kwargs),
objectpath(gramex_conf, 'handlers.FormHandler', {}),
'setdefault')
cls.headers = conf_kwargs.pop('headers', {})
# Top level formats: key is special. Don't treat it as data
cls.formats = conf_kwargs.pop('formats', {})
default_config = conf_kwargs.pop('default', None)
# Remove other known special keys from dataset configuration
cls.clear_special_keys(conf_kwargs)
# If top level has url: then data spec is at top level. Else it's a set of sub-keys
if 'url' in conf_kwargs:
cls.datasets = {'data': conf_kwargs}
cls.single = True
else:
if 'modify' in conf_kwargs:
cls.modify_all = staticmethod(build_transform(
conf={'function': conf_kwargs.pop('modify', None)},
vars=cls.function_vars['modify'],
filename='%s.%s' % (cls.name, 'modify'), iter=False))
cls.datasets = conf_kwargs
cls.single = False
# Apply defaults to each key
if isinstance(default_config, dict):
for key in cls.datasets:
config = cls.datasets[key].get('default', {})
cls.datasets[key]['default'] = merge(config, default_config, mode='setdefault')
# Ensure that each dataset is a dict with a url: key at least
for key, dataset in list(cls.datasets.items()):
if not isinstance(dataset, dict):
app_log.error('%s: %s: must be a dict, not %r' % (cls.name, key, dataset))
del cls.datasets[key]
elif 'url' not in dataset:
app_log.error('%s: %s: does not have a url: key' % (cls.name, key))
del cls.datasets[key]
# Ensure that id: is a list -- if it exists
if 'id' in dataset and not isinstance(dataset['id'], list):
dataset['id'] = [dataset['id']]
# Convert function: into a data = transform(data) function
conf = {
'function': dataset.pop('function', None),
'args': dataset.pop('args', None),
'kwargs': dataset.pop('kwargs', None)
}
if conf['function'] is not None:
fn_name = '%s.%s.transform' % (cls.name, key)
dataset['transform'] = build_transform(
conf, vars={'data': None, 'handler': None}, filename=fn_name, iter=False)
# Convert modify: and prepare: into a data = modify(data) function
for fn, fn_vars in cls.function_vars.items():
if fn in dataset:
dataset[fn] = build_transform(
conf={'function': dataset[fn]},
vars=fn_vars,
filename='%s.%s.%s' % (cls.name, key, fn), iter=False)
def redirect_method(handler):
next_uri = method(handler)
if next_uri is not None:
target = urlparse(next_uri)
if not target.scheme and not target.netloc:
return next_uri
req = handler.request
if req.protocol == target.scheme and req.host == target.netloc:
return next_uri
app_log.error('Not redirecting to external url: %s', next_uri)
def transforms(self, content):
for transform in self.transform:
for value in transform(content, self):
if isinstance(value, dict):
content = value
elif value is not None:
app_log.error(
'UploadHandler %s: transform returned %r, not dict',
self.name, value)
return content
def _run_console(cmd, **kwargs):
'''Run cmd and pipe output to console (sys.stdout / sys.stderr)'''
cmd = shlex.split(cmd)
try:
proc = Popen(cmd, bufsize=-1, stdout=sys.stdout, stderr=sys.stderr,
universal_newlines=True, **kwargs)
except OSError:
app_log.error('Cannot find command: %s', cmd[0])
raise
proc.communicate()
def service(args, kwargs):
try:
import gramex.winservice
except ImportError:
app_log.error('Unable to load winservice. Is this Windows?')
raise
if len(args) < 1:
app_log.error(show_usage('service'))
return
gramex.winservice.GramexService.setup(args, **kwargs)
def init(args, kwargs):
'''Create Gramex scaffolding files.'''
if len(args) > 1:
app_log.error(show_usage('init'))
return
kwargs.setdefault('target', os.getcwd())
app_log.info('Initializing Gramex project at %s', kwargs.target)
data = {
'appname': os.path.basename(kwargs.target),
'author': _check_output('git config user.name', default='Author'),
'email': _check_output('git config user.email',
default='*****@*****.**'),
'date': datetime.datetime.today().strftime('%Y-%m-%d'),
'version': gramex.__version__,
}
# Ensure that appname is a valid Python module name
appname = slug.module(data['appname'])
if appname[0] not in string.ascii_lowercase:
appname = 'app' + appname
data['appname'] = appname
# Create a git repo. But if git fails, do not stop. Continue with the rest.
try:
_run_console('git init')
except OSError:
pass
# Install Git LFS if available. Set git_lfs=None if it fails, so .gitignore ignores assets/**
data['git_lfs'] = which('git-lfs')
if data['git_lfs']:
try:
_run_console('git lfs install')
_run_console('git lfs track "assets/**"')
except OSError:
data['git_lfs'] = None
# Copy all directories & files (as templates)
source_dir = os.path.join(variables['GRAMEXPATH'], 'apps', 'init')
for root, dirs, files in os.walk(source_dir):
for name in dirs + files:
source = os.path.join(root, name)
relpath = os.path.relpath(root, start=source_dir)
target = os.path.join(kwargs.target, relpath,
name.replace('appname', appname))
_copy(source, target, template_data=data)
for empty_dir in ('img', 'data'):
_mkdir(os.path.join(kwargs.target, 'assets', empty_dir))
# Copy error files as-is (not as templates)
error_dir = os.path.join(kwargs.target, 'error')
_mkdir(error_dir)
for source in glob(
os.path.join(variables['GRAMEXPATH'], 'handlers', '?0?.html')):
target = os.path.join(error_dir, os.path.basename(source))
_copy(source, target)
run_setup(kwargs.target)
def load(self, key, default=None):
result = self.store.get(key)
if result is None:
return default
try:
return json.loads(
result, object_pairs_hook=AttrDict, cls=CustomJSONDecoder)
except ValueError:
app_log.error('RedisStore("%s").load("%s") is not JSON ("%r..."")',
self.store, key, result)
return default
def create_mail(data):
'''
Return kwargs that can be passed to a mailer.mail
'''
mail = {}
for key in ['bodyfile', 'htmlfile', 'markdownfile']:
target = key.replace('file', '')
if key in alert and target not in alert:
path = _tmpl(alert[key]).generate(**data).decode('utf-8')
tmpl = gramex.cache.open(path, 'template')
mail[target] = tmpl.generate(**data).decode('utf-8')
for key in addr_fields + ['subject', 'body', 'html', 'markdown']:
if key not in alert:
continue
if isinstance(alert[key], list):
mail[key] = [
_tmpl(v).generate(**data).decode('utf-8')
for v in alert[key]
]
else:
mail[key] = _tmpl(alert[key]).generate(**data).decode('utf-8')
headers = {}
# user: {id: ...} creates an X-Gramex-User header to mimic the user
if 'user' in alert:
user = deepcopy(alert['user'])
for key, val, node in walk(user):
node[key] = _tmpl(val).generate(**data).decode('utf-8')
user = json.dumps(user, ensure_ascii=True, separators=(',', ':'))
headers['X-Gramex-User'] = tornado.web.create_signed_value(
info.app.settings['cookie_secret'], 'user', user)
if 'markdown' in mail:
mail['html'] = _markdown_convert(mail.pop('markdown'))
if 'images' in alert:
mail['images'] = {}
for cid, val in alert['images'].items():
urlpath = _tmpl(val).generate(**data).decode('utf-8')
urldata = urlfetch(urlpath, info=True, headers=headers)
if urldata['content_type'].startswith('image/'):
mail['images'][cid] = urldata['name']
else:
with io.open(urldata['name'], 'rb') as temp_file:
bytestoread = 80
first_line = temp_file.read(bytestoread)
# TODO: let admin know that the image was not processed
app_log.error(
'alert: %s: %s: %d (%s) not an image: %s\n%r', name,
cid, urldata['r'].status_code, urldata['content_type'],
urlpath, first_line)
if 'attachments' in alert:
mail['attachments'] = [
urlfetch(_tmpl(v).generate(**data).decode('utf-8'),
headers=headers) for v in alert['attachments']
]
return mail
def build_log_info(keys, *vars):
'''
Creates a ``handler.method(vars)`` that returns a dictionary of computed
values. ``keys`` defines what keys are returned in the dictionary. The values
are computed using the formulas in the code.
'''
# Define direct keys. These can be used as-is
direct_vars = {
'name': 'handler.name',
'class': 'handler.__class__.__name__',
'time': 'round(time.time() * 1000, 0)',
'datetime': 'datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ")',
'method': 'handler.request.method',
'uri': 'handler.request.uri',
'ip': 'handler.request.remote_ip',
'status': 'handler.get_status()',
'duration': 'round(handler.request.request_time() * 1000, 0)',
'port': 'conf.app.listen.port',
# TODO: get_content_size() is not available in RequestHandler
# 'size': 'handler.get_content_size()',
'user': '******',
'session': 'handler.session.get("id", "")',
'error': 'getattr(handler, "_exception", "")',
}
# Define object keys for us as key.value. E.g. cookies.sid, user.email, etc
object_vars = {
'args': 'handler.get_argument("{val}", "")',
'request': 'getattr(handler.request, "{val}", "")',
'headers': 'handler.request.headers.get("{val}", "")',
'cookies': 'handler.request.cookies["{val}"].value ' +
'if "{val}" in handler.request.cookies else ""',
'user': '******',
'env': 'os.environ.get("{val}", "")',
}
vals = []
for key in keys:
if key in vars:
vals.append('"{}": {},'.format(key, key))
continue
if key in direct_vars:
vals.append('"{}": {},'.format(key, direct_vars[key]))
continue
if '.' in key:
prefix, value = key.split('.', 2)
if prefix in object_vars:
vals.append('"{}": {},'.format(key, object_vars[prefix].format(val=value)))
continue
app_log.error('Skipping unknown key %s', key)
code = compile('def fn(handler, %s):\n\treturn {%s}' % (', '.join(vars), ' '.join(vals)),
filename='log', mode='exec')
context = {'os': os, 'time': time, 'datetime': datetime, 'conf': conf, 'AttrDict': AttrDict}
# The code is constructed entirely by this function. Using exec is safe
exec(code, context) # nosec
return context['fn']
def mail(args, kwargs):
# Get config file location
default_dir = os.path.join(variables['GRAMEXDATA'], 'mail')
_mkdir(default_dir)
if 'conf' in kwargs:
confpath = kwargs.conf
elif os.path.exists('gramex.yaml'):
confpath = os.path.abspath('gramex.yaml')
else:
confpath = os.path.join(default_dir, 'gramexmail.yaml')
if not os.path.exists(confpath):
if 'init' in kwargs:
with io.open(confpath, 'w', encoding='utf-8') as handle:
handle.write(default_mail_config.format(confpath=confpath))
app_log.info('Initialized %s', confpath)
elif not args and not kwargs:
app_log.error(show_usage('mail'))
else:
app_log.error('Missing config %s. Use --init to generate skeleton',
confpath)
return
conf = PathConfig(confpath)
if 'list' in kwargs:
for key, alert in conf.get('alert', {}).items():
to = alert.get('to', '')
if isinstance(to, list):
to = ', '.join(to)
gramex.console('{:15}\t"{}" to {}'.format(key,
alert.get('subject'),
to))
return
if 'init' in kwargs:
app_log.error('Config already exists at %s', confpath)
return
if len(args) < 1:
app_log.error(show_usage('mail'))
return
from gramex.services import email as setup_email, create_alert
alert_conf = conf.get('alert', {})
email_conf = conf.get('email', {})
setup_email(email_conf)
sys.path += os.path.dirname(confpath)
for key in args:
if key not in alert_conf:
app_log.error('Missing key %s in %s', key, confpath)
continue
alert = create_alert(key, alert_conf[key])
alert()
def _start(self):
'''
Check if capture is already running at ``url``. If not, start ``cmd``
and check again. Print logs from ``cmd``.
'''
self.started = False
script = self.engine.script
try:
# Check if capture.js is at the url specified
app_log.info('Pinging %s at %s', script, self.url)
r = requests.get(self.url, timeout=self.timeout)
self._validate_server(r)
self.started = True
except requests.ReadTimeout:
# If capture.js doesn't respond immediately, we haven't started
app_log.error('url: %s timed out', self.url)
except requests.ConnectionError:
# Try starting the process again
app_log.info('Starting %s via %s', script, self.cmd)
self.close()
# self.cmd is taken from the YAML configuration. Safe to run
self.proc = Popen(shlex.split(self.cmd),
stdout=PIPE,
stderr=STDOUT) # nosec
self.proc.poll()
atexit.register(self.close)
# TODO: what if readline() does not return quickly?
line = self.proc.stdout.readline().strip()
if not self.first_line_re.search(line):
return app_log.error('cmd: %s invalid. Returned "%s"',
self.cmd, line)
app_log.info('Pinging %s at %s', script, self.url)
try:
r = requests.get(self.url, timeout=self.timeout)
self._validate_server(r)
pid = self.proc.pid
app_log.info(line.decode('utf-8') + ' live (pid=%s)', pid)
self.started = True
# Keep logging capture.js output until proc is killed by another thread
while hasattr(self, 'proc'):
line = self.proc.stdout.readline().strip()
if len(line) == 0:
app_log.info('%s terminated: pid=%d', script, pid)
self.started = False
break
# Capture won't print anything, unless there's a problem, or if debug is on.
# So log it at warning level not info.
app_log.warning(line.decode('utf-8'))
except Exception:
app_log.exception('Ran %s. But %s not at %s', self.cmd, script,
self.url)
except Exception:
app_log.exception('Cannot start Capture')
def sass(handler, template=uicomponents_path):
'''
Return a bootstrap theme based on the custom SASS variables provided.
'''
args = dict(variables.get('ui-bootstrap', {}))
args.update({key: handler.get_arg(key) for key in handler.args})
args = {key: val for key, val in args.items() if val}
# Set default args
config = gramex.cache.open(config_file)
merge(args, config.get('defaults'), mode='setdefault')
cache_key = {'template': template, 'args': args}
cache_key = json.dumps(cache_key, sort_keys=True,
ensure_ascii=True).encode('utf-8')
cache_key = md5(cache_key).hexdigest()[:5]
# Replace fonts from config file, if available
google_fonts = set()
for key in ('font-family-base', 'headings-font-family'):
if key in args and args[key] in config['fonts']:
fontinfo = config['fonts'][args[key]]
args[key] = fontinfo['stack']
if 'google' in fontinfo:
google_fonts.add(fontinfo['google'])
# Cache based on the dict and config as template.<cache-key>.css
base = os.path.splitext(os.path.basename(template))[0] + '.' + cache_key
cache_path = join(cache_dir, base + '.css')
if not os.path.exists(cache_path) or os.stat(template).st_mtime > os.stat(
cache_path).st_mtime:
# Create a SCSS file based on the args
scss_path = join(cache_dir, base + '.scss')
with io.open(scss_path, 'wb') as handle:
result = gramex.cache.open(template, 'template').generate(
variables=args,
uicomponents_path=uicomponents_path.replace('\\', '/'),
bootstrap_path=bootstrap_path.replace('\\', '/'),
google_fonts=google_fonts,
)
handle.write(result)
# Run sass to generate the output
options = ['--output-style', 'compressed']
proc = gramex.cache.Subprocess(
['node', sass_path, scss_path, cache_path] + options)
out, err = yield proc.wait_for_exit()
if proc.proc.returncode:
app_log.error('node-sass error: %s', err)
raise RuntimeError('Compilation failure')
handler.set_header('Content-Type', 'text/css')
raise Return(gramex.cache.open(cache_path, 'bin', mode='rb'))
def load(self, key, default=None):
# Keys cannot contain / in HDF5 store. Escape it
key = self._escape(key).replace('/', '\t')
result = self.store.get(key, None)
if result is None:
return default
result = result[()]
try:
return json.loads(result, object_pairs_hook=AttrDict, cls=CustomJSONDecoder)
except ValueError:
app_log.error('HDF5Store("%s").load("%s") is not JSON ("%r..."")',
self.path, key, result)
return default
def license(cmd, args):
if len(cmd) == 0:
gramex.console(gramex.license.EULA)
if gramex.license.is_accepted():
gramex.console('License already ACCEPTED. Run "gramex license reject" to reject')
else:
gramex.console('License NOT YET accepted. Run "gramex license accept" to accept')
elif cmd[0] == 'accept':
gramex.license.accept(force=True)
elif cmd[0] == 'reject':
gramex.license.reject()
else:
app_log.error('Invalid command license %s', cmd[0])
def setup(args, kwargs):
for target in args:
run_setup(target)
return
if 'all' in kwargs:
root = os.path.join(variables['GRAMEXPATH'], 'apps')
for filename in os.listdir(root):
target = os.path.join(root, filename)
# Only run setup on directories. Ignore __pycache__, etc
if os.path.isdir(target) and not filename.startswith('_'):
run_setup(target)
return
app_log.error(show_usage('setup'))
def commandline(args=None):
'''
usage: slidesense [config.yaml] [url-name] [--source=...] [--target=...] [--data=...]
Generates target PPTX from a source PPTX, applying rules in config file and opens it.
If no config file is specified, uses `gramex.yaml` in the current directory.
The config file can have a pptgen configuration like {source: ..., target: ..., rules: ...}
or be a `gramex.yaml` with `url: {url-name: {handler: PPTXHandler, kwargs: {source: ...}}}`
Rules are picked up from the first PPTXHandler URL that matches `url-name`,
or the first PPTXHandler in `gramex.yaml`.
--source=... overrides source PPTX path in config file
--target=... overrides target PPTX path in config file (defaults to output.pptx)
--data=... overrides data file in config path
--no-open don't open target PPTX after generating it
'''
args = gramex.parse_command_line(sys.argv[1:] if args is None else args)
if 'help' in args or (not args['_'] and not os.path.exists('gramex.yaml')):
return gramex.console(dedent(commandline.__doc__).strip())
config_file, *urls = args.pop('_') or ['gramex.yaml']
conf = gramex.cache.open(config_file, 'config')
if 'url' in conf:
for key, spec in conf.url.items():
if spec.handler == 'PPTXHandler':
if not urls or any(url in key for url in urls):
rules = spec.kwargs
break
else:
return app_log.error(
f'No PPTXHandler matched in file: {config_file}')
elif any(key in conf for key in ('source', 'target', 'data', 'rules')):
rules = conf
else:
return app_log.error(f'No rules found in file: {config_file}')
gramex.config.merge(rules, args)
rules.setdefault('target', 'output.pptx')
rules.setdefault('mode', 'expr')
# Allow importing python files in current directory
sys.path.append('.')
# Generate output
gramex.pptgen2.pptgen(**rules)
# If --no-open is specified, or the OS doesn't have startfile (e.g. Linux), stop here.
# Otherwise, open the output PPTX created
if not rules.get('no-open', False) and hasattr(os, 'startfile'):
# os.startfile() is safe since the target is an explicit file we've created
os.startfile(rules['target']) # nosec
def js(self, code=None, path=None, **kwargs):
if self.conn is None:
try:
self.conn = yield websocket_connect(
self.url, connect_timeout=self.timeout)
except OSError as exc:
import errno
if exc.errno != errno.ECONNREFUSED:
raise
# TODO: node_path
self.proc = yield daemon(
[which('node'), self._path,
'--port=%s' % self.port],
first_line=re.compile('pynode: 1.\d+.\d+ port: %s' %
self.port),
cwd=self.cwd,
)
self.conn = yield websocket_connect(
self.url, connect_timeout=self.timeout)
# code= takes preference over path=
if code is not None:
kwargs['code'] = code
elif path is not None:
kwargs['path'] = path
# Send the commands. If node has died, clear the connection.
try:
yield self.conn.write_message(json.dumps(kwargs))
except WebSocketClosedError:
self.conn = None
raise
# Receive the response.
# Note: read_message() cannot be called again while a request is running.
# (Yes, that's odd. Maybe Anand is missing something.)
# So wait until the read_future is cleared.
while self.conn.read_future is not None:
yield sleep(self._delay)
msg = yield self.conn.read_message()
# If node has died, clear the connection to restart it.
if msg is None:
self.conn = None
raise WebSocketClosedError()
# Parse the result as JSON. Log errors if any
result = json.loads(msg)
if result['error']:
app_log.error(result['error']['stack'])
raise Return(result)
def _error_fn(cls, error_code, error_config):
template_kwargs = {}
if 'autoescape' in error_config:
if not error_config['autoescape']:
template_kwargs['autoescape'] = None
else:
app_log.error('url:%s.error.%d.autoescape can only be false', cls.name, error_code)
if 'whitespace' in error_config:
template_kwargs['whitespace'] = error_config['whitespace']
def error(*args, **kwargs):
tmpl = gramex.cache.open(error_config['path'], 'template', **template_kwargs)
return tmpl.generate(*args, **kwargs)
return error
def fetch_tweets(self, tweet_params):
oauth = oauth1.Client(
client_key=self.params['key'],
client_secret=self.params['secret'],
resource_owner_key=self.params['access_key'],
resource_owner_secret=self.params['access_secret'])
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'User-Agent': 'Gramex',
}
url, headers, data = oauth.sign(
self.url, 'POST', body=urlencode(tweet_params), headers=headers)
self.req = tornado.httpclient.HTTPRequest(
method='POST', url=url, body=data, headers=headers,
request_timeout=864000, # Keep request alive for 10 days
streaming_callback=self._stream,
header_callback=self.header_callback)
try:
self.headers = None
self.client.fetch(self.req)
self.delay = 0
except tornado.httpclient.HTTPError as e:
# HTTPError is raised for non-200 HTTP status codes.
# For rate limiting, start with 1 minute and double each attempt
if e.code in {RATE_LIMITED, TOO_MANY_REQUESTS}:
self.delay = self.delay * 2 if self.delay else 60
app_log.error('TwitterStream HTTP %d (rate limited): %s. Retry: %ss',
e.code, e.response, self.delay)
# For Tornado timeout errors, reconnect immediately
elif e.code == CLIENT_TIMEOUT:
self.delay = 0
app_log.error('TwitterStream HTTP %d (timeout): %s. Retry: %ss',
e.code, e.response, self.delay)
# For server errors, start with 5 seconds and double until 320 seconds
elif INTERNAL_SERVER_ERROR <= e.code <= GATEWAY_TIMEOUT:
self.delay = min(320, self.delay * 2 if self.delay else 1) # noqa: 320 seconds
app_log.error('TwitterStream HTTP %d: %s. Retry: %ss',
e.code, e.response, self.delay)
# For client errors (e.g. wrong params), disable connection
else:
self.delay, self.enabled = 5, False
app_log.error('TwitterStream HTTP %d: %s. Disabling', e.code, e.response)
except Exception as e:
# Other errors are possible, such as IOError.
# Increase the delay in reconnects by 250ms each attempt, up to 16 seconds.
self.delay = min(16, self.delay + 0.25) # noqa: 16 seconds, 0.25 seconds
app_log.error('TwitterStream exception %s. Retry: %ss', e, self.delay)
def init(cmd, args):
'''Create Gramex scaffolding files.'''
if len(cmd) > 1:
app_log.error(show_usage('init'))
return
args.setdefault('target', os.getcwd())
app_log.info('Initializing Gramex project at %s', args.target)
data = {
'appname': os.path.basename(args.target),
'author': _check_output('git config user.name', default='Author'),
'email': _check_output('git config user.email',
default='*****@*****.**'),
'date': datetime.datetime.today().strftime('%Y-%m-%d'),
'version': gramex.__version__,
}
# Ensure that appname is a valid Python module name
appname = re.sub(r'[^a-z0-9_]+', '_', data['appname'].lower())
if appname[0] not in string.ascii_lowercase:
appname = 'app' + appname
data['appname'] = appname
# Copy all directories & files (as templates)
source_dir = os.path.join(variables['GRAMEXPATH'], 'apps', 'init')
for root, dirs, files in os.walk(source_dir):
for name in dirs + files:
source = os.path.join(root, name)
relpath = os.path.relpath(root, start=source_dir)
target = os.path.join(args.target, relpath,
name.replace('appname', appname))
_copy(source, target, template_data=data)
for empty_dir in ('img', 'data'):
_mkdir(os.path.join(args.target, 'assets', empty_dir))
# Copy error files as-is (not as templates)
error_dir = os.path.join(args.target, 'error')
_mkdir(error_dir)
for source in glob(
os.path.join(variables['GRAMEXPATH'], 'handlers', '?0?.html')):
target = os.path.join(error_dir, os.path.basename(source))
_copy(source, target)
# Create a git repo if none exists.
# But if git is not installed, do not stop. Continue with the rest.
if not os.path.exists(os.path.join(args.target, '.git')):
try:
_run_console('git init')
except OSError:
pass
run_setup(args.target)
def _google_translate(q, source, target, key):
import requests
params = {'q': q, 'target': target, 'key': key}
if source:
params['source'] = source
try:
r = requests.post(
'https://translation.googleapis.com/language/translate/v2',
data=params)
except requests.RequestException:
return app_log.exception('Cannot connect to Google Translate')
response = r.json()
if 'error' in response:
return app_log.error('Google Translate API error: %s',
response['error'])
return {
'q':
q,
't': [t['translatedText'] for t in response['data']['translations']],
'source': [
t.get('detectedSourceLanguage', params.get('source', None))
for t in response['data']['translations']
],
'target': [target] * len(q),
}
def __init__(self, flush=None, purge=None, purge_keys=None, **kwargs):
'''Initialise the KeyStore at path'''
self.store = {}
if callable(purge_keys):
self.purge_keys = purge_keys
elif purge_keys is not None:
app_log.error(
'KeyStore: purge_keys=%r invalid. Must be function(dict)',
purge_keys)
# Periodically flush and purge buffers
if flush is not None:
PeriodicCallback(self.flush, callback_time=flush * 1000).start()
if purge is not None:
PeriodicCallback(self.purge, callback_time=purge * 1000).start()
# Call close() when Python gracefully exits
atexit.register(self.close)
