def install(args, kwargs):
if len(args) < 1:
app_log.error(show_usage('install'))
return
appname = args[0]
app_log.info('Installing: %s', appname)
app_config = get_app_config(appname, kwargs)
if len(args) == 2:
app_config.url = args[1]
run_install(app_config)
elif 'url' in app_config:
run_install(app_config)
elif 'cmd' in app_config:
returncode = run_command(app_config)
if returncode != 0:
app_log.error(
'Command failed with return code %d. Aborting installation',
returncode)
return
else:
app_log.error('Use --url=... or --cmd=... to specific source of %s',
appname)
return
# Post-installation
app_config.target = run_setup(app_config.target)
app_config['installed'] = {'time': datetime.datetime.utcnow()}
save_user_config(appname, app_config)
app_log.info('Installed. Run `gramex run %s`', appname)
def reload_module(*modules):
'''
Reloads one or more modules if they are outdated, i.e. only if required the
underlying source file has changed.
For example::
import mymodule # Load cached module
reload_module(mymodule) # Reload module if the source has changed
This is most useful during template development. If your changes are in a
Python module, add adding these lines to pick up new module changes when
the template is re-run.
'''
for module in modules:
name = getattr(module, '__name__', None)
path = getattr(module, '__file__', None)
if name is None or path is None or not os.path.exists(path):
app_log.warning('Path for module %s is %s: not found', name, path)
continue
# On Python 3, __file__ points to the .py file. In Python 2, it's the .pyc file
# https://www.python.org/dev/peps/pep-3147/#file
if path.lower().endswith('.pyc'):
path = path[:-1]
if not os.path.exists(path):
app_log.warning('Path for module %s is %s: not found', name,
path)
continue
# The first time, don't reload it. Thereafter, if it's older or resized, reload it
fstat = stat(path)
if fstat != _MODULE_CACHE.get(name, fstat):
app_log.info('Reloading module %s', name)
six.moves.reload_module(module)
_MODULE_CACHE[name] = fstat
def _copy(source, target, template_data=None):
'''
Copy single directory or file (as binary) from source to target.
Warn if target exists, or source is not file/directory, and exit.
If template_data is specified, treat source as a Tornado template.
'''
if os.path.exists(target):
app_log.warning('Skip existing %s', target)
elif os.path.isdir(source):
_mkdir(target)
elif os.path.isfile(source):
app_log.info('Copy file %s', source)
with io.open(source, 'rb') as handle:
result = handle.read()
from mimetypes import guess_type
filetype = guess_type(source)[0]
basetype = 'text' if filetype is None else filetype.split('/')[0]
if template_data is not None:
if basetype in {'text'
} or filetype in {'application/javascript'}:
result = Template(result).generate(**template_data)
with io.open(target, 'wb') as handle:
handle.write(result)
else:
app_log.warning('Skip unknown file %s', source)
def run(args, kwargs):
if len(args) < 1:
app_log.error(show_usage('run'))
return
if len(args) > 1:
app_log.error('Can only run one app. Ignoring %s', ', '.join(args[1:]))
appname = args.pop(0)
app_config = get_app_config(appname, kwargs)
target = app_config.target
if 'dir' in app_config:
target = os.path.join(target, app_config.dir)
if os.path.isdir(target):
os.chdir(target)
gramex.paths['base'] = Path('.')
# If we run with updated parameters, save for next run under the .run config
run_config = app_config.setdefault('run', {})
for key, val in kwargs.items():
if key not in app_keys:
run_config[key] = app_config.pop(key)
save_user_config(appname, app_config)
# Tell the user what configs are used
cline = ' '.join('--%s=%s' % arg
for arg in flatten_config(app_config.get('run', {})))
app_log.info('Gramex %s | %s %s | %s | Python %s',
gramex.__version__, appname, cline, os.getcwd(),
sys.version.replace('\n', ' '))
gramex.init(args=AttrDict(app=app_config['run']))
elif appname in apps_config['user']:
# The user configuration has a wrong path. Inform user
app_log.error('%s: no directory %s', appname, app_config.target)
app_log.error('Run "gramex uninstall %s" and try again.', appname)
else:
app_log.error('%s: no directory %s', appname, app_config.target)
def run_command(config):
'''
Run config.cmd. If the command has a TARGET, replace it with config.target.
Else append config.target as an argument.
'''
appcmd = config.cmd
# Split the command into an array of words
if isinstance(appcmd, six.string_types):
appcmd = shlex.split(appcmd)
# If the app is a Cygwin app, TARGET should be a Cygwin path too.
target = config.target
cygcheck, cygpath, kwargs = which('cygcheck'), which('cygpath'), {'universal_newlines': True}
if cygcheck is not None and cygpath is not None:
app_path = check_output([cygpath, '-au', which(appcmd[0])], **kwargs).strip() # nosec
is_cygwin_app = check_output([cygcheck, '-f', app_path], **kwargs).strip() # nosec
if is_cygwin_app:
target = check_output([cygpath, '-au', target], **kwargs).strip() # n osec
# Replace TARGET with the actual target
if 'TARGET' in appcmd:
appcmd = [target if arg == 'TARGET' else arg for arg in appcmd]
else:
appcmd.append(target)
app_log.info('Running %s', ' '.join(appcmd))
if not safe_rmtree(config.target):
app_log.error('Cannot delete target %s. Aborting installation', config.target)
return
proc = Popen(appcmd, bufsize=-1, stdout=sys.stdout, stderr=sys.stderr, **kwargs) # nosec
proc.communicate()
return proc.returncode
def mail(self, **kwargs):
'''
Sends an email. It accepts any email parameter in RFC 2822
'''
sender = kwargs.get('sender', self.email)
# SES allows restricting the From: address. https://amzn.to/2Kqwh2y
# Mailgun suggests From: be the same as Sender: http://bit.ly/2tGS5wt
kwargs.setdefault('from', sender)
to = recipients(**kwargs)
msg = message(**kwargs)
tls = self.client.get('tls', True)
# Test cases specify stub: true. This uses a stub that logs emails
if self.stub:
server = SMTPStub(self.client['host'],
self.client.get('port', self.ports[tls]),
self.stub)
else:
server = smtplib.SMTP(self.client['host'],
self.client.get('port', self.ports[tls]))
if tls:
server.starttls()
if self.email is not None and self.password is not None:
server.login(self.email, self.password)
server.sendmail(sender, to, msg.as_string())
server.quit()
app_log.info('Email sent via %s (%s) to %s', self.client['host'],
self.email, ', '.join(to))
def alert(conf):
from . import scheduler
_stop_all_tasks(info.alert)
schedule_keys = 'minutes hours dates months weekdays years startup utc'.split(
)
for name, alert in conf.items():
_key = cache_key('alert', alert)
if _key in _cache:
task = info.alert[name] = _cache[_key]
task.call_later()
continue
app_log.info('Initialising alert: %s', name)
schedule = {key: alert[key] for key in schedule_keys if key in alert}
if 'thread' in alert:
schedule['thread'] = alert['thread']
schedule['function'] = create_alert(name, alert)
if schedule['function'] is not None:
try:
_cache[_key] = scheduler.Task(name,
schedule,
info.threadpool,
ioloop=info._main_ioloop)
info.alert[name] = _cache[_key]
except Exception:
app_log.exception('Failed to initialize alert: %s', name)
def callback_commandline(commands):
'''
Find what method should be run based on the command line programs. This
refactoring allows us to test gramex.commandline() to see if it processes
the command line correctly, without actually running the commands.
Returns a callback method and kwargs for the callback method.
'''
# Set logging config at startup. (Services may override this.)
log_config = (+PathConfig(paths['source'] / 'gramex.yaml')).get(
'log', AttrDict())
log_config.root.level = logging.INFO
from . import services
services.log(log_config)
# kwargs has all optional command line args as a dict of values / lists.
# args has all positional arguments as a list.
kwargs = parse_command_line(commands)
args = kwargs.pop('_')
# If --help or -V --version is specified, print a message and end
if kwargs.get('V') is True or kwargs.get('version') is True:
return console, {'msg': 'Gramex %s' % __version__}
# Any positional argument is treated as a gramex command
if len(args) > 0:
base_command = args.pop(0).lower()
method = 'install' if base_command == 'update' else base_command
if method in {
'install',
'uninstall',
'setup',
'run',
'service',
'init',
'mail',
'license',
}:
import gramex.install
if 'help' in kwargs:
return console, {'msg': gramex.install.show_usage(method)}
return getattr(gramex.install, method), {
'args': args,
'kwargs': kwargs
}
raise NotImplementedError('Unknown gramex command: %s' % base_command)
elif kwargs.get('help') is True:
return console, {'msg': __doc__.strip().format(**globals())}
# Use current dir as base (where gramex is run from) if there's a gramex.yaml.
if not os.path.isfile('gramex.yaml'):
return console, {
'msg': 'No gramex.yaml. See https://learn.gramener.com/guide/'
}
# Run gramex.init(cmd={command line arguments like YAML variables})
app_log.info('Gramex %s | %s | Python %s', __version__, os.getcwd(),
sys.version.replace('\n', ' '))
return init, {'cmd': AttrDict(app=kwargs)}
def run(self, *args, **kwargs):
'''Run task. Then set up next callback.'''
app_log.info('Running %s', self.name)
try:
self.result = self.function(*args, **kwargs)
finally:
# Run again, if not stopped via self.stop() or end of schedule
if self.callback is not None:
self.call_later()
def run(self):
'''Run task. Then set up next callback.'''
app_log.info('Running %s', self.name)
try:
self.result = self.function()
finally:
# Do not schedule if stopped (e.g. via self.stop())
if self.callback is not None:
self.call_later()
def merge(path, force=False):
'''Merge log file from path into database'''
if not os.path.exists(path):
return
src = os.path.split(path)[-1]
if src in merged and not force:
return
app_log.info('consolidating %s', src)
result = []
for line in io.open(path, 'r', encoding='utf-8'):
row = json.loads(line)
row['src'] = src
# uname is a list. Convert into system data
(row['system'], row['node'], row['release'], row['version'],
row['machine'], row['processor']) = row.pop('uname')
row_data = row.pop('data')
row_data = json.loads(row_data) if row_data else {} # parse. Ignore missing data
# If data is double-encoded, decode again. TODO: figure out when & why
if isinstance(row_data, six.string_types):
row_data = json.loads(row_data)
# Startup args are a list. Join with spaces
if 'args' in row_data and isinstance(row_data['args'], list):
row_data['args'] = ' '.join(row_data['args'])
row.update(row_data)
result.append(row)
# Post-process results
result = pd.DataFrame(result)
ns = 1E9 # nanosecond conversion
result['date'] = pd.to_datetime(result['time'] * ns).dt.strftime('%Y-%m-%d')
result['time'] = result['time'].astype(int)
# Replace rows for file currently processed
engine.execute('DELETE FROM logs WHERE src=?', src)
# SQLite supports 999 variables in an insert by default.
# chunksize=60 ensures that 15 columns x 60 = 750 is within the limit.
result.to_sql('logs', engine, if_exists='append', index=False, chunksize=60)
# Summarize monthly results into "mau" (Monthly Average Users) table
engine.execute('DROP TABLE IF EXISTS mau')
engine.execute('''
CREATE TABLE mau as
SELECT month, COUNT(DISTINCT node) as nodes FROM (
SELECT SUBSTR(date, 0, 8) AS month, node, COUNT(node) AS times
FROM logs
WHERE node NOT LIKE 'travis-%' /* Travis */
AND node NOT LIKE 'runner-%' /* Gitlab CI */
AND release NOT LIKE '%-linuxkit' /* Docker */
GROUP BY month, node
) WHERE times > 2 /* CI nodes startup/shutdown only once */
GROUP BY month
''')
def init(args, kwargs):
'''Create Gramex scaffolding files.'''
if len(args) > 1:
app_log.error(show_usage('init'))
return
kwargs.setdefault('target', os.getcwd())
app_log.info('Initializing Gramex project at %s', kwargs.target)
data = {
'appname': os.path.basename(kwargs.target),
'author': _check_output('git config user.name', default='Author'),
'email': _check_output('git config user.email',
default='*****@*****.**'),
'date': datetime.datetime.today().strftime('%Y-%m-%d'),
'version': gramex.__version__,
}
# Ensure that appname is a valid Python module name
appname = slug.module(data['appname'])
if appname[0] not in string.ascii_lowercase:
appname = 'app' + appname
data['appname'] = appname
# Create a git repo. But if git fails, do not stop. Continue with the rest.
try:
_run_console('git init')
except OSError:
pass
# Install Git LFS if available. Set git_lfs=None if it fails, so .gitignore ignores assets/**
data['git_lfs'] = which('git-lfs')
if data['git_lfs']:
try:
_run_console('git lfs install')
_run_console('git lfs track "assets/**"')
except OSError:
data['git_lfs'] = None
# Copy all directories & files (as templates)
source_dir = os.path.join(variables['GRAMEXPATH'], 'apps', 'init')
for root, dirs, files in os.walk(source_dir):
for name in dirs + files:
source = os.path.join(root, name)
relpath = os.path.relpath(root, start=source_dir)
target = os.path.join(kwargs.target, relpath,
name.replace('appname', appname))
_copy(source, target, template_data=data)
for empty_dir in ('img', 'data'):
_mkdir(os.path.join(kwargs.target, 'assets', empty_dir))
# Copy error files as-is (not as templates)
error_dir = os.path.join(kwargs.target, 'error')
_mkdir(error_dir)
for source in glob(
os.path.join(variables['GRAMEXPATH'], 'handlers', '?0?.html')):
target = os.path.join(error_dir, os.path.basename(source))
_copy(source, target)
run_setup(kwargs.target)
def _start(self):
'''
Check if capture is already running at ``url``. If not, start ``cmd``
and check again. Print logs from ``cmd``.
'''
self.started = False
script = self.engine.script
try:
# Check if capture.js is at the url specified
app_log.info('Pinging %s at %s', script, self.url)
r = requests.get(self.url, timeout=self.timeout)
self._validate_server(r)
self.started = True
except requests.ReadTimeout:
# If capture.js doesn't respond immediately, we haven't started
app_log.error('url: %s timed out', self.url)
except requests.ConnectionError:
# Try starting the process again
app_log.info('Starting %s via %s', script, self.cmd)
self.close()
# self.cmd is taken from the YAML configuration. Safe to run
self.proc = Popen(shlex.split(self.cmd),
stdout=PIPE,
stderr=STDOUT) # nosec
self.proc.poll()
atexit.register(self.close)
# TODO: what if readline() does not return quickly?
line = self.proc.stdout.readline().strip()
if not self.first_line_re.search(line):
return app_log.error('cmd: %s invalid. Returned "%s"',
self.cmd, line)
app_log.info('Pinging %s at %s', script, self.url)
try:
r = requests.get(self.url, timeout=self.timeout)
self._validate_server(r)
pid = self.proc.pid
app_log.info(line.decode('utf-8') + ' live (pid=%s)', pid)
self.started = True
# Keep logging capture.js output until proc is killed by another thread
while hasattr(self, 'proc'):
line = self.proc.stdout.readline().strip()
if len(line) == 0:
app_log.info('%s terminated: pid=%d', script, pid)
self.started = False
break
# Capture won't print anything, unless there's a problem, or if debug is on.
# So log it at warning level not info.
app_log.warning(line.decode('utf-8'))
except Exception:
app_log.exception('Ran %s. But %s not at %s', self.cmd, script,
self.url)
except Exception:
app_log.exception('Cannot start Capture')
def close(self):
'''Stop capture.js if it has been started by this object'''
if hasattr(self, 'proc'):
try:
process = psutil.Process(self.proc.pid)
for proc in process.children(recursive=True):
proc.kill()
process.kill()
except psutil.NoSuchProcess:
app_log.info('%s PID %d already killed', self.engine.script, self.proc.pid)
pass
delattr(self, 'proc')
def mail(args, kwargs):
# Get config file location
default_dir = os.path.join(variables['GRAMEXDATA'], 'mail')
_mkdir(default_dir)
if 'conf' in kwargs:
confpath = kwargs.conf
elif os.path.exists('gramex.yaml'):
confpath = os.path.abspath('gramex.yaml')
else:
confpath = os.path.join(default_dir, 'gramexmail.yaml')
if not os.path.exists(confpath):
if 'init' in kwargs:
with io.open(confpath, 'w', encoding='utf-8') as handle:
handle.write(default_mail_config.format(confpath=confpath))
app_log.info('Initialized %s', confpath)
elif not args and not kwargs:
app_log.error(show_usage('mail'))
else:
app_log.error('Missing config %s. Use --init to generate skeleton',
confpath)
return
conf = PathConfig(confpath)
if 'list' in kwargs:
for key, alert in conf.get('alert', {}).items():
to = alert.get('to', '')
if isinstance(to, list):
to = ', '.join(to)
gramex.console('{:15}\t"{}" to {}'.format(key,
alert.get('subject'),
to))
return
if 'init' in kwargs:
app_log.error('Config already exists at %s', confpath)
return
if len(args) < 1:
app_log.error(show_usage('mail'))
return
from gramex.services import email as setup_email, create_alert
alert_conf = conf.get('alert', {})
email_conf = conf.get('email', {})
setup_email(email_conf)
sys.path += os.path.dirname(confpath)
for key in args:
if key not in alert_conf:
app_log.error('Missing key %s in %s', key, confpath)
continue
alert = create_alert(key, alert_conf[key])
alert()
def languagetool_download():
if _languagetool['installed']:
return
import requests, zipfile, io # noqa
target = _languagetool['defaults']['LT_TARGET']
if not os.path.isdir(target):
os.makedirs(target)
src = _languagetool['defaults']['LT_SRC'].format(**_languagetool['defaults'])
app_log.info('Downloading languagetools from %s', src)
stream = io.BytesIO(requests.get(src).content)
app_log.info('Unzipping languagetools to %s', target)
zipfile.ZipFile(stream).extractall(target)
_languagetool['installed'] = True
def send(self, to, subject, sender):
result = self.client.publish(PhoneNumber=to,
Message=subject,
MessageAttributes={
'AWS.SNS.SMS.SenderID': {
'DataType': 'String',
'StringValue': sender,
},
'AWS.SNS.SMS.SMSType': {
'DataType': 'String',
'StringValue': self.smstype,
}
})
app_log.info('SMS sent. SNS MessageId: %s', result['MessageId'])
return result
def init(cmd, args):
'''Create Gramex scaffolding files.'''
if len(cmd) > 1:
app_log.error(show_usage('init'))
return
args.setdefault('target', os.getcwd())
app_log.info('Initializing Gramex project at %s', args.target)
data = {
'appname': os.path.basename(args.target),
'author': _check_output('git config user.name', default='Author'),
'email': _check_output('git config user.email',
default='*****@*****.**'),
'date': datetime.datetime.today().strftime('%Y-%m-%d'),
'version': gramex.__version__,
}
# Ensure that appname is a valid Python module name
appname = re.sub(r'[^a-z0-9_]+', '_', data['appname'].lower())
if appname[0] not in string.ascii_lowercase:
appname = 'app' + appname
data['appname'] = appname
# Copy all directories & files (as templates)
source_dir = os.path.join(variables['GRAMEXPATH'], 'apps', 'init')
for root, dirs, files in os.walk(source_dir):
for name in dirs + files:
source = os.path.join(root, name)
relpath = os.path.relpath(root, start=source_dir)
target = os.path.join(args.target, relpath,
name.replace('appname', appname))
_copy(source, target, template_data=data)
for empty_dir in ('img', 'data'):
_mkdir(os.path.join(args.target, 'assets', empty_dir))
# Copy error files as-is (not as templates)
error_dir = os.path.join(args.target, 'error')
_mkdir(error_dir)
for source in glob(
os.path.join(variables['GRAMEXPATH'], 'handlers', '?0?.html')):
target = os.path.join(error_dir, os.path.basename(source))
_copy(source, target)
# Create a git repo if none exists.
# But if git is not installed, do not stop. Continue with the rest.
if not os.path.exists(os.path.join(args.target, '.git')):
try:
_run_console('git init')
except OSError:
pass
run_setup(args.target)
def flags():
"""
fetch flags data into database.sqlite3/flags
"""
try:
flags = pd.read_sql_table('flags', engine)
if len(flags) >= 1:
return
except DatabaseError:
pass
if os.path.exists(filepath):
app_log.warning('database.sqlite3 corrupted. Removing it')
os.unlink(filepath)
url = os.path.join(folder, 'flags.csv')
flags = pd.read_csv(url, encoding='cp1252')
flags.to_sql('flags', engine, index=False)
app_log.info('database.sqlite3 created with flags table')
def uninstall(cmd, args):
if len(cmd) < 1:
app_log.error(show_usage('uninstall'))
return
if len(cmd) > 1 and args:
app_log.error('Arguments allowed only with single app. Ignoring %s', ', '.join(cmd[1:]))
cmd = cmd[:1]
for appname in cmd:
app_log.info('Uninstalling: %s', appname)
# Delete the target directory if it exists
app_config = get_app_config(appname, args)
if os.path.exists(app_config.target):
safe_rmtree(app_config.target)
else:
app_log.error('No directory %s to remove', app_config.target)
save_user_config(appname, None)
def setup(cls, cmd, user=None, password=None, startup='manual', cwd=None, wait=0):
from gramex.config import app_log
name, service_name = cls._svc_display_name_, cls._svc_name_
port = getattr(cls, '_svc_port_', None)
if cwd is None:
cwd = os.getcwd()
info = (name, cwd, 'port %s' % port if port is not None else '')
service_class = win32serviceutil.GetServiceClassString(cls)
startup = cls.startup_map[startup]
running = win32service.SERVICE_RUNNING
if cmd[0] == 'install':
win32serviceutil.InstallService(
service_class, service_name, displayName=name, description=cls._svc_description_,
startType=startup, userName=user, password=password)
win32serviceutil.SetServiceCustomOption(cls._svc_name_, 'cwd', cwd)
app_log.info('Installed service. %s will run from %s %s' % info)
elif cmd[0] in {'update', 'change'}:
win32serviceutil.ChangeServiceConfig(
service_class, service_name, displayName=name, description=cls._svc_description_,
startType=startup, userName=user, password=password)
win32serviceutil.SetServiceCustomOption(cls._svc_name_, 'cwd', cwd)
app_log.info('Updated service. %s will run from %s %s' % info)
elif cmd[0] in {'remove', 'uninstall'}:
try:
win32serviceutil.StopService(service_name)
except pywintypes.error as e:
if e.args[0] != winerror.ERROR_SERVICE_NOT_ACTIVE:
raise
win32serviceutil.RemoveService(service_name)
app_log.info('Removed service. %s ran from %s %s' % info)
elif cmd[0] == 'start':
win32serviceutil.StartService(service_name, cmd[1:])
if wait:
win32serviceutil.WaitForServiceStatus(service_name, running, wait)
app_log.info('Started service %s at %s %s' % info)
elif cmd[0] == 'stop':
if wait:
win32serviceutil.StopServiceWithDeps(service_name, waitSecs=wait)
else:
win32serviceutil.StopService(service_name)
app_log.info('Stopped service %s at %s %s' % info)
elif cmd[0]:
app_log.error('Unknown command: %s' % cmd[0])
def call(self, inputs):
height, width = inputs.shape[1:3]
_, block_height, block_width, _ = self.size
to_resize = False
new_width = width
if width % block_width != 0:
new_width = block_width * (width // block_width + 1)
to_resize = True
new_height = height
if height % block_height != 0:
new_height = block_height * (height // block_height + 1)
to_resize = True
if to_resize:
inputs = tf.image.resize(inputs, (new_height, new_width))
if self.multires:
tile_heights = tile_widths = _get_patch_sizes(
new_height, new_width)
app_log.info(
f"{len(tile_heights)} patches found for image of size ({height}, {width})"
)
with tf.device("cpu"):
patches = [
tf.image.extract_patches(
inputs,
(1, h, w, 1),
(1, h / 4, w / 4, 1),
rates=[1, 1, 1, 1],
padding="VALID",
) for h, w in zip(tile_heights, tile_widths)
]
patches = [
self._resize(p, h, w, 3)
for p, h, w in zip(patches, tile_heights, tile_widths)
]
else:
patches = tf.image.extract_patches(inputs,
self.size,
self.stride,
rates=[1, 1, 1, 1],
padding="VALID")
patches = self._resize(patches, *self.size[1:-1], inputs.shape[-1])
return patches
def schedule(conf):
'''Set up the Gramex PeriodicCallback scheduler'''
# Create tasks running on ioloop for the given schedule, store it in info.schedule
from . import scheduler
_stop_all_tasks(info.schedule)
for name, sched in conf.items():
_key = cache_key('schedule', sched)
if _key in _cache:
task = info.schedule[name] = _cache[_key]
task.call_later()
continue
try:
app_log.info('Initialising schedule:%s', name)
_cache[_key] = scheduler.Task(name,
sched,
info.threadpool,
ioloop=info._main_ioloop)
info.schedule[name] = _cache[_key]
except Exception as e:
app_log.exception(e)
def languagetoolrequest(text, lang='en-us', **kwargs):
"""Check grammar by making a request to the LanguageTool server.
Parameters
----------
text : str
Text to check
lang : str, optional
Language. See a list of supported languages here: https://languagetool.org/api/v2/languages
"""
client = AsyncHTTPClient()
url = kwargs['LT_URL'].format(**kwargs)
query = six.moves.urllib_parse.urlencode({'language': lang, 'text': text})
url = url + query
tries = 2 # See: https://github.com/gramener/gramex/pull/125#discussion_r266200480
while tries:
try:
result = yield client.fetch(url)
tries = 0
except ConnectionRefusedError:
# Start languagetool
from gramex.cache import daemon
cmd = [p.format(**kwargs) for p in kwargs['LT_CMD']]
app_log.info('Starting: %s', ' '.join(cmd))
if 'proc' not in _languagetool:
import re
_languagetool['proc'] = daemon(
cmd,
cwd=kwargs['LT_CWD'],
first_line=re.compile(r"Server started\s*$"),
stream=True,
timeout=5,
buffer_size=512)
try:
result = yield client.fetch(url)
tries = 0
except ConnectionRefusedError:
yield sleep(1)
tries -= 1
raise Return(result.body)
def run_setup(target):
'''
Install any setup file in target directory. Target directory can be:
- An absolute path
- A relative path to current directory
- A relative path to the Gramex apps/ folder
Returns the absolute path of the final target path.
This supports:
- ``make`` (if Makefile exists)
- ``powershell -File setup.ps1``
- ``bash setup.sh``
- ``pip install -r requirements.txt``
- ``python setup.py``
- ``yarn install`` else ``npm install``
- ``bower --allow-root install``
'''
if not os.path.exists(target):
app_target = os.path.join(variables['GRAMEXPATH'], 'apps', target)
if not os.path.exists(app_target):
raise OSError('No directory %s' % target)
target = app_target
target = os.path.abspath(target)
app_log.info('Setting up %s', target)
for file, runners in setup_paths.items():
setup_file = os.path.join(target, file)
if not os.path.exists(setup_file):
continue
for exe, cmd in runners.items():
exe_path = which(exe)
if exe_path is not None:
cmd = cmd.format(FILE=setup_file, EXE=exe_path)
app_log.info('Running %s', cmd)
_run_console(cmd, cwd=target)
break
else:
app_log.warning('Skipping %s. No %s found', setup_file, exe)
def _fit(model, x, y, path=None, name=None):
app_log.info('Starting training...')
getattr(model, 'partial_fit', model.fit)(x, y)
app_log.info('Done training...')
joblib.dump(model, path)
app_log.info(f'{name}: Model saved at {path}.')
return model
def merge(path, force=False):
'''Merge log file from path into database'''
src = os.path.split(path)[-1]
if src in merged and not force:
return
app_log.info('consolidating %s', src)
result = []
for line in io.open(path, 'r', encoding='utf-8'):
row = json.loads(line)
row['src'] = src
# uname is a list. Convert into system data
(row['system'], row['node'], row['release'], row['version'],
row['machine'], row['processor']) = row.pop('uname')
row_data = row.pop('data')
row_data = json.loads(row_data) if row_data else {} # parse. Ignore missing data
# If data is double-encoded, decode again. TODO: figure out when & why
if isinstance(row_data, six.string_types):
row_data = json.loads(row_data)
# Startup args are a list. Join with spaces
if 'args' in row_data and isinstance(row_data['args'], list):
row_data['args'] = ' '.join(row_data['args'])
row.update(row_data)
result.append(row)
# Post-process results
result = pd.DataFrame(result)
ns = 1E9 # nanosecond conversion
result['date'] = pd.to_datetime(result['time'] * ns).dt.strftime('%Y-%m-%d')
result['time'] = result['time'].astype(int)
# Replace rows for file currently processed
engine.execute('DELETE FROM logs WHERE src=?', src)
# SQLite supports 999 variables in an insert by default.
# chunksize=60 ensures that 15 columns x 60 = 750 is within the limit.
result.to_sql('logs', engine, if_exists='append', index=False, chunksize=60)
def run_install(config):
'''
Download config.url into config.target.
If config.url is a directory, copy it.
If config.url is a file or a URL (http, https, ftp), unzip it.
If config.contentdir is True, skip parent folders with single subfolder.
If no files match, log a warning.
'''
url, target = config.url, config.target
# If the URL is a directory, copy it
if os.path.isdir(url):
if os.path.exists(target):
url = os.path.abspath(url).lower().rstrip(os.sep)
target = os.path.abspath(target).lower().rstrip(os.sep)
if url != target:
if not safe_rmtree(target):
return
if url != target:
shutil.copytree(url, target)
app_log.info('Copied %s into %s', url, target)
config.url = url
return
# If it's a file, unzip it
if os.path.exists(url):
handle = url
else:
# Otherwise, assume that it's a URL containing a ZIP file
app_log.info('Downloading: %s', url)
response = requests.get(url)
response.raise_for_status()
handle = io.BytesIO(response.content)
# Identify relevant files from the ZIP file
zipfile = ZipFile(handle)
files = zipfile.infolist()
if config.get('contentdir', True):
prefix = os.path.commonprefix(zipfile.namelist())
if prefix.endswith('/'):
files = zip_prefix_filter(files, prefix)
# Extract relevant files from ZIP file
if safe_rmtree(target):
zipfile.extractall(target, files)
app_log.info('Extracted %d files into %s', len(files), target)
def run_alert(callback=None, args=None):
'''
Runs the configured alert. If a callback is specified, calls the
callback with all email arguments. Else sends the email.
If args= is specified, add it as data['args'].
'''
app_log.info('alert: %s running', name)
data, each, fail = {
'config': alert,
'args': {} if args is None else args
}, [], []
try:
load_datasets(data, each)
except Exception as e:
app_log.exception('alert: %s data processing failed', name)
fail.append({'error': e})
retval = []
for index, row in each:
data['index'], data['row'], data['config'] = index, row, alert
try:
retval.append(
AttrDict(index=index, row=row, mail=create_mail(data)))
except Exception as e:
app_log.exception('alert: %s[%s] templating (row=%r)', name,
index, row)
fail.append({'index': index, 'row': row, 'error': e})
callback = mailer.mail if not callable(callback) else callback
done = []
for v in retval:
try:
callback(**v.mail)
except Exception as e:
fail.append({
'index': v.index,
'row': v.row,
'mail': v.mail,
'error': e
})
app_log.exception('alert: %s[%s] delivery (row=%r)', name,
v.index, v.row)
else:
done.append(v)
event = {
'alert':
name,
'service':
service,
'from':
mailer.email or '',
'to':
'',
'cc':
'',
'bcc':
'',
'subject':
'',
'datetime':
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ")
}
event.update({k: v for k, v in v.mail.items() if k in event})
event['attachments'] = ', '.join(v.mail.get('attachments', []))
alert_logger.info(event)
# Run notifications
args = {'done': done, 'fail': fail}
for notification_name in alert.get('notify', []):
notify = info.alert.get(notification_name)
if notify is not None:
notify.run(callback=callback, args=args)
else:
app_log.error('alert: %s.notify: alert %s not defined', name,
notification_name)
return args
def callback():
'''Called after all services are started. Opens browser if required'''
if ioloop_running(ioloop):
return
# If enterprise version is installed, user must accept license
try:
import gramexenterprise # noqa
gramex.license.accept()
except ImportError:
pass
app_log.info('Listening on port %d', conf.listen.port)
app_log_extra['port'] = conf.listen.port
# browser: True opens the application home page on localhost.
# browser: url opens the application to a specific URL
url = 'http://127.0.0.1:%d/' % conf.listen.port
if conf.browser:
if isinstance(conf.browser, str):
url = urlparse.urljoin(url, conf.browser)
try:
browser = webbrowser.get()
app_log.info('Opening %s in %s browser', url,
browser.__class__.__name__)
browser.open(url)
except webbrowser.Error:
app_log.info('Unable to open browser')
else:
app_log.info(
'<Ctrl-B> opens the browser. <Ctrl-D> starts the debugger.'
)
# Ensure that we call shutdown() on Ctrl-C.
# On Windows, Tornado does not exit on Ctrl-C. This also fixes that.
# When Ctrl-C is pressed, signal_handler() sets _exit to [True].
# check_exit() periodically watches and calls shutdown().
# But signal handlers can only be set in the main thread.
# So ignore if we're not in the main thread (e.g. for nosetests, Windows service)
#
# Note: The PeriodicCallback takes up a small amount of CPU time.
# Note: getch() doesn't handle keyboard buffer queue.
# Note: This is no guarantee that shutdown() will be called.
if isinstance(threading.current_thread(), threading._MainThread):
exit = [False]
def check_exit():
if exit[0] is True:
shutdown()
# If Ctrl-D is pressed, run the Python debugger
char = debug.getch()
if char == b'\x04':
import ipdb as pdb # noqa
pdb.set_trace() # noqa
# If Ctrl-B is pressed, start the browser
if char == b'\x02':
browser = webbrowser.get()
browser.open(url)
def signal_handler(signum, frame):
exit[0] = True
try:
signal.signal(signal.SIGINT, signal_handler)
except ValueError:
# When running as a Windows Service (winservice.py), python
# itself is on a thread, I think. So ignore the
# ValueError: signal only works in main thread.
pass
else:
tornado.ioloop.PeriodicCallback(check_exit,
callback_time=500).start()
info._main_ioloop = ioloop
ioloop.start()