def test_walk_list(self):
# Test gramex.config.walk with lists
o = yaml.load('''
- 1
- 2
- 3
''',
Loader=ConfigYAMLLoader)
result = list(walk(o))
eq_(result, [(0, 1, [1, 2, 3]), (1, 2, [1, 2, 3]), (2, 3, [1, 2, 3])])
o = yaml.load('''
-
x: 1
-
x: 2
-
x: 3
''',
Loader=ConfigYAMLLoader)
result = list(walk(o))
eq_([('x', 1), (0, {
'x': 1
}), ('x', 2), (1, {
'x': 2
}), ('x', 3), (2, {
'x': 3
})], [(key, val) for key, val, node in result])
def test_walk_dict(self):
# Test gramex.config.walk with dicts
o = yaml.load('''
a:
b:
c: 1
d: 2
e: 3
f:
g:
h: 4
i: 5
j: 6
k: 7
''', Loader=ConfigYAMLLoader)
result = list(walk(o))
eq_(
[key for key, val, node in result],
list('cdbehigfjak'))
eq_(
[val for key, val, node in result],
[o.a.b.c, o.a.b.d, o.a.b, o.a.e, o.a.f.g.h, o.a.f.g.i, o.a.f.g,
o.a.f, o.a.j, o.a, o.k])
eq_(
[node for key, val, node in result],
[o.a.b, o.a.b, o.a, o.a, o.a.f.g, o.a.f.g, o.a.f,
o.a, o.a, o, o])
def create_mail(data):
'''
Return kwargs that can be passed to a mailer.mail
'''
mail = {}
for key in ['bodyfile', 'htmlfile', 'markdownfile']:
target = key.replace('file', '')
if key in alert and target not in alert:
path = _tmpl(alert[key]).generate(**data).decode('utf-8')
tmpl = gramex.cache.open(path, 'template')
mail[target] = tmpl.generate(**data).decode('utf-8')
for key in addr_fields + ['subject', 'body', 'html', 'markdown']:
if key not in alert:
continue
if isinstance(alert[key], list):
mail[key] = [
_tmpl(v).generate(**data).decode('utf-8')
for v in alert[key]
]
else:
mail[key] = _tmpl(alert[key]).generate(**data).decode('utf-8')
headers = {}
# user: {id: ...} creates an X-Gramex-User header to mimic the user
if 'user' in alert:
user = deepcopy(alert['user'])
for key, val, node in walk(user):
node[key] = _tmpl(val).generate(**data).decode('utf-8')
user = json.dumps(user, ensure_ascii=True, separators=(',', ':'))
headers['X-Gramex-User'] = tornado.web.create_signed_value(
info.app.settings['cookie_secret'], 'user', user)
if 'markdown' in mail:
mail['html'] = _markdown_convert(mail.pop('markdown'))
if 'images' in alert:
mail['images'] = {}
for cid, val in alert['images'].items():
urlpath = _tmpl(val).generate(**data).decode('utf-8')
urldata = urlfetch(urlpath, info=True, headers=headers)
if urldata['content_type'].startswith('image/'):
mail['images'][cid] = urldata['name']
else:
with io.open(urldata['name'], 'rb') as temp_file:
bytestoread = 80
first_line = temp_file.read(bytestoread)
# TODO: let admin know that the image was not processed
app_log.error(
'alert: %s: %s: %d (%s) not an image: %s\n%r', name,
cid, urldata['r'].status_code, urldata['content_type'],
urlpath, first_line)
if 'attachments' in alert:
mail['attachments'] = [
urlfetch(_tmpl(v).generate(**data).decode('utf-8'),
headers=headers) for v in alert['attachments']
]
return mail
def badgerfish(content, handler=None, mapping={}, doctype='<!DOCTYPE html>'):
'''
A transform that converts string content to YAML, then maps nodes
using other functions, and renders the output as HTML.
The specs for this function are in progress.
'''
data = yaml.load(content, Loader=AttrDictYAMLLoader) # nosec
if handler is not None and hasattr(handler, 'file'):
load_imports(data, handler.file)
maps = {
tag: build_transform(trans, vars={'val': None}, filename='badgerfish')
for tag, trans in mapping.items()
}
for tag, value, node in walk(data):
if tag in maps:
node[tag] = yield maps[tag](value)
raise tornado.gen.Return(
lxml.html.tostring(xmljson.badgerfish.etree(data)[0],
doctype=doctype,
encoding='unicode'))
def run_alert(callback=None):
'''
Runs the configured alert. If a callback is specified, calls the
callback with all email arguments. Else sends the email.
'''
app_log.info('alert: %s running', name)
data = {'config': alert}
for key, dataset in datasets.items():
# Allow raw data in lists as-is. Treat dicts as {url: ...}
data[key] = dataset if isinstance(
dataset, list) else gramex.data.filter(**dataset)
result = condition(**data)
# Avoiding isinstance(result, pd.DataFrame) to avoid importing pandas
if type(result).__name__ == 'DataFrame':
data['data'] = result
elif isinstance(result, dict):
data.update(result)
elif not result:
app_log.debug('alert: %s stopped. condition = %s', name, result)
return
each = [(None, None)]
if 'each' in alert:
each_data = data[alert['each']]
if isinstance(each_data, dict):
each = list(each_data.items())
elif isinstance(each_data, list):
each = list(enumerate(each_data))
elif hasattr(each_data, 'iterrows'):
each = list(each_data.iterrows())
else:
app_log.error(
'alert: %s: each: requires data.%s to be a dict/list/DataFrame',
name, alert['each'])
return
kwargslist = []
for index, row in each:
data['index'], data['row'], data['config'] = index, row, alert
# Generate email content
kwargs = {}
kwargslist.append(kwargs)
for key in ['bodyfile', 'htmlfile', 'markdownfile']:
target = key.replace('file', '')
if key in alert and target not in alert:
path = _tmpl(alert[key]).generate(**data).decode('utf-8')
tmpl = gramex.cache.open(path, 'template')
kwargs[target] = tmpl.generate(**data).decode('utf-8')
try:
for key in [
'to', 'cc', 'bcc', 'from', 'subject', 'body', 'html',
'markdown'
]:
if key in alert:
if isinstance(alert[key], list):
kwargs[key] = [
_tmpl(val).generate(**data).decode('utf-8')
for val in alert[key]
]
else:
kwargs[key] = _tmpl(
alert[key]).generate(**data).decode('utf-8')
except Exception:
# If any template raises an exception, log it and continue with next email
app_log.exception('alert: %s(#%s).%s: Template exception',
name, index, key)
continue
headers = {}
# user: {id: ...} creates an X-Gramex-User header to mimic the user
if 'user' in alert:
user = deepcopy(alert['user'])
for key, val, node in walk(user):
node[key] = _tmpl(val).generate(**data).decode('utf-8')
user = json.dumps(user,
ensure_ascii=True,
separators=(',', ':'))
headers['X-Gramex-User'] = tornado.web.create_signed_value(
info.app.settings['cookie_secret'], 'user', user)
if 'markdown' in kwargs:
kwargs['html'] = _markdown_convert(kwargs.pop('markdown'))
if 'images' in alert:
kwargs['images'] = {}
for cid, val in alert['images'].items():
urlpath = _tmpl(val).generate(**data).decode('utf-8')
urldata = urlfetch(urlpath, info=True, headers=headers)
if urldata['content_type'].startswith('image/'):
kwargs['images'][cid] = urldata['name']
else:
with io.open(urldata['name'], 'rb') as temp_file:
bytestoread = 80
first_line = temp_file.read(bytestoread)
app_log.error(
'alert: %s: %s: %d (%s) not an image: %s\n%r',
name, cid, urldata['r'].status_code,
urldata['content_type'], urlpath, first_line)
if 'attachments' in alert:
kwargs['attachments'] = [
urlfetch(
_tmpl(attachment).generate(**data).decode('utf-8'),
headers=headers) for attachment in alert['attachments']
]
if callable(callback):
return callback(**kwargs)
# Email recipient. TODO: run this in a queue. (Anand)
mailer.mail(**kwargs)
# Log the event
event = {
'alert':
name,
'service':
service,
'from':
mailer.email or '',
'to':
'',
'cc':
'',
'bcc':
'',
'subject':
'',
'datetime':
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ")
}
event.update({k: v for k, v in kwargs.items() if k in event})
event['attachments'] = ', '.join(kwargs.get('attachments', []))
alert_logger.info(event)
return kwargslist