def load_datasets(data, each):
'''
Modify data by load datasets and filter by condition.
Modify each to apply the each: argument, else return (None, None)
'''
for key, val in datasets.items():
# Allow raw data in lists as-is. Treat dicts as {url: ...}
data[key] = val if isinstance(val, list) else gramex.data.filter(
**val)
result = condition(**data)
# Avoiding isinstance(result, pd.DataFrame) to avoid importing pandas
if type(result).__name__ == 'DataFrame':
data['data'] = result
elif isinstance(result, dict):
data.update(result)
elif not result:
app_log.debug('alert: %s stopped. condition = %s', name, result)
return
if 'each' in alert:
each_data = data[alert['each']]
if isinstance(each_data, dict):
each += list(each_data.items())
elif isinstance(each_data, list):
each += list(enumerate(each_data))
elif hasattr(each_data, 'iterrows'):
each += list(each_data.iterrows())
else:
raise ValueError(
'alert: %s: each: data.%s must be dict/list/DF, not %s' %
(name, alert['each'], type(each_data)))
else:
each.append((0, None))
def update_label(handler):
"""
pre-populated: update and delete options
user input: insert
"""
chart_id = handler.path_args[0]
data = gramex.cache.query(
"SELECT annotation_id FROM annotations WHERE chart_id={}".format(
chart_id),
engine,
state="SELECT COUNT(*) FROM annotations",
)["annotation_id"].tolist()
annotations = json.loads(handler.request.body.decode("utf8"))
chart_ids = [item["id"] for item in annotations]
# add incoming annotation that doesn't exist already
to_delete = set(data) - set(chart_ids)
[data.remove(k) for k in to_delete]
# intersection operation
to_update = set(data) & set(chart_ids)
# add incoming annotation that doesn't exist already
to_insert = set(chart_ids) - set(data)
for annotation in annotations:
value = annotation.pop("value")
for key in "x y width height".split():
annotation[key] = value[key]
annotation["label"] = value["rectanglelabels"][0]
annotation["chart_id"] = int(chart_id)
annotation["annotation_id"] = annotation.pop("id")
[k.update({"user": handler.current_user.email}) for k in annotations]
df = pd.DataFrame.from_records(annotations)
# to_delete
gdata.delete(
variables["COARSE_LABELS"],
table="annotations",
id="annotation_id",
args={"annotation_id": list(to_delete)},
)
# to update
for _id in to_update:
args = df[df["annotation_id"] == _id]
gdata.update(
variables["COARSE_LABELS"],
table="annotations",
id="annotation_id",
args=args.to_dict(orient="list"),
)
args = df[df["annotation_id"].isin(to_insert)]
gdata.insert(
variables["COARSE_LABELS"],
table="annotations",
id="annotation_id",
args=args.to_dict(orient="list"),
)
def run_alert(callback=None):
'''
Runs the configured alert. If a callback is specified, calls the
callback with all email arguments. Else sends the email.
'''
app_log.info('alert: %s running', name)
data = {'config': alert}
for key, dataset in datasets.items():
# Allow raw data in lists as-is. Treat dicts as {url: ...}
data[key] = dataset if isinstance(
dataset, list) else gramex.data.filter(**dataset)
result = condition(**data)
# Avoiding isinstance(result, pd.DataFrame) to avoid importing pandas
if type(result).__name__ == 'DataFrame':
data['data'] = result
elif isinstance(result, dict):
data.update(result)
elif not result:
app_log.debug('alert: %s stopped. condition = %s', name, result)
return
each = [(None, None)]
if 'each' in alert:
each_data = data[alert['each']]
if isinstance(each_data, dict):
each = list(each_data.items())
elif isinstance(each_data, list):
each = list(enumerate(each_data))
elif hasattr(each_data, 'iterrows'):
each = list(each_data.iterrows())
else:
app_log.error(
'alert: %s: each: requires data.%s to be a dict/list/DataFrame',
name, alert['each'])
return
kwargslist = []
for index, row in each:
data['index'], data['row'], data['config'] = index, row, alert
# Generate email content
kwargs = {}
kwargslist.append(kwargs)
for key in ['bodyfile', 'htmlfile', 'markdownfile']:
target = key.replace('file', '')
if key in templates and target not in templates:
path = templates[key].generate(**data).decode('utf-8')
tmpl = gramex.cache.open(path, 'template')
kwargs[target] = tmpl.generate(**data).decode('utf-8')
try:
for key in [
'to', 'cc', 'bcc', 'from', 'subject', 'body', 'html',
'markdown'
]:
if key in templates:
tmpl = templates[key]
if isinstance(tmpl, list):
kwargs[key] = []
for subtmpl in tmpl:
kwargs[key].append(
subtmpl.generate(**data).decode('utf-8'))
else:
kwargs[key] = tmpl.generate(**data).decode('utf-8')
except Exception:
# If any template raises an exception, log it and continue with next email
app_log.exception('alert: %s(#%s).%s: Template exception',
name, index, key)
continue
headers = {}
# user: {id: ...} creates an X-Gramex-User header to mimic the user
if 'user' in alert:
user = json.dumps(alert['user'],
ensure_ascii=True,
separators=(',', ':'))
headers['X-Gramex-User'] = tornado.web.create_signed_value(
info.app.settings['cookie_secret'], 'user', user)
if 'markdown' in kwargs:
kwargs['html'] = _markdown_convert(kwargs.pop('markdown'))
if 'images' in templates:
kwargs['images'] = {}
for cid, val in templates['images'].items():
urlpath = val.generate(**data).decode('utf-8')
urldata = urlfetch(urlpath, info=True, headers=headers)
if urldata['content_type'].startswith('image/'):
kwargs['images'][cid] = urldata['name']
else:
with io.open(urldata['name'], 'rb') as temp_file:
bytestoread = 80
first_line = temp_file.read(bytestoread)
app_log.error(
'alert: %s: %s: %d (%s) not an image: %s\n%r',
name, cid, urldata['r'].status_code,
urldata['content_type'], urlpath, first_line)
if 'attachments' in templates:
kwargs['attachments'] = [
urlfetch(attachment.generate(**data).decode('utf-8'),
headers=headers)
for attachment in templates['attachments']
]
if callable(callback):
return callback(**kwargs)
# Email recipient. TODO: run this in a queue. (Anand)
mailer.mail(**kwargs)
# Log the event
event = {
'alert':
name,
'service':
service,
'from':
mailer.email or '',
'to':
'',
'cc':
'',
'bcc':
'',
'subject':
'',
'datetime':
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ")
}
event.update({k: v for k, v in kwargs.items() if k in event})
event['attachments'] = ', '.join(kwargs.get('attachments', []))
alert_logger.info(event)
return kwargslist