def check_for_new_tables(db):
tables = db.db.table_names()
monitored_tables = MonitoredTable.get_monitored_tables(db.name)
monitored_tables_names = set(
[table.table_name for table in monitored_tables])
for table_name in tables:
if table_name not in monitored_tables_names:
table = MonitoredTable.setup_for_source_table(db, table_name)
if table:
insert_schema_changed_record(table, 'table created', None,
None, None)
def run_compute_alerts(db, conf):
for namespace in db.namespaces:
tables = MonitoredTable.get_monitored_tables_per_namespace(db.name, namespace)
for table in tables:
run_compute_alerts_for_table(db, table, conf)
def delay_alert(db):
tables = MonitoredTable.get_monitored_tables(db.name)
for table in tables:
sql_df = get_last_results(db, table, 'metrics_data_delay')
checked_txt = f'delay since last data'
alert_on_z_score(sql_df, table, 'value', 'delay_alert', checked_txt)
def volume_alert(db):
tables = MonitoredTable.get_monitored_tables(db.name)
for table in tables:
sql_df = get_last_results(db, table, 'metrics_data_volume')
for interval in settings.VOLUME_INTERVAL:
filtered = sql_df[sql_df['time_interval'] == interval]
checked_txt = f'volume in interval: {interval}'
alert_on_z_score(filtered, table, 'count', 'volume_alert',
checked_txt)
def check_for_new_tables(db, conf):
results = []
for namespace in db.namespaces:
tables = db.table_names(namespace)
monitored_tables = MonitoredTable.get_monitored_tables_per_namespace(
db.name, namespace)
monitored_tables_names = set(
[table.table_name for table in monitored_tables])
for table_name in tables:
if table_name not in monitored_tables_names:
table = MonitoredTable.setup_for_source_table(
db, table_name, namespace)
if table:
create_for_detected_table(db, table)
for check in table.checks:
if check.name == Metric.SCHEMA_CHANGE:
MetricFromCheck.add_metrics([
schema_changed_record('table detected', None,
None, None, conf)
], check, conf)
def create_dashboards():
grafana_api = GrafanaFace(auth=(settings.GF_SECURITY_ADMIN_USER,
settings.GF_SECURITY_ADMIN_PASSWORD),
host='grafana:3000')
create_source_in_grafana(grafana_api)
dashboards = []
for db in source_dbs:
monitored_tables = MonitoredTable.get_monitored_tables(db.name)
for table in monitored_tables:
dash_data = create_dashboard_for_table(grafana_api, db, table)
dashboards.append(dash_data)
create_home_dashboard(grafana_api, dashboards)
def create_dashboards():
grafana_api = GrafanaFace(
auth=(settings.GF_SECURITY_ADMIN_USER, settings.GF_SECURITY_ADMIN_PASSWORD),
host=f'{settings.GRAFANA_WEB_HOST}:{settings.GRAFANA_WEB_PORT}'
)
create_source_in_grafana(grafana_api)
dashboards = []
for db in source_dbs:
monitored_tables = MonitoredTable.get_monitored_tables(db.name)
for table in monitored_tables:
dash_data = create_dashboard_for_table(grafana_api, db, table)
dashboards.append(dash_data)
home_response = create_home_dashboard(grafana_api, dashboards)
star_home_dashboard(grafana_api, home_response)
def values_alert(db):
tables = MonitoredTable.get_monitored_tables(db.name)
for table in tables:
sql_df = get_last_results(db, table, 'metrics_data_values')
checks_df = sql_df[['check_name', 'time_interval',
'column_name']].drop_duplicates()
for i, row in checks_df.iterrows():
df = sql_df[(sql_df['check_name'] == row['check_name'])
& (sql_df['time_interval'] == row['time_interval']) &
(sql_df['column_name'] == row['column_name'])]
check_text = f'values for {row.check_name} in column: {row.column_name}, interval: {row.time_interval}'
alert_on_z_score(df, table, 'check_value', row['check_name'],
check_text)
def create_dashboards():
grafana_api = GrafanaFace(
auth=(settings.GF_SECURITY_ADMIN_USER,
settings.GF_SECURITY_ADMIN_PASSWORD),
host=f'{settings.GRAFANA_WEB_HOST}:{settings.GRAFANA_WEB_PORT}')
create_source_in_grafana(grafana_api)
create_notifcation_channels(grafana_api)
dashboards = []
for db in DataSource.source_dbs():
monitored_tables = MonitoredTable.get_monitored_tables(db.name)
for table in monitored_tables:
dash_data = create_dashboard_for_table(grafana_api, db, table)
table.grafana_url = dash_data['dashboard']['url']
dashboards.append(dash_data)
metrics_session.commit()
home_response = create_home_dashboard(grafana_api, dashboards)
star_home_dashboard(grafana_api, home_response)
def run_checks(db):
tables = MonitoredTable.get_monitored_tables(db.name)
for table in tables:
run_checks_for_table(db, table)
#class MonitoredTable (db.Model):
# __tablename__ = 'monitored_table'
# id = db.Column('id',db.Integer, primary_key=True)
# created_at = db.Column('created_at',db.TIMESTAMP, default=datetime.datetime.utcnow)
# source_db = db.Column('source_db',db.String, default=None)
#active = db.Column('active',db.Boolean, default=True)
# table_name = db.Column('table_name',db.String)
# time_column = db.Column('time_column',db.String)
# time_column_type = db.Column('time_column_type',db.String)
# schema = db.Column('schema',db.JSON)
class MonitoredTableView(ModelView):
def _user_formatter_time(view, context, model, name):
if model.created_at:
return model.created_at.strftime("%Y-%m-%d %H:%M:%S")
else:
return ""
column_formatters = {'created_at': _user_formatter_time}
column_editable_list = ['active', 'time_column']
column_exclude_list = ['schema']
admin = Admin(app, name='Redata', template_mode='bootstrap3')
admin.add_view(MonitoredTableView(MonitoredTable(db.Model), db.session))
app.run()