def delay_alert(db):
tables = MonitoredTable.get_monitored_tables(db.name)
for table in tables:
sql_df = get_last_results(db, table, 'metrics_data_delay')
checked_txt = f'delay since last data'
alert_on_z_score(sql_df, table, 'value', 'delay_alert', checked_txt)
def check_for_new_tables(db):
tables = db.db.table_names()
monitored_tables = MonitoredTable.get_monitored_tables(db.name)
monitored_tables_names = set(
[table.table_name for table in monitored_tables])
for table_name in tables:
if table_name not in monitored_tables_names:
table = MonitoredTable.setup_for_source_table(db, table_name)
if table:
insert_schema_changed_record(table, 'table created', None,
None, None)
def volume_alert(db):
tables = MonitoredTable.get_monitored_tables(db.name)
for table in tables:
sql_df = get_last_results(db, table, 'metrics_data_volume')
for interval in settings.VOLUME_INTERVAL:
filtered = sql_df[sql_df['time_interval'] == interval]
checked_txt = f'volume in interval: {interval}'
alert_on_z_score(filtered, table, 'count', 'volume_alert',
checked_txt)
def create_dashboards():
grafana_api = GrafanaFace(auth=(settings.GF_SECURITY_ADMIN_USER,
settings.GF_SECURITY_ADMIN_PASSWORD),
host='grafana:3000')
create_source_in_grafana(grafana_api)
dashboards = []
for db in source_dbs:
monitored_tables = MonitoredTable.get_monitored_tables(db.name)
for table in monitored_tables:
dash_data = create_dashboard_for_table(grafana_api, db, table)
dashboards.append(dash_data)
create_home_dashboard(grafana_api, dashboards)
def create_dashboards():
grafana_api = GrafanaFace(
auth=(settings.GF_SECURITY_ADMIN_USER, settings.GF_SECURITY_ADMIN_PASSWORD),
host=f'{settings.GRAFANA_WEB_HOST}:{settings.GRAFANA_WEB_PORT}'
)
create_source_in_grafana(grafana_api)
dashboards = []
for db in source_dbs:
monitored_tables = MonitoredTable.get_monitored_tables(db.name)
for table in monitored_tables:
dash_data = create_dashboard_for_table(grafana_api, db, table)
dashboards.append(dash_data)
home_response = create_home_dashboard(grafana_api, dashboards)
star_home_dashboard(grafana_api, home_response)
def values_alert(db):
tables = MonitoredTable.get_monitored_tables(db.name)
for table in tables:
sql_df = get_last_results(db, table, 'metrics_data_values')
checks_df = sql_df[['check_name', 'time_interval',
'column_name']].drop_duplicates()
for i, row in checks_df.iterrows():
df = sql_df[(sql_df['check_name'] == row['check_name'])
& (sql_df['time_interval'] == row['time_interval']) &
(sql_df['column_name'] == row['column_name'])]
check_text = f'values for {row.check_name} in column: {row.column_name}, interval: {row.time_interval}'
alert_on_z_score(df, table, 'check_value', row['check_name'],
check_text)
def create_dashboards():
grafana_api = GrafanaFace(
auth=(settings.GF_SECURITY_ADMIN_USER,
settings.GF_SECURITY_ADMIN_PASSWORD),
host=f'{settings.GRAFANA_WEB_HOST}:{settings.GRAFANA_WEB_PORT}')
create_source_in_grafana(grafana_api)
create_notifcation_channels(grafana_api)
dashboards = []
for db in DataSource.source_dbs():
monitored_tables = MonitoredTable.get_monitored_tables(db.name)
for table in monitored_tables:
dash_data = create_dashboard_for_table(grafana_api, db, table)
table.grafana_url = dash_data['dashboard']['url']
dashboards.append(dash_data)
metrics_session.commit()
home_response = create_home_dashboard(grafana_api, dashboards)
star_home_dashboard(grafana_api, home_response)
def run_checks(db):
tables = MonitoredTable.get_monitored_tables(db.name)
for table in tables:
run_checks_for_table(db, table)
