def process_run():
try:
scan = Scan.get_not_started_run()
if scan is not None:
scan.status = "pending"
metrics_session.commit()
for_time = scan.start_date
while for_time <= scan.end_date:
conf = Conf(for_time)
for source_db in DataSource.source_dbs():
run_check_for_new_tables(source_db, conf)
run_checks(source_db, conf)
run_compute_alerts(source_db, conf)
for_time += timedelta(days=1)
generate_grafana()
scan.status = "success"
metrics_session.commit()
except Exception:
scan.status = "error"
metrics_session.commit()
def after_model_change(self, form, model, is_created):
# Discover tables and added data source
conf = Conf(datetime.utcnow())
db = model.get_db_object()
check_for_new_tables(db, conf)
create_dashboards()
def process_run():
run = Run.get_not_started_run()
if run is not None:
run.status = 'pending'
metrics_session.commit()
conf = Conf(run.for_date)
for source_db in DataSource.source_dbs():
run_check_for_new_tables(source_db, conf)
run_checks(source_db, conf)
run_compute_alerts(source_db, conf)
generate_grafana()
run.status = 'success'
metrics_session.commit()
def main():
parser = argparse.ArgumentParser(
description="Either create dashboards in grafana or run manual data ingestion of data metrics"
)
parser.add_argument(
"--grafana",
action="store_true",
help="Setup grafana dashboards, based on existing metrics",
)
parser.add_argument("--metrics", action="store_true", help="Push metrics to redata")
parser.add_argument(
"--generate-sample-data",
action="store_true",
help="Add sample data to REDATA DB for demonstration",
)
parser.add_argument(
"--create-admin-user", action="store_true", help="Generate admin user"
)
parser.add_argument(
"--backfill",
dest="backfill_days",
action="store",
nargs="?",
type=int,
help="Run backfill for last X days of metrics data",
)
args = parser.parse_args()
if not any(
(
args.grafana,
args.metrics,
args.backfill_days,
args.generate_sample_data,
args.create_admin_user,
)
):
print("No arugments supplied, write -h to get list of possible commands")
if args.grafana:
create_dashboards()
if args.metrics:
for db in DataSource.source_dbs():
print("run_check_for_new_table")
run_check_for_new_tables(db, Conf(datetime.utcnow()))
print("run_checks")
run_checks(db, Conf(datetime.utcnow()))
print("run alerts")
run_compute_alerts(db, Conf(datetime.utcnow()))
if args.generate_sample_data:
create_sample_tables_in_redata()
if args.backfill_days:
days = args.backfill_days
for db in DataSource.source_dbs():
run_check_for_new_tables(db, Conf(datetime.utcnow()))
past = datetime.utcnow() - timedelta(days=days)
while past <= datetime.utcnow():
run_checks(db, Conf(past))
run_compute_alerts(db, Conf(past))
past += timedelta(days=1)
if args.create_admin_user:
User.create_admin_user_if_not_exist()
def main():
parser = argparse.ArgumentParser(
description=
"Either create dashboards in grafana or run manual data ingestion of data metrics"
)
parser.add_argument(
"--grafana",
action="store_true",
help="Setup grafana dashboards, based on existing metrics")
parser.add_argument("--metrics",
action="store_true",
help="Push metrics to redata")
parser.add_argument("--generate-sample-data",
action="store_true",
help="Add sample data to REDATA DB for demonstration")
parser.add_argument("--backfill",
dest='backfill_days',
action="store",
nargs='?',
type=int,
help="Run backfill for last X days of metrics data")
args = parser.parse_args()
if not any((args.grafana, args.metrics, args.backfill_days,
args.generate_sample_data)):
print("Specify at least one of --grafana --metrics ")
if args.grafana:
create_dashboards()
if args.metrics:
for db in source_dbs:
print("run_check_for_new_table")
run_check_for_new_tables(db, Conf(datetime.utcnow()))
print("run_checks")
run_checks(db, Conf(datetime.utcnow()))
print("run alerts")
run_compute_alerts(db, Conf(datetime.utcnow()))
if args.generate_sample_data:
create_sample_tables_in_redata()
if args.backfill_days:
days = args.backfill_days
for db in source_dbs:
run_check_for_new_tables(db, Conf(datetime.utcnow()))
past = datetime.utcnow() - timedelta(days=days)
while past <= datetime.utcnow():
run_checks(db, Conf(past))
run_compute_alerts(db, Conf(past))
past += timedelta(days=1)
def run_compute_alerts_for_table(db, table, conf):
print (f"Checking alerts for table:{table.table_name} [BEGIN]")
check_alert.volume_alert(db, table, conf)
check_alert.delay_alert(db, table, conf)
check_alert.values_alert(db, table, conf)
print (f"Checking alerts for table:{table.table_name} [DONE]")
with DAG('validation_dag', description='Validate data',
schedule_interval=settings.REDATA_AIRFLOW_SCHEDULE_INTERVAL,
start_date=datetime(2017, 3, 20), catchup=False) as dag:
for source_db in source_dbs:
run_checks_op = PythonOperator(
task_id='run_checks_{}'.format(source_db.name),
python_callable=run_checks,
op_kwargs={'db': source_db, 'conf': Conf(datetime.utcnow())},
dag=dag
)
check_new_tables_op = PythonOperator(
task_id='run_check_for_new_tables_{}'.format(source_db.name),
python_callable=run_check_for_new_tables,
op_kwargs={'db': source_db, 'conf': Conf(datetime.utcnow())},
dag=dag
)
compute_alerts_op = PythonOperator(
task_id='compute_alerts_{}'.format(source_db.name),
python_callable=run_compute_alerts,
op_kwargs={'db': source_db, 'conf': Conf(datetime.utcnow())},
dag=dag