def process_run(): try: scan = Scan.get_not_started_run() if scan is not None: scan.status = "pending" metrics_session.commit() for_time = scan.start_date while for_time <= scan.end_date: conf = Conf(for_time) for source_db in DataSource.source_dbs(): run_check_for_new_tables(source_db, conf) run_checks(source_db, conf) run_compute_alerts(source_db, conf) for_time += timedelta(days=1) generate_grafana() scan.status = "success" metrics_session.commit() except Exception: scan.status = "error" metrics_session.commit()
def after_model_change(self, form, model, is_created): # Discover tables and added data source conf = Conf(datetime.utcnow()) db = model.get_db_object() check_for_new_tables(db, conf) create_dashboards()
def process_run(): run = Run.get_not_started_run() if run is not None: run.status = 'pending' metrics_session.commit() conf = Conf(run.for_date) for source_db in DataSource.source_dbs(): run_check_for_new_tables(source_db, conf) run_checks(source_db, conf) run_compute_alerts(source_db, conf) generate_grafana() run.status = 'success' metrics_session.commit()
def main(): parser = argparse.ArgumentParser( description="Either create dashboards in grafana or run manual data ingestion of data metrics" ) parser.add_argument( "--grafana", action="store_true", help="Setup grafana dashboards, based on existing metrics", ) parser.add_argument("--metrics", action="store_true", help="Push metrics to redata") parser.add_argument( "--generate-sample-data", action="store_true", help="Add sample data to REDATA DB for demonstration", ) parser.add_argument( "--create-admin-user", action="store_true", help="Generate admin user" ) parser.add_argument( "--backfill", dest="backfill_days", action="store", nargs="?", type=int, help="Run backfill for last X days of metrics data", ) args = parser.parse_args() if not any( ( args.grafana, args.metrics, args.backfill_days, args.generate_sample_data, args.create_admin_user, ) ): print("No arugments supplied, write -h to get list of possible commands") if args.grafana: create_dashboards() if args.metrics: for db in DataSource.source_dbs(): print("run_check_for_new_table") run_check_for_new_tables(db, Conf(datetime.utcnow())) print("run_checks") run_checks(db, Conf(datetime.utcnow())) print("run alerts") run_compute_alerts(db, Conf(datetime.utcnow())) if args.generate_sample_data: create_sample_tables_in_redata() if args.backfill_days: days = args.backfill_days for db in DataSource.source_dbs(): run_check_for_new_tables(db, Conf(datetime.utcnow())) past = datetime.utcnow() - timedelta(days=days) while past <= datetime.utcnow(): run_checks(db, Conf(past)) run_compute_alerts(db, Conf(past)) past += timedelta(days=1) if args.create_admin_user: User.create_admin_user_if_not_exist()
def main(): parser = argparse.ArgumentParser( description= "Either create dashboards in grafana or run manual data ingestion of data metrics" ) parser.add_argument( "--grafana", action="store_true", help="Setup grafana dashboards, based on existing metrics") parser.add_argument("--metrics", action="store_true", help="Push metrics to redata") parser.add_argument("--generate-sample-data", action="store_true", help="Add sample data to REDATA DB for demonstration") parser.add_argument("--backfill", dest='backfill_days', action="store", nargs='?', type=int, help="Run backfill for last X days of metrics data") args = parser.parse_args() if not any((args.grafana, args.metrics, args.backfill_days, args.generate_sample_data)): print("Specify at least one of --grafana --metrics ") if args.grafana: create_dashboards() if args.metrics: for db in source_dbs: print("run_check_for_new_table") run_check_for_new_tables(db, Conf(datetime.utcnow())) print("run_checks") run_checks(db, Conf(datetime.utcnow())) print("run alerts") run_compute_alerts(db, Conf(datetime.utcnow())) if args.generate_sample_data: create_sample_tables_in_redata() if args.backfill_days: days = args.backfill_days for db in source_dbs: run_check_for_new_tables(db, Conf(datetime.utcnow())) past = datetime.utcnow() - timedelta(days=days) while past <= datetime.utcnow(): run_checks(db, Conf(past)) run_compute_alerts(db, Conf(past)) past += timedelta(days=1)
def run_compute_alerts_for_table(db, table, conf): print (f"Checking alerts for table:{table.table_name} [BEGIN]") check_alert.volume_alert(db, table, conf) check_alert.delay_alert(db, table, conf) check_alert.values_alert(db, table, conf) print (f"Checking alerts for table:{table.table_name} [DONE]") with DAG('validation_dag', description='Validate data', schedule_interval=settings.REDATA_AIRFLOW_SCHEDULE_INTERVAL, start_date=datetime(2017, 3, 20), catchup=False) as dag: for source_db in source_dbs: run_checks_op = PythonOperator( task_id='run_checks_{}'.format(source_db.name), python_callable=run_checks, op_kwargs={'db': source_db, 'conf': Conf(datetime.utcnow())}, dag=dag ) check_new_tables_op = PythonOperator( task_id='run_check_for_new_tables_{}'.format(source_db.name), python_callable=run_check_for_new_tables, op_kwargs={'db': source_db, 'conf': Conf(datetime.utcnow())}, dag=dag ) compute_alerts_op = PythonOperator( task_id='compute_alerts_{}'.format(source_db.name), python_callable=run_compute_alerts, op_kwargs={'db': source_db, 'conf': Conf(datetime.utcnow())}, dag=dag