def source_dbs(cls):
if not getattr(cls, "source_db_objects", None):
sources = metrics_session.query(cls).all()
source_dbs = [s.get_db_object() for s in sources]
cls.source_db_objects = source_dbs
return cls.source_db_objects
def schema_changes(self):
return (
metrics_session.query(MetricFromCheck)
.filter_by(table_id=self.id, metric="schema_change")
.order_by("created_at")
.all()
)
def get_all_tables_per_namespace(cls, db_name, namespace):
return (
metrics_session.query(cls)
.filter(cls.source_db == db_name)
.filter(cls.namespace == namespace)
.all()
)
def get_monitored_tables(cls, db_name):
return (
metrics_session.query(cls)
.filter(cls.active == True)
.filter(cls.source_db == db_name)
.all()
)
def get_monitored_tables(cls, data_source):
return (
metrics_session.query(cls)
.filter(cls.active == True)
.filter(cls.data_source == data_source)
.all()
)
def last_records_added(self):
return (
metrics_session.query(MetricFromCheck)
.filter_by(table_id=self.id, metric="delay")
.order_by(MetricFromCheck.created_at.desc())
.limit(1)
.first()
)
def run_checks_for_table(db, table, conf):
checks = metrics_session.query(Check).filter(Check.table_id == table.id).all()
for check in checks:
query = check.query
if check.query["type"] == "standard":
func = get_function(check.query["path"])
result = func(
db=db, table=table, check=check, conf=conf, **check.query["params"]
)
elif check.query["type"] == "sql":
sql_code = check.query["sql"]
result = db.check_custom_query(check.table, sql_code, conf=conf)
MetricFromCheck.add_metrics(result, check, conf)
def create_admin_user_if_not_exist(cls):
assert os.environ.get(
'REDATA_ADMIN_USER'), 'please set env variable for admin user'
assert os.environ.get('REDATA_ADMIN_PASSWORD'
), 'please set env variable for admin password'
is_admin = metrics_session.query(cls).filter(
cls.login == os.environ.get('REDATA_ADMIN_USER')).count()
if not is_admin:
user = cls(login=os.environ.get('REDATA_ADMIN_USER'),
password=generate_password_hash(
os.environ.get('REDATA_ADMIN_PASSWORD')))
metrics_session.add(user)
metrics_session.commit()
print("Created admin user")
def run_checks_for_table(db, table, conf):
checks = metrics_session.query(Check).filter(
Check.table_id == table.id).all()
for check in checks:
query = check.query
if check.query['type'] == 'standard':
func = get_function(check.query['path'])
result = func(db=db,
table=table,
check=check,
conf=conf,
**check.query['params'])
else:
#TODO run raw query on DB
result = None
MetricFromCheck.add_metrics(result, check, conf)
def update_schema_for_table(cls, table, schema_cols):
table = metrics_session.query(cls).filter(
cls.table_name == table).first()
table.schema = {'columns': schema_cols}
metrics_session.commit()
def get_user(self):
return metrics_session.query(User).filter_by(
login=self.login.data).first()
def load_user(user_id):
return metrics_session.query(User).get(user_id)
def get_pending_run(cls):
return (metrics_session.query(cls).filter(cls.status == "pending")).first()
def get_not_started_run(cls):
return (metrics_session.query(cls).filter(cls.status == "not started")).first()
