def create(root_table: str):
try:
create_tree(root_table, load_global_config())
except CreationError as e:
send_slack_notification(e.message, f"Error while creating {e.table}")
except Exception as e:
send_slack_notification(str(e))
def setup_logger(name: str = "duro", stdout: bool = False) -> Logger:
if stdout:
return logzero.setup_logger(name=name, level=INFO)
path = load_global_config().logs_path
makedirs(path, exist_ok=True)
logfile = f"{path}/{name}.log"
return logzero.setup_logger(name=name,
logfile=logfile,
level=INFO,
maxBytes=1_000_000)
def set_table_for_update(db_connection,
table: str,
force_tree: int,
config_path="config.conf"):
if force_tree:
propagate_force_flag(db_connection, table,
load_global_config(config_path).graph)
else:
db_connection.execute(
"""
UPDATE tables
SET force = 1
WHERE table_name = ?
""",
(table, ),
)
db_connection.commit()
format_as_date,
format_interval,
skip_none,
format_job,
prepare_table_details,
)
from server.sqlite import (
get_all_tables,
get_jobs,
get_table_details,
set_table_for_update,
get_overview_stats,
)
from utils.global_config import load_global_config
DATABASE = load_global_config().db_path
app = Flask(__name__)
app.config.update({"DATABASE": DATABASE})
def before_request():
app.jinja_env.cache = {}
app.before_request(before_request)
app.config.update(DEBUG=True, TESTING=True, TEMPLATES_AUTO_RELOAD=True)
def get_db(use_rowfactory: bool = True):
db = getattr(g, "_database", None)
def update_version(connection, new_version: int):
connection.execute(
"""UPDATE version
SET major = ?, minor = ?""",
(new_version // 100, new_version % 100),
)
connection.commit()
def get_connection(db: str):
return sqlite3.connect(db)
def update_db(db: str):
logger = setup_logger("update_db")
connection = get_connection(db)
version = get_version(connection)
logger.info(f"Current db version: {version // 100}.{version % 100}")
new_updates = [upd for upd in updates if upd[0] > version]
for update in new_updates:
logger.info(f"Updating to {update[0] // 100}.{update[0] % 100}")
apply_update(connection, update[1])
update_version(connection, update[0])
logger.info(f"Updated to {update[0] // 100}.{update[0] % 100}")
logger.info("Table schema is up-to-date")
if __name__ == "__main__":
update_db(load_global_config().db_path)
import sqlite3
from server.sqlite import get_overview_stats
from notifications.slack import send_slack_notification
from utils.global_config import load_global_config
if __name__ == "__main__":
hours = 24
db = sqlite3.connect(load_global_config().db_path)
tables, updates, pct = get_overview_stats(db, hours)
db.close()
message = f"I’m working! " f"{tables} tables updated. " f"{updates} updates run during last {hours} hours. " f"{pct}% of time spent recreating views."
send_slack_notification(message, title="Duro", message_type="log")
from utils.errors import CreationError
from notifications.slack import send_slack_notification
from utils.global_config import load_global_config
def create(root_table: str):
try:
create_tree(root_table, load_global_config())
except CreationError as e:
send_slack_notification(e.message, f"Error while creating {e.table}")
except Exception as e:
send_slack_notification(str(e))
if __name__ == "__main__":
db_path = load_global_config().db_path
reset_all_starts(db_path)
while True:
new_tables = get_tables_to_create(db_path)
if not new_tables:
msg = "No tables in queue"
elif len(new_tables) == 1:
msg = "One table in queue"
else:
msg = f"{len(new_tables)} tables in queue"
print(f"{datetime.now()}: {msg}")
for t in new_tables:
create(t[0])
tables_and_configs = build_table_configs(graph, views_path)
check_config_fields(tables_and_configs, views_path)
new, updated = save_to_db(db_path, tables_and_configs, latest_commit)
message = build_notification_message(new, updated)
if use_git:
logger.info(f"Rescheduled for commit {latest_commit}. {message}")
else:
logger.info(f"Rescheduled. {message}")
if updated or new:
send_slack_notification(message,
"Rescheduled views",
message_type="success")
if __name__ == "__main__":
global_config = load_global_config()
try:
schedule(
global_config.views_path,
global_config.db_path,
strict=False,
use_git=global_config.use_git,
)
except SchedulerError as e:
send_slack_notification(str(e), "Scheduler error")
logger.error("Couldn‘t build a schedule for this views folder")