def compress(file_path: Path) -> tuple[typing.IO, bool]:
"""
Compress the file using gzip into a tempfile.TemporaryFile.
Returns a two elements tuple.
The first one is a file-like object, which is removed when it is closed.
The second one is True if the file has been recompressed, False otherwise.
This should always be called before syncing the
file to a storage provider.
"""
# If the file is already considered as an archive, we don't recompress it
if file_path.suffix in ARCHIVE_SUFFIXES:
log.debug(f"File {file_path.name} is already compressed.")
return open(file_path, "rb"), False
temp_file = tempfile.NamedTemporaryFile(suffix=f"-{file_path.name}")
log.debug(f"Compressing to temporary file: {temp_file.name}")
with file_path.open(mode="rb") as f_in:
with gzip.open(temp_file, "wb") as f_out:
shutil.copyfileobj(f_in, f_out)
temp_file.seek(0)
return temp_file, True
def backup(self, backup_path) -> None:
"""Dump all the data to a file and then return the filepath."""
# Run the backup, and store the outcome.
self.success, self.output = run_command(
"redis-cli "
f"-h {self.config.get('host')} "
f"-p {str(self.config.get('port', '6379'))} "
f"--rdb {backup_path}",
REDISCLI_AUTH=self.config.get("password"))
log.debug(self.output)
def backup(self, backup_path) -> None:
"""Dump all the data to a file and then return the filepath."""
# Run the backup, and store the outcome.
self.success, self.output = run_command(
f"pg_dumpall --file={backup_path}",
PGUSER=self.config["username"],
PGPASSWORD=self.config["password"],
PGHOST=self.config["host"],
PGPORT=str(self.config.get("port", "5432")),
)
log.debug(self.output)
def backup(self, backup_path) -> None:
"""Dump all the data to a file and then return the filepath."""
# Run the backup, and store the outcome in this object.
self.success, self.output = run_command(
f"mongodump "
f"--uri={self.config['connection_string']} "
"--gzip "
"--forceTableScan "
f"--archive={backup_path}"
)
log.debug(self.output)
def notify(self):
"""Convert report dict to string and send via Telegram."""
bot = telebot.TeleBot(self.config["token"])
try:
bot.send_message(
chat_id=self.config["chat_id"],
text=self._parse_report(),
)
except ApiTelegramException:
log.debug(
"Telegram API key or user_id is wrong "
"or you forgot to press /start in your bot."
)
def is_on_cooldown(cooldown) -> bool:
"""Check if we can send notification, main function."""
log.debug(f'Cooldown period set to {cooldown}')
delta = parse_config_cooldown(cooldown)
if os.path.exists(get_project_root() / 'notify.json'):
data = read_config()
last_send = datetime.strptime(data['last_send'], "%m/%d/%Y, %H:%M:%S")
if should_notify(last_send, delta):
write_config()
return False
log.debug('On cooldown')
return True
else:
write_config()
return False
def backup(self) -> Path:
"""Dump all the data to a file and then return the filepath."""
date = datetime.date.today().strftime("%d_%m_%Y")
archive_file = Path.home() / f"mongodb_blackbox_{date}.archive"
# Run the backup, and store the outcome in this object.
self.success, self.output = run_command(f"mongodump "
f"--uri={self.connstring} "
"--gzip "
"--forceTableScan "
f"--archive={archive_file}")
log.debug(self.output)
# Return the path to the backup file
return archive_file
def sync(self, file_path: Path) -> None:
"""Sync a file to Dropbox."""
# Check if Dropbox token is valid.
if self.valid is False:
error = "Dropbox token is invalid!"
self.success = False
self.output = error
log.error(error)
return None
# This is size what can be uploaded as one chunk.
# When file is bigger than that, this will be uploaded
# in multiple parts.
chunk_size = 4 * 1024 * 1024
temp_file, recompressed = self.compress(file_path)
upload_path = f"{self.upload_base}{file_path.name}{'.gz' if recompressed else ''}"
try:
with temp_file as f:
file_size = os.stat(f.name).st_size
log.debug(file_size)
if file_size <= chunk_size:
self.client.files_upload(f.read(), upload_path,
WriteMode.overwrite)
else:
session_start = self.client.files_upload_session_start(
f.read(chunk_size))
cursor = UploadSessionCursor(session_start.session_id,
offset=f.tell())
# Commit contains path in Dropbox and write mode about file
commit = CommitInfo(upload_path, WriteMode.overwrite)
while f.tell() < file_size:
if (file_size - f.tell()) <= chunk_size:
self.client.files_upload_session_finish(
f.read(chunk_size), cursor, commit)
else:
self.client.files_upload_session_append(
f.read(chunk_size), cursor.session_id,
cursor.offset)
cursor.offset = f.tell()
self.success = True
except (ApiError, HttpError) as e:
log.error(e)
self.success = False
self.output = str(e)
def backup(self, backup_path) -> None:
"""Dump all the data to a file and then return the filepath."""
user = self.config["username"]
password = self.config["password"]
host = self.config["host"]
port = str(self.config.get("port", "3306"))
# Run the backup, and store the outcome.
self.success, self.output = run_command(
f"mysqldump -h {host} -u {user} --password='******' "
f"--port={port} --all-databases > {backup_path}")
log.debug(self.output)
# Explicitly check if error message is occurred.
# Somehow mysqldump is always successful.
if "error" in self.output.lower():
self.success = False
log.debug("mysqldump has error(s) in log")
def backup(self) -> Path:
"""Dump all the data to a file and then return the filepath."""
date = datetime.date.today().strftime("%d_%m_%Y")
backup_path = Path.home() / f"postgres_blackbox_{date}.sql"
# Run the backup, and store the outcome.
self.success, self.output = run_command(
f"pg_dumpall --file={backup_path}",
PGUSER=self.config.get("user"),
PGPASSWORD=self.config.get("password"),
PGHOST=self.config.get("host"),
PGPORT=self.config.get("port"),
)
log.debug(self.output)
# Return the path to the backup file
return backup_path
def read_config() -> dict:
"""Read last notification time."""
log.debug('Found json, reading it...')
with open(get_project_root() / 'notify.json') as infile:
data = json.load(infile)
return data
def write_config():
"""Write down successful notification."""
data = {'last_send': datetime.now().strftime("%m/%d/%Y, %H:%M:%S")}
log.debug(f"Sending notification at {data}")
with open(get_project_root() / 'notify.json', 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=4)
def run() -> bool:
"""
Implement the main blackbox process.
Returns whether or not the process is a success.
"""
# Overall program success
success = True
# Parse configuration and instantiate handlers
if not CONFIG.databases or not CONFIG.storage:
raise exceptions.ImproperlyConfigured(
"You have to define least one database and storage")
database_handlers = workflows.get_configured_handlers(CONFIG.databases)
storage_handlers = workflows.get_configured_handlers(CONFIG.storage)
notifier_handlers = workflows.get_configured_handlers(CONFIG.notifiers)
all_workflows = workflows.get_workflows(database_handlers,
storage_handlers,
notifier_handlers)
with TemporaryDirectory() as backup_dir:
log.info(f"Backing up to folder: {backup_dir}")
backup_dir = Path(backup_dir)
date = datetime.date.today().strftime("%d_%m_%Y")
backup_files = []
for workflow in all_workflows:
database = workflow.database
# Do a backup, then return the path to the backup file
backup_filename = f"{database.config['id']}_blackbox_{date}{database.backup_extension}"
backup_path = backup_dir / backup_filename
database.backup(backup_path)
backup_files.append(backup_path)
database_id = database.get_id_for_retention()
database.teardown()
# Add report to notifiers
report = DatabaseReport(database.config["id"], database.success,
database.output)
for notifier in workflow.notifiers:
notifier.add_database(report)
# If backup failed, continue to next database. No need to sync.
if not database.success:
continue
for storage in workflow.storage_providers:
# Sync the provider, then rotate and cleanup
storage.sync(backup_path)
storage.rotate(database_id)
storage.teardown()
# Store the outcome to the database report
report.report_storage(storage.config["id"], storage.success,
storage.output)
# Set overall program success to False if workflow is unsuccessful
if report.success is False:
success = False
cooldown = CONFIG['cooldown']
logging.debug(f"Cooldown setting is {cooldown}")
if cooldown:
is_on_cooldown_ = is_on_cooldown(cooldown)
# Send a report for each notifier configured
for notifier in notifier_handlers["all"]:
# Don't send a notification if no database uses the notifier
if notifier.report.is_empty:
continue
# If cooldown is not set or if report is failed: just notify.
if cooldown is None or not notifier.report.success:
log.debug(
'Config not found or backup failed, sending notification.')
notifier.notify()
# But otherwise let's check do we have a right to notify
else:
if not is_on_cooldown_:
notifier.notify()
notifier.teardown()
return success
