def make_database_backup(self, database, task):
worker_name = get_worker_name()
task_history = TaskHistory.register(
request=self.request, worker_name=worker_name, task_history=task
)
if not database.pin_task(task):
task.error_in_lock(database)
return False
task_history.add_detail('Starting database {} backup'.format(database))
instances = _get_backup_instance(database, task)
if not instances:
task.set_status_error('Could not find eligible instances', database)
return False
_check_snapshot_limit(instances, task)
group = BackupGroup()
group.save()
has_warning = False
for instance in instances:
snapshot = _create_database_backup(instance, task, group)
if not snapshot:
task.set_status_error(
'Backup was unsuccessful in {}'.format(instance), database
)
return False
snapshot.is_automatic = False
snapshot.save()
if not has_warning:
has_warning = snapshot.has_warning
if has_warning:
task.set_status_warning('Backup was warning', database)
else:
task.set_status_success('Backup was successful', database)
return True
def make_database_backup(self, database, task):
worker_name = get_worker_name()
task_history = TaskHistory.register(
request=self.request, worker_name=worker_name, task_history=task
)
if not database.pin_task(task):
task.error_in_lock(database)
return False
task_history.add_detail('Starting database {} backup'.format(database))
instances = _get_backup_instance(database, task)
if not instances:
task.set_status_error('Could not find eligible instances', database)
return False
_check_snapshot_limit(instances, task)
group = BackupGroup()
group.save()
has_warning = False
for instance in instances:
snapshot = _create_database_backup(instance, task, group)
if not snapshot:
task.set_status_error(
'Backup was unsuccessful in {}'.format(instance), database
)
return False
snapshot.is_automatic = False
snapshot.save()
if not has_warning:
has_warning = snapshot.has_warning
if has_warning:
task.set_status_warning('Backup was warning', database)
else:
task.set_status_success('Backup was successful', database)
return True
def make_databases_backup(self):
LOG.info("Making databases backups")
worker_name = get_worker_name()
task_history = TaskHistory.register(
request=self.request, worker_name=worker_name, user=None
)
task_history.relevance = TaskHistory.RELEVANCE_ERROR
backup_group_interval = Configuration.get_by_name_as_int(
'backup_group_interval', default=1
)
waiting_msg = "\nWaiting {} minute(s) to start the next backup group".format(
backup_group_interval
)
status = TaskHistory.STATUS_SUCCESS
environments = Environment.objects.all()
prod_envs = Environment.prod_envs()
dev_envs = Environment.dev_envs()
env_names_order = list(prod_envs) + list(dev_envs)
if not env_names_order:
env_names_order = [env.name for env in environments]
current_time = datetime.now()
current_hour = current_time.hour
# Get all infras with a backup today until the current hour
infras_with_backup_today = DatabaseInfra.objects.filter(
instances__backup_instance__status=Snapshot.SUCCESS,
backup_hour__lt=current_hour,
plan__has_persistence=True,
instances__backup_instance__end_at__year=current_time.year,
instances__backup_instance__end_at__month=current_time.month,
instances__backup_instance__end_at__day=current_time.day).distinct()
# Get all infras with pending backups based on infras_with_backup_today
infras_pending_backup = DatabaseInfra.objects.filter(
backup_hour__lt=current_hour,
plan__has_persistence=True,
).exclude(pk__in=[infra.pk for infra in infras_with_backup_today])
# Get all infras to backup on the current hour
infras_current_hour = DatabaseInfra.objects.filter(
plan__has_persistence=True,
backup_hour=current_time.hour
)
# Merging pending and current infras to backup list
infras = infras_current_hour | infras_pending_backup
for env_name in env_names_order:
try:
env = environments.get(name=env_name)
except Environment.DoesNotExist:
continue
msg = '\nStarting Backup for env {}'.format(env.name)
task_history.update_details(persist=True, details=msg)
databaseinfras_by_env = infras.filter(environment=env)
error = {}
backup_number = 0
backups_per_group = len(infras) / 12
for infra in databaseinfras_by_env:
if not infra.databases.first():
continue
if backups_per_group > 0:
if backup_number < backups_per_group:
backup_number += 1
else:
backup_number = 0
task_history.update_details(waiting_msg, True)
sleep(backup_group_interval*60)
group = BackupGroup()
group.save()
instances_backup = infra.instances.filter(
read_only=False, is_active=True
)
for instance in instances_backup:
try:
driver = instance.databaseinfra.get_driver()
is_eligible = driver.check_instance_is_eligible_for_backup(
instance
)
if not is_eligible:
LOG.info(
'Instance {} is not eligible for backup'.format(
instance
)
)
continue
except Exception as e:
status = TaskHistory.STATUS_ERROR
msg = "Backup for %s was unsuccessful. Error: %s" % (
str(instance), str(e))
LOG.error(msg)
time_now = str(strftime("%m/%d/%Y %H:%M:%S"))
start_msg = "\n{} - Starting backup for {} ...".format(
time_now, instance
)
task_history.update_details(persist=True, details=start_msg)
try:
snapshot = make_instance_snapshot_backup(
instance=instance, error=error, group=group,
current_hour=current_hour
)
if snapshot and snapshot.was_successful:
msg = "Backup for %s was successful" % (str(instance))
LOG.info(msg)
elif snapshot and snapshot.was_error:
status = TaskHistory.STATUS_ERROR
msg = "Backup for %s was unsuccessful. Error: %s" % (
str(instance), error['errormsg'])
LOG.error(msg)
else:
status = TaskHistory.STATUS_WARNING
msg = "Backup for %s has warning" % (str(instance))
LOG.info(msg)
LOG.info(msg)
except Exception as e:
status = TaskHistory.STATUS_ERROR
msg = "Backup for %s was unsuccessful. Error: %s" % (
str(instance), str(e))
LOG.error(msg)
time_now = str(strftime("%m/%d/%Y %H:%M:%S"))
msg = "\n{} - {}".format(time_now, msg)
task_history.update_details(persist=True, details=msg)
task_history.update_status_for(status, details="\nBackup finished")
return
def make_databases_backup(self):
LOG.info("Making databases backups")
worker_name = get_worker_name()
task_history = TaskHistory.register(request=self.request,
worker_name=worker_name,
user=None)
task_history.relevance = TaskHistory.RELEVANCE_ERROR
waiting_msg = "\nWaiting 5 minutes to start the next backup group"
status = TaskHistory.STATUS_SUCCESS
environments = Environment.objects.all()
prod_envs = Configuration.get_by_name_as_list('prod_envs')
dev_envs = Configuration.get_by_name_as_list('dev_envs')
env_names_order = prod_envs + dev_envs
if not env_names_order:
env_names_order = [env.name for env in environments]
infras = DatabaseInfra.objects.filter(plan__has_persistence=True)
for env_name in env_names_order:
try:
env = environments.get(name=env_name)
except Environment.DoesNotExist:
continue
msg = '\nStarting Backup for env {}'.format(env.name)
task_history.update_details(persist=True, details=msg)
databaseinfras_by_env = infras.filter(environment=env)
error = {}
backup_number = 0
backups_per_group = len(infras) / 12
for infra in databaseinfras_by_env:
if not infra.databases.first():
continue
if backups_per_group > 0:
if backup_number < backups_per_group:
backup_number += 1
else:
backup_number = 0
task_history.update_details(waiting_msg, True)
sleep(300)
group = BackupGroup()
group.save()
for instance in infra.instances.filter(read_only=False):
try:
driver = instance.databaseinfra.get_driver()
is_eligible = driver.check_instance_is_eligible_for_backup(
instance)
if not is_eligible:
LOG.info(
'Instance {} is not eligible for backup'.format(
instance))
continue
except Exception as e:
status = TaskHistory.STATUS_ERROR
msg = "Backup for %s was unsuccessful. Error: %s" % (
str(instance), str(e))
LOG.error(msg)
time_now = str(strftime("%m/%d/%Y %H:%M:%S"))
start_msg = "\n{} - Starting backup for {} ...".format(
time_now, instance)
task_history.update_details(persist=True, details=start_msg)
try:
snapshot = make_instance_snapshot_backup(instance=instance,
error=error,
group=group)
if snapshot and snapshot.was_successful:
msg = "Backup for %s was successful" % (str(instance))
LOG.info(msg)
elif snapshot and snapshot.has_warning:
status = TaskHistory.STATUS_WARNING
msg = "Backup for %s has warning" % (str(instance))
LOG.info(msg)
else:
status = TaskHistory.STATUS_ERROR
msg = "Backup for %s was unsuccessful. Error: %s" % (
str(instance), error['errormsg'])
LOG.error(msg)
LOG.info(msg)
except Exception as e:
status = TaskHistory.STATUS_ERROR
msg = "Backup for %s was unsuccessful. Error: %s" % (
str(instance), str(e))
LOG.error(msg)
time_now = str(strftime("%m/%d/%Y %H:%M:%S"))
msg = "\n{} - {}".format(time_now, msg)
task_history.update_details(persist=True, details=msg)
task_history.update_status_for(status, details="\nBackup finished")
return
def make_databases_backup(self):
LOG.info("Making databases backups")
worker_name = get_worker_name()
task_history = TaskHistory.register(
request=self.request, worker_name=worker_name, user=None
)
task_history.relevance = TaskHistory.RELEVANCE_ERROR
waiting_msg = "\nWaiting 5 minutes to start the next backup group"
status = TaskHistory.STATUS_SUCCESS
environments = Environment.objects.all()
prod_envs = Configuration.get_by_name_as_list('prod_envs')
dev_envs = Configuration.get_by_name_as_list('dev_envs')
env_names_order = prod_envs + dev_envs
if not env_names_order:
env_names_order = [env.name for env in environments]
infras = DatabaseInfra.objects.filter(plan__has_persistence=True)
for env_name in env_names_order:
try:
env = environments.get(name=env_name)
except Environment.DoesNotExist:
continue
msg = '\nStarting Backup for env {}'.format(env.name)
task_history.update_details(persist=True, details=msg)
databaseinfras_by_env = infras.filter(environment=env)
error = {}
backup_number = 0
backups_per_group = len(infras) / 12
for infra in databaseinfras_by_env:
if not infra.databases.first():
continue
if backups_per_group > 0:
if backup_number < backups_per_group:
backup_number += 1
else:
backup_number = 0
task_history.update_details(waiting_msg, True)
sleep(300)
group = BackupGroup()
group.save()
for instance in infra.instances.filter(read_only=False):
try:
driver = instance.databaseinfra.get_driver()
is_eligible = driver.check_instance_is_eligible_for_backup(
instance
)
if not is_eligible:
LOG.info(
'Instance {} is not eligible for backup'.format(
instance
)
)
continue
except Exception as e:
status = TaskHistory.STATUS_ERROR
msg = "Backup for %s was unsuccessful. Error: %s" % (
str(instance), str(e))
LOG.error(msg)
time_now = str(strftime("%m/%d/%Y %H:%M:%S"))
start_msg = "\n{} - Starting backup for {} ...".format(
time_now, instance
)
task_history.update_details(persist=True, details=start_msg)
try:
snapshot = make_instance_snapshot_backup(
instance=instance, error=error, group=group
)
if snapshot and snapshot.was_successful:
msg = "Backup for %s was successful" % (str(instance))
LOG.info(msg)
elif snapshot and snapshot.has_warning:
status = TaskHistory.STATUS_WARNING
msg = "Backup for %s has warning" % (str(instance))
LOG.info(msg)
else:
status = TaskHistory.STATUS_ERROR
msg = "Backup for %s was unsuccessful. Error: %s" % (
str(instance), error['errormsg'])
LOG.error(msg)
LOG.info(msg)
except Exception as e:
status = TaskHistory.STATUS_ERROR
msg = "Backup for %s was unsuccessful. Error: %s" % (
str(instance), str(e))
LOG.error(msg)
time_now = str(strftime("%m/%d/%Y %H:%M:%S"))
msg = "\n{} - {}".format(time_now, msg)
task_history.update_details(persist=True, details=msg)
task_history.update_status_for(status, details="\nBackup finished")
return