def run(self):
files = os.listdir(self.failed_dir)
for f in files:
logger.info("Retry to download {}...".format(f))
file_path = os.path.join(self.failed_dir, f)
with open(file_path, 'rb') as fp:
file = pickle.load(fp)
rds_instance = RDSInstance(
AcsClient(
self.config.get_accesskey_id(),
self.config.get_accesskey_secret(),
file.region_id
),
file.region_id,
file.instance_id,
)
file.set_rds_instance(rds_instance)
logger.info("File information has been extracted.")
if not file.backup(self.backup_dir):
# Remove pickle file if succeeded
os.remove(file_path)
self.succeeded_files.append(file)
else:
self.failed_files.append(file)
# Send backup report email
if (
len(self.succeeded_files) > 0 or
len(self.failed_files) > 0
):
self.postman.send_backup_report(
self.succeeded_files,
self.failed_files
)
def reset_download_url(self):
logger.info("Trying to refresh download url...")
if self.rds_instance is None:
logger.warning("RDS instance was not found")
return 1 # RDS instance was not found
start_time = self.start_time
end_time = self.start_time
if self.file_type == "binlog":
start_time -= timedelta(seconds=2)
end_time += timedelta(seconds=1)
if self.file_type == "full":
start_time -= timedelta(days=1)
end_time += timedelta(days=1)
backup_files = self.rds_instance.get_backup_files(
self.file_type,
start_time,
end_time
)
if not backup_files:
logger.warning("No backup file was found")
return 2 # Get none backup file
for backup_file in backup_files:
if (
backup_file.file_type == self.file_type and
backup_file.start_time == self.start_time and
backup_file.end_time == self.end_time
):
self.download_url = backup_file.get_download_url()
return 0
def get_fullbackup_files(self,
start_time,
end_time=None,
top=0,
dummy=False):
files = list()
request = DescribeBackupsRequest.DescribeBackupsRequest()
# Aliyun SDK works on this way: [Backup End Time]
# BETWEEN [search start time] AND [search end time].
# And full backup is taken once at most per day.
# So it is safe to add 1 day on last backup end time as
# current start time to avoid reundant downloading.
start_time += timedelta(days=1)
request.set_StartTime(start_time.strftime("%Y-%m-%dT00:00Z"))
if end_time:
search_end_time = end_time
else:
search_end_time = datetime.utcnow()
# Add 1 day to end time because it is exclusive in SDK
search_end_time += timedelta(days=1)
request.set_EndTime(search_end_time.strftime("%Y-%m-%dT00:00Z"))
request.set_DBInstanceId(self.instance_id)
request.set_PageSize(100)
read_record_cnt = 0
page_num = 1
while True:
request.set_PageNumber(page_num)
response = json.loads(
self.client.do_action_with_exception(request))
for bkp in response['Items']['Backup']:
download_url = bkp["BackupDownloadURL"]
host_id = bkp["HostInstanceID"]
file_status = 0 if bkp["BackupStatus"] == "Success" else 1
file_size = bkp["BackupSize"]
file_start_time = datetime.strptime(bkp["BackupStartTime"],
"%Y-%m-%dT%H:%M:%SZ")
file_end_time = datetime.strptime(bkp["BackupEndTime"],
"%Y-%m-%dT%H:%M:%SZ")
file = DBFile(download_url,
host_id,
self.region_id,
self.instance_id,
file_start_time,
file_end_time,
file_type='full',
file_status=file_status,
file_size=file_size)
if not dummy:
logger.info(file)
files.append(file)
read_record_cnt += response["PageRecordCount"]
page_num += 1
if ((top > 0 and read_record_cnt >= top)
or read_record_cnt >= response["TotalRecordCount"]):
break
return files
def get_backup_files(self, backup_type, start_time, end_time=None):
if backup_type == 'full':
logger.info("get_fullbackup_files('{}', '{}')".format(
start_time, end_time))
return self.get_fullbackup_files(start_time, end_time)
elif backup_type == 'binlog':
logger.info("get_binlog_files('{}', '{}')".format(
start_time, end_time))
return self.get_binlog_files(start_time, end_time)
else:
return None
def download_db_files(self, instance, rds_instance, backup_dir,
backup_type):
if self.scheduler.is_triggered_now(
self.config.get_schedule(instance, backup_type)):
last_bkp_time = self.config.get_last_backup_time(
instance, backup_type)
bkp_files = rds_instance.get_backup_files(backup_type,
start_time=last_bkp_time)
logger.info("Backup files information has been collected.")
bkp_files.sort(key=lambda f: f.get_end_time())
for f in bkp_files:
curr_bkp_time = f.get_end_time()
if curr_bkp_time > last_bkp_time:
last_bkp_time = curr_bkp_time
if f.backup(backup_dir) == 0:
self.succeeded_files.append(f)
else:
self.failed_files.append(f)
self.config.set_last_bkp_time(instance, backup_type,
last_bkp_time)
# Update last backup time in config file
self.config.update_config_file()
def search_binlog_files(self, host_id, start_time, end_time=None):
files = list()
# Aliyun SDK works on this way: [Backup End Time]
# BETWEEN [search start time] AND [search end time].
# So it is safe to add 1 sec on last backup end time as
# current start time to avoid reundant downloading.
start_time += timedelta(seconds=1)
request = DescribeBinlogFilesRequest.DescribeBinlogFilesRequest()
request.set_StartTime(start_time.strftime("%Y-%m-%dT%H:%M:%SZ"))
if end_time:
request.set_EndTime(end_time.strftime("%Y-%m-%dT%H:%M:%SZ"))
else:
request.set_EndTime(
datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"))
request.set_DBInstanceId(self.instance_id)
request.set_PageSize(100)
read_record_cnt = 0
page_num = 1
while True:
request.set_PageNumber(page_num)
response = json.loads(
self.client.do_action_with_exception(request))
for binlog in response['Items']['BinLogFile']:
if binlog['HostInstanceID'] == host_id:
download_url = binlog['DownloadLink']
file_size = binlog['FileSize']
checksum = binlog['Checksum']
file_start_time = datetime.strptime(
binlog['LogBeginTime'], "%Y-%m-%dT%H:%M:%SZ")
file_end_time = datetime.strptime(binlog['LogEndTime'],
"%Y-%m-%dT%H:%M:%SZ")
file = DBFile(download_url,
host_id,
self.region_id,
self.instance_id,
file_start_time,
file_end_time,
file_type='binlog',
file_size=file_size,
checksum=checksum)
logger.info(file)
files.append(file)
read_record_cnt += response["PageRecordCount"]
logger.info("{} records has been read".format(
response["PageRecordCount"]))
logger.info("Total records:{}".format(
response['TotalRecordCount']))
page_num += 1
if read_record_cnt >= response['TotalRecordCount']:
break
return files
def download(self, dest_file, retry=5, download_timeout=7200):
rest = retry
while rest > 0:
try:
logger.info("Start downloading - {}".format(self.download_url))
error_count = 0
response = requests.get(self.download_url, timeout=30, stream=True)
content_type = response.headers.get('content-type')
if content_type == "application/xml": # Download link is expired
rest -= 1
logger.info("Download link is not valid any more.")
if self.reset_download_url() > 0:
error_count += 1
logger.error("Failed to reset download url! Retry after 5 seconds...")
else:
with open(dest_file, 'wb') as f:
download_start = datetime.now()
# Download 10MiB per chunk
for chunk in response.iter_content(1024 * 1024 * 10):
f.write(chunk)
elapsed = datetime.now() - download_start
if elapsed.seconds > download_timeout:
logger.error("Download timeout! Retry after 5 seconds...")
time.sleep(5)
error_count += 1
rest -= 1
break
except Exception as e:
logger.error("An error occurred when downloading! Retry after 20 seconds...")
rest -= 1
time.sleep(20) # Retry after some seconds
else:
if error_count == 0:
logger.info("Downloaded successfully - {}".format(dest_file))
return 0
logger.info("Fail to download - {}".format(dest_file))
return 1