def run(self): files = os.listdir(self.failed_dir) for f in files: logger.info("Retry to download {}...".format(f)) file_path = os.path.join(self.failed_dir, f) with open(file_path, 'rb') as fp: file = pickle.load(fp) rds_instance = RDSInstance( AcsClient( self.config.get_accesskey_id(), self.config.get_accesskey_secret(), file.region_id ), file.region_id, file.instance_id, ) file.set_rds_instance(rds_instance) logger.info("File information has been extracted.") if not file.backup(self.backup_dir): # Remove pickle file if succeeded os.remove(file_path) self.succeeded_files.append(file) else: self.failed_files.append(file) # Send backup report email if ( len(self.succeeded_files) > 0 or len(self.failed_files) > 0 ): self.postman.send_backup_report( self.succeeded_files, self.failed_files )
def reset_download_url(self): logger.info("Trying to refresh download url...") if self.rds_instance is None: logger.warning("RDS instance was not found") return 1 # RDS instance was not found start_time = self.start_time end_time = self.start_time if self.file_type == "binlog": start_time -= timedelta(seconds=2) end_time += timedelta(seconds=1) if self.file_type == "full": start_time -= timedelta(days=1) end_time += timedelta(days=1) backup_files = self.rds_instance.get_backup_files( self.file_type, start_time, end_time ) if not backup_files: logger.warning("No backup file was found") return 2 # Get none backup file for backup_file in backup_files: if ( backup_file.file_type == self.file_type and backup_file.start_time == self.start_time and backup_file.end_time == self.end_time ): self.download_url = backup_file.get_download_url() return 0
def get_fullbackup_files(self, start_time, end_time=None, top=0, dummy=False): files = list() request = DescribeBackupsRequest.DescribeBackupsRequest() # Aliyun SDK works on this way: [Backup End Time] # BETWEEN [search start time] AND [search end time]. # And full backup is taken once at most per day. # So it is safe to add 1 day on last backup end time as # current start time to avoid reundant downloading. start_time += timedelta(days=1) request.set_StartTime(start_time.strftime("%Y-%m-%dT00:00Z")) if end_time: search_end_time = end_time else: search_end_time = datetime.utcnow() # Add 1 day to end time because it is exclusive in SDK search_end_time += timedelta(days=1) request.set_EndTime(search_end_time.strftime("%Y-%m-%dT00:00Z")) request.set_DBInstanceId(self.instance_id) request.set_PageSize(100) read_record_cnt = 0 page_num = 1 while True: request.set_PageNumber(page_num) response = json.loads( self.client.do_action_with_exception(request)) for bkp in response['Items']['Backup']: download_url = bkp["BackupDownloadURL"] host_id = bkp["HostInstanceID"] file_status = 0 if bkp["BackupStatus"] == "Success" else 1 file_size = bkp["BackupSize"] file_start_time = datetime.strptime(bkp["BackupStartTime"], "%Y-%m-%dT%H:%M:%SZ") file_end_time = datetime.strptime(bkp["BackupEndTime"], "%Y-%m-%dT%H:%M:%SZ") file = DBFile(download_url, host_id, self.region_id, self.instance_id, file_start_time, file_end_time, file_type='full', file_status=file_status, file_size=file_size) if not dummy: logger.info(file) files.append(file) read_record_cnt += response["PageRecordCount"] page_num += 1 if ((top > 0 and read_record_cnt >= top) or read_record_cnt >= response["TotalRecordCount"]): break return files
def get_backup_files(self, backup_type, start_time, end_time=None): if backup_type == 'full': logger.info("get_fullbackup_files('{}', '{}')".format( start_time, end_time)) return self.get_fullbackup_files(start_time, end_time) elif backup_type == 'binlog': logger.info("get_binlog_files('{}', '{}')".format( start_time, end_time)) return self.get_binlog_files(start_time, end_time) else: return None
def download_db_files(self, instance, rds_instance, backup_dir, backup_type): if self.scheduler.is_triggered_now( self.config.get_schedule(instance, backup_type)): last_bkp_time = self.config.get_last_backup_time( instance, backup_type) bkp_files = rds_instance.get_backup_files(backup_type, start_time=last_bkp_time) logger.info("Backup files information has been collected.") bkp_files.sort(key=lambda f: f.get_end_time()) for f in bkp_files: curr_bkp_time = f.get_end_time() if curr_bkp_time > last_bkp_time: last_bkp_time = curr_bkp_time if f.backup(backup_dir) == 0: self.succeeded_files.append(f) else: self.failed_files.append(f) self.config.set_last_bkp_time(instance, backup_type, last_bkp_time) # Update last backup time in config file self.config.update_config_file()
def search_binlog_files(self, host_id, start_time, end_time=None): files = list() # Aliyun SDK works on this way: [Backup End Time] # BETWEEN [search start time] AND [search end time]. # So it is safe to add 1 sec on last backup end time as # current start time to avoid reundant downloading. start_time += timedelta(seconds=1) request = DescribeBinlogFilesRequest.DescribeBinlogFilesRequest() request.set_StartTime(start_time.strftime("%Y-%m-%dT%H:%M:%SZ")) if end_time: request.set_EndTime(end_time.strftime("%Y-%m-%dT%H:%M:%SZ")) else: request.set_EndTime( datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")) request.set_DBInstanceId(self.instance_id) request.set_PageSize(100) read_record_cnt = 0 page_num = 1 while True: request.set_PageNumber(page_num) response = json.loads( self.client.do_action_with_exception(request)) for binlog in response['Items']['BinLogFile']: if binlog['HostInstanceID'] == host_id: download_url = binlog['DownloadLink'] file_size = binlog['FileSize'] checksum = binlog['Checksum'] file_start_time = datetime.strptime( binlog['LogBeginTime'], "%Y-%m-%dT%H:%M:%SZ") file_end_time = datetime.strptime(binlog['LogEndTime'], "%Y-%m-%dT%H:%M:%SZ") file = DBFile(download_url, host_id, self.region_id, self.instance_id, file_start_time, file_end_time, file_type='binlog', file_size=file_size, checksum=checksum) logger.info(file) files.append(file) read_record_cnt += response["PageRecordCount"] logger.info("{} records has been read".format( response["PageRecordCount"])) logger.info("Total records:{}".format( response['TotalRecordCount'])) page_num += 1 if read_record_cnt >= response['TotalRecordCount']: break return files
def download(self, dest_file, retry=5, download_timeout=7200): rest = retry while rest > 0: try: logger.info("Start downloading - {}".format(self.download_url)) error_count = 0 response = requests.get(self.download_url, timeout=30, stream=True) content_type = response.headers.get('content-type') if content_type == "application/xml": # Download link is expired rest -= 1 logger.info("Download link is not valid any more.") if self.reset_download_url() > 0: error_count += 1 logger.error("Failed to reset download url! Retry after 5 seconds...") else: with open(dest_file, 'wb') as f: download_start = datetime.now() # Download 10MiB per chunk for chunk in response.iter_content(1024 * 1024 * 10): f.write(chunk) elapsed = datetime.now() - download_start if elapsed.seconds > download_timeout: logger.error("Download timeout! Retry after 5 seconds...") time.sleep(5) error_count += 1 rest -= 1 break except Exception as e: logger.error("An error occurred when downloading! Retry after 20 seconds...") rest -= 1 time.sleep(20) # Retry after some seconds else: if error_count == 0: logger.info("Downloaded successfully - {}".format(dest_file)) return 0 logger.info("Fail to download - {}".format(dest_file)) return 1