def backup(config, logger=None):
"""
@param config : an array with the configuration
"""
backup_dir = datetime.now().isoformat(' ')
_backup_size = backup_size(config)
if _backup_size > config.get('max_size'):
raise Exception("The backup has a greater size than allowed !!")
# informations
user = config.get('server:user')
password = config.get('server:password')
# Connect to the FTP server backup
ftp = Ftp(config.get('server:host'))
ftp.login(user, password)
root = config.get('server:backup_root')
if root != '/' and not ftp.exists(root):
ftp.mkdirs(root)
#ftp.clear(root)
ftp.cwd(root)
while max_versions_reached(ftp, config):
delete_oldest_version(ftp, config, logger)
while backups_total_size(ftp, root, config) + _backup_size > config.get('max_size'):
delete_oldest_version(ftp, config, logger)
# create the backup directory and go to it
ftp.mkd(backup_dir)
ftp.cwd(backup_dir)
#upload bk_file
f = tempfile.TemporaryFile(mode='w+b')
f.write("%d" % (_backup_size))
f.seek(0)
ftp.storbinary("STOR %s" % (config.get('backup_size_file')), f)
f.close()
# Get file or dirs to upload
files = config.get('files')
# Get separators
server_separator = config.get('server:separator')
local_separator = os.sep
for f in files:
local_file = f['path']
logger.info("Save %s" % (local_file))
# ensure parent directory exists
local_parent, filename = os.path.split(local_file)
ftp.mkdirs(local_parent, server_separator)
# remote parent is the same as local parent but without the beginning slash
remote_parent = ftp.normpath(local_separator.join((root, backup_dir, local_parent[1:])))
tar_files = config.has('default:tar') and config.get('default:tar') == True
if 'tar' in f:
tar_files = (f['tar'] == True)
if tar_files:
tmp_dir = tempfile.gettempdir()
tar_filename = "%s.tar" % (filename)
tar_path = os.path.join(tmp_dir, tar_filename)
logger.debug("Create Tar file in temp directory : %s" % (tar_path))
tar = tarfile.open(tar_path, "w")
tar.add(os.path.join(local_parent, filename))
tar.close()
#for info in tar:
#print info
upload(ftp, tmp_dir, remote_parent, tar_filename, separator=server_separator, logger=logger)
logger.debug("Remove %s" % (tar_path))
os.remove(tar_path)
else:
# upload files
upload(ftp, local_parent, remote_parent, filename, separator=server_separator, logger=logger)
# close the connection
ftp.close()
from ftp import Ftp
import traceback
import datetime
ftp = Ftp('192.168.0.1')
ftp.login('admin', 'zdd2093419')
today = datetime.date.today()
remote_file = '/sda/91/' + str(today) + '/'
try:
ftp.mkd(remote_file)
except:
traceback.print_exc()
ftp.close()
def go(self):
html = self.__get_content(1)
#print(html)
page_num = self.__get_page_num(html)
print('页数=' + str(page_num))
today = datetime.date.today()
total_num = 0
for index in range(0, page_num):
result_all = []
html = self.__get_content(index + 1)
result = self.__analysis(html)
result_all.extend(result)
#print(result_all)
result_all = self.__refine(result_all)
#print(result_all)
result_all = self.__sort(result_all)
self.show(result_all)
result_all = self.__filter(result_all)
self.show(result_all)
print(str(today) + "有 " + str(len(result_all)) + " 部待上传")
if len(result_all) == 0:
continue
dir = Spider.PATH + str(today) + "\\"
if not os.path.exists(dir):
os.makedirs(dir)
ftp = Ftp('192.168.0.1')
ftp.login('admin', 'zdd2093419')
remote_file = '/KINGSTON-b/91/' + str(today) + '/'
try:
ftp.mkd(remote_file)
except:
traceback.print_exc()
ftp.close()
for index in range(0, len(result_all)):
video = result_all[index]
download_file = DownloadFile(dir, video['title'] + '.m3u8',
video['source_url'])
download_file.download_progress()
# 根据m3u8文件获取到所有ts的链接
result = spider.m3u8file(
re.findall(Spider.path_pattern, video['source_url'])[0],
dir + video['title'] + '.m3u8')
print(result)
all_ts_name = ''
# 下载所有ts
os.system("del " + dir + "*.ts")
for ts in result:
all_ts_name += dir + ts['file_name'] + '|'
download_file = DownloadFile(dir, ts['file_name'],
ts['sour_url'])
download_file.download_progress()
# 合并ts
os.system('ffmpeg -i "concat:' + all_ts_name.strip('|') +
'" -acodec copy -vcodec copy -absf aac_adtstoasc ' +
dir + video['title'] + '.mp4')
os.system("del " + dir + "*.ts")
# 下载完一个就上传一个
ftp = Ftp('192.168.0.1')
ftp.login('admin', 'zdd2093419')
try:
ftp.upload_file(dir + video['title'] + '.mp4',
remote_file + video['title'] + '.mp4')
Spider.mongo_collection.insert_one(video)
print("============ " + " 已上传 " + str(index + 1) +
" 部 ============")
total_num += 1
except:
traceback.print_exc()
ftp.close()
print("============ " + str(today) + " 总共上传 " + str(total_num) +
" 部 ============")