def upload(self, bangumi_tag='', debug_file=''):
first_connect = True # 标记是否是第一次连接, 第一次连接会删除临时缓存目录
tmp_dir = str(self._sn) + '-uploading-by-aniGamerPlus'
if debug_file:
self.local_video_path = debug_file
if not os.path.exists(self.local_video_path): # 如果文件不存在,直接返回失败
return self.upload_succeed_flag
if not self._video_filename: # 用于仅上传, 将文件名提取出来
self._video_filename = os.path.split(self.local_video_path)[-1]
socket.setdefaulttimeout(20) # 超时时间20s
if self._settings['ftp']['tls']:
ftp = FTP_TLS() # FTP over TLS
else:
ftp = FTP()
def connect_ftp(show_err=True):
ftp.encoding = 'utf-8' # 解决中文乱码
err_counter = 0
connect_flag = False
while err_counter <= 3:
try:
ftp.connect(self._settings['ftp']['server'],
self._settings['ftp']['port']) # 连接 FTP
ftp.login(self._settings['ftp']['user'],
self._settings['ftp']['pwd']) # 登陆
connect_flag = True
break
except ftplib.error_temp as e:
if show_err:
if 'Too many connections' in str(e):
detail = self._video_filename + ' 当前FTP連接數過多, 5分鐘后重試, 最多重試三次: ' + str(
e)
err_print(self._sn, 'FTP狀態', detail, status=1)
else:
detail = self._video_filename + ' 連接FTP時發生錯誤, 5分鐘后重試, 最多重試三次: ' + str(
e)
err_print(self._sn, 'FTP狀態', detail, status=1)
err_counter = err_counter + 1
for i in range(5 * 60):
time.sleep(1)
except BaseException as e:
if show_err:
detail = self._video_filename + ' 在連接FTP時發生無法處理的異常:' + str(
e)
err_print(self._sn, 'FTP狀態', detail, status=1)
break
if not connect_flag:
err_print(self._sn, '上傳失败', self._video_filename, status=1)
return connect_flag # 如果连接失败, 直接放弃
ftp.voidcmd('TYPE I') # 二进制模式
if self._settings['ftp']['cwd']:
try:
ftp.cwd(self._settings['ftp']['cwd']) # 进入用户指定目录
except ftplib.error_perm as e:
if show_err:
err_print(self._sn,
'FTP狀態',
'進入指定FTP目錄時出錯: ' + str(e),
status=1)
if bangumi_tag: # 番剧分类
try:
ftp.cwd(bangumi_tag)
except ftplib.error_perm:
try:
ftp.mkd(bangumi_tag)
ftp.cwd(bangumi_tag)
except ftplib.error_perm as e:
if show_err:
err_print(self._sn,
'FTP狀態',
'創建目錄番劇目錄時發生異常, 你可能沒有權限創建目錄: ' + str(e),
status=1)
# 归类番剧
ftp_bangumi_dir = Config.legalize_filename(
self._bangumi_name) # 保证合法
try:
ftp.cwd(ftp_bangumi_dir)
except ftplib.error_perm:
try:
ftp.mkd(ftp_bangumi_dir)
ftp.cwd(ftp_bangumi_dir)
except ftplib.error_perm as e:
if show_err:
detail = '你可能沒有權限創建目錄(用於分類番劇), 視頻文件將會直接上傳, 收到異常: ' + str(
e)
err_print(self._sn, 'FTP狀態', detail, status=1)
# 删除旧的临时文件夹
nonlocal first_connect
if first_connect: # 首次连接
remove_dir(tmp_dir)
first_connect = False # 标记第一次连接已完成
# 创建新的临时文件夹
# 创建临时文件夹是因为 pure-ftpd 在续传时会将文件名更改成不可预测的名字
# 正常中斷传输会把名字改回来, 但是意外掉线不会, 为了处理这种情况
# 需要获取 pure-ftpd 未知文件名的续传缓存文件, 为了不和其他视频的缓存文件混淆, 故建立一个临时文件夹
try:
ftp.cwd(tmp_dir)
except ftplib.error_perm:
ftp.mkd(tmp_dir)
ftp.cwd(tmp_dir)
return connect_flag
def exit_ftp(show_err=True):
try:
ftp.quit()
except BaseException as e:
if show_err and self._settings['ftp']['show_error_detail']:
err_print(self._sn, 'FTP狀態',
'將强制關閉FTP連接, 因爲在退出時收到異常: ' + str(e))
ftp.close()
def remove_dir(dir_name):
try:
ftp.rmd(dir_name)
except ftplib.error_perm as e:
if 'Directory not empty' in str(e):
# 如果目录非空, 则删除内部文件
ftp.cwd(dir_name)
del_all_files()
ftp.cwd('..')
ftp.rmd(dir_name) # 删完内部文件, 删除文件夹
elif 'No such file or directory' in str(e):
pass
else:
# 其他非空目录报错
raise e
def del_all_files():
try:
for file_need_del in ftp.nlst():
if not re.match(r'^(\.|\.\.)$', file_need_del):
ftp.delete(file_need_del)
# print('删除了文件: ' + file_need_del)
except ftplib.error_perm as resp:
if not str(resp) == "550 No files found":
raise
if not connect_ftp(): # 连接 FTP
return self.upload_succeed_flag # 如果连接失败
err_print(self._sn, '正在上傳',
self._video_filename + ' title=' + self._title + '……')
try_counter = 0
video_filename = self._video_filename # video_filename 将可能会储存 pure-ftpd 缓存文件名
max_try_num = self._settings['ftp']['max_retry_num']
local_size = os.path.getsize(self.local_video_path) # 本地文件大小
while try_counter <= max_try_num:
try:
if try_counter > 0:
# 传输遭中断后处理
detail = self._video_filename + ' 发生异常, 重連FTP, 續傳文件, 將重試最多' + str(
max_try_num) + '次……'
err_print(self._sn, '上傳狀態', detail, status=1)
if not connect_ftp(): # 重连
return self.upload_succeed_flag
# 解决操蛋的 Pure-Ftpd 续传一次就改名导致不能再续传问题.
# 一般正常关闭文件传输 Pure-Ftpd 会把名字改回来, 但是遇到网络意外中断, 那么就不会改回文件名, 留着临时文件名
# 本段就是处理这种情况
try:
for i in ftp.nlst():
if 'pureftpd-upload' in i:
# 找到 pure-ftpd 缓存, 直接抓缓存来续传
video_filename = i
except ftplib.error_perm as resp:
if not str(resp
) == "550 No files found": # 非文件不存在错误, 抛出异常
raise
# 断点续传
try:
# 需要 FTP Server 支持续传
ftp_binary_size = ftp.size(video_filename) # 远程文件字节数
except ftplib.error_perm:
# 如果不存在文件
ftp_binary_size = 0
except OSError:
try_counter = try_counter + 1
continue
ftp.voidcmd('TYPE I') # 二进制模式
conn = ftp.transfercmd('STOR ' + video_filename,
ftp_binary_size) # ftp服务器文件名和offset偏移地址
with open(self.local_video_path, 'rb') as f:
f.seek(ftp_binary_size) # 从断点处开始读取
while True:
block = f.read(1048576) # 读取1M
conn.sendall(block) # 送出 block
if not block:
time.sleep(3) # 等待一下, 让sendall()完成
break
conn.close()
err_print(self._sn, '上傳狀態', '檢查遠端文件大小是否與本地一致……')
exit_ftp(False)
connect_ftp(False)
# 不重连的话, 下面查询远程文件大小会返回 None, 懵逼...
# sendall()没有完成将会 500 Unknown command
err_counter = 0
remote_size = 0
while err_counter < 3:
try:
remote_size = ftp.size(video_filename) # 远程文件大小
break
except ftplib.error_perm as e1:
err_print(self._sn, 'FTP狀態',
'ftplib.error_perm: ' + str(e1))
remote_size = 0
break
except OSError as e2:
err_print(self._sn, 'FTP狀態', 'OSError: ' + str(e2))
remote_size = 0
connect_ftp(False) # 掉线重连
err_counter = err_counter + 1
if remote_size is None:
err_print(self._sn, 'FTP狀態', 'remote_size is None')
remote_size = 0
# 远程文件大小获取失败, 可能文件不存在或者抽风
# 那上面获取远程字节数将会是0, 导致重新下载, 那么此时应该清空缓存目录下的文件
# 避免后续找错文件续传
if remote_size == 0:
del_all_files()
if remote_size != local_size:
# 如果远程文件大小与本地不一致
# print('remote_size='+str(remote_size))
# print('local_size ='+str(local_size))
detail = self._video_filename + ' 在遠端為' + str(
round(remote_size / float(1024 * 1024),
2)) + 'MB' + ' 與本地' + str(
round(local_size / float(1024 * 1024), 2)
) + 'MB 不一致! 將重試最多' + str(max_try_num) + '次'
err_print(self._sn, '上傳狀態', detail, status=1)
try_counter = try_counter + 1
continue # 续传
# 顺利上传完后
ftp.cwd('..') # 返回上级目录, 即退出临时目录
try:
# 如果同名文件存在, 则删除
ftp.size(self._video_filename)
ftp.delete(self._video_filename)
except ftplib.error_perm:
pass
ftp.rename(tmp_dir + '/' + video_filename,
self._video_filename) # 将视频从临时文件移出, 顺便重命名
remove_dir(tmp_dir) # 删除临时目录
self.upload_succeed_flag = True # 标记上传成功
break
except ConnectionResetError as e:
if self._settings['ftp']['show_error_detail']:
detail = self._video_filename + ' 在上傳過程中網絡被重置, 將重試最多' + str(
max_try_num) + '次' + ', 收到異常: ' + str(e)
err_print(self._sn, '上傳狀態', detail, status=1)
try_counter = try_counter + 1
except TimeoutError as e:
if self._settings['ftp']['show_error_detail']:
detail = self._video_filename + ' 在上傳過程中超時, 將重試最多' + str(
max_try_num) + '次, 收到異常: ' + str(e)
err_print(self._sn, '上傳狀態', detail, status=1)
try_counter = try_counter + 1
except socket.timeout as e:
if self._settings['ftp']['show_error_detail']:
detail = self._video_filename + ' 在上傳過程socket超時, 將重試最多' + str(
max_try_num) + '次, 收到異常: ' + str(e)
err_print(self._sn, '上傳狀態', detail, status=1)
try_counter = try_counter + 1
if not self.upload_succeed_flag:
err_print(self._sn,
'上傳失敗',
self._video_filename + ' 放棄上傳!',
status=1)
exit_ftp()
return self.upload_succeed_flag
err_print(self._sn, '上傳完成', self._video_filename, status=2)
exit_ftp() # 登出 FTP
return self.upload_succeed_flag
# Download oldest & newest image
for image in images_to_upload:
print 'Downloading %s...' % image
ftps.retrbinary('RETR %s' % image, open(image, 'wb').write)
# Check if directory for old images exists, if not create it
if 'old' not in directory_list:
print 'Creating dir "old"...'
ftps.mkd('old')
# Move checked images to old
for image in image_list:
#date_str = image.split('_')[1].split('.')[0]
#img_date = datetime.strptime(date_str, '%Y%m%d-%H%M%S')
print 'Moving %s to "old"...' % image
ftps.rename(image, 'old/%s' % image)
# Disconnect from FTP server
ftps.quit()
def upload_image(filename):
image_file = {'file': open(filename, 'rb')}
print 'Uploading %s to GroupMe image service...' % filename
r = requests.post('%s?access_token=%s' % (GROUPME_IMAGE_URL, TOKEN), files=image_file)
# Get URL of image as given by GroupMe image service
return r.json()['payload']['url']
# Upload images to GroupMe image service
for image in images_to_upload:
image_url = upload_image(image)
# Delete image from local storage
ftp_username = "******"
ftp_password = "******"
# remote server paths
dest_dir = "/production/deploy-" + timestamp
backup_dir = "/production/archive/www-" + timestamp
production_dir = "/production/www"
if __name__ == '__main__':
os.chdir(os.path.dirname(__file__))
print "Connecting to FTP host " + ftp_host
ftp = FTP_TLS(ftp_host, ftp_username, ftp_password)
wmsg = FTP_TLS.getwelcome(ftp)
print wmsg
#Upload the local dir excluding any directories or files as listed
print "Starting directory transfer of " + local_dir
senddir(local_dir, dest_dir, ftp, exclude_dirs, exclude_files)
#Archive the current production directory
print "Renaming " + production_dir + " to " + backup_dir
ftp.rename(production_dir, backup_dir)
#Rename the uploaded directory to production dirrectory
print "Renaming " + dest_dir + " to " + production_dir
ftp.rename(dest_dir, production_dir)
print "Transfer complete"
# Download oldest & newest image
for image in images_to_upload:
print 'Downloading %s...' % image
ftps.retrbinary('RETR %s' % image, open(image, 'wb').write)
# Check if directory for old images exists, if not create it
if 'old' not in directory_list:
print 'Creating dir "old"...'
ftps.mkd('old')
# Move checked images to old
for image in image_list:
#date_str = image.split('_')[1].split('.')[0]
#img_date = datetime.strptime(date_str, '%Y%m%d-%H%M%S')
print 'Moving %s to "old"...' % image
ftps.rename(image, 'old/%s' % image)
# Disconnect from FTP server
ftps.quit()
def upload_image(filename):
image_file = {'file': open(filename, 'rb')}
print 'Uploading %s to GroupMe image service...' % filename
r = requests.post('%s?access_token=%s' % (GROUPME_IMAGE_URL, TOKEN),
files=image_file)
# Get URL of image as given by GroupMe image service
return r.json()['payload']['url']
# Upload images to GroupMe image service
