def upload(self, bangumi_tag='', debug_file=''): first_connect = True # 标记是否是第一次连接, 第一次连接会删除临时缓存目录 tmp_dir = str(self._sn) + '-uploading-by-aniGamerPlus' if debug_file: self.local_video_path = debug_file if not os.path.exists(self.local_video_path): # 如果文件不存在,直接返回失败 return self.upload_succeed_flag if not self._video_filename: # 用于仅上传, 将文件名提取出来 self._video_filename = os.path.split(self.local_video_path)[-1] socket.setdefaulttimeout(20) # 超时时间20s if self._settings['ftp']['tls']: ftp = FTP_TLS() # FTP over TLS else: ftp = FTP() def connect_ftp(show_err=True): ftp.encoding = 'utf-8' # 解决中文乱码 err_counter = 0 connect_flag = False while err_counter <= 3: try: ftp.connect(self._settings['ftp']['server'], self._settings['ftp']['port']) # 连接 FTP ftp.login(self._settings['ftp']['user'], self._settings['ftp']['pwd']) # 登陆 connect_flag = True break except ftplib.error_temp as e: if show_err: if 'Too many connections' in str(e): detail = self._video_filename + ' 当前FTP連接數過多, 5分鐘后重試, 最多重試三次: ' + str( e) err_print(self._sn, 'FTP狀態', detail, status=1) else: detail = self._video_filename + ' 連接FTP時發生錯誤, 5分鐘后重試, 最多重試三次: ' + str( e) err_print(self._sn, 'FTP狀態', detail, status=1) err_counter = err_counter + 1 for i in range(5 * 60): time.sleep(1) except BaseException as e: if show_err: detail = self._video_filename + ' 在連接FTP時發生無法處理的異常:' + str( e) err_print(self._sn, 'FTP狀態', detail, status=1) break if not connect_flag: err_print(self._sn, '上傳失败', self._video_filename, status=1) return connect_flag # 如果连接失败, 直接放弃 ftp.voidcmd('TYPE I') # 二进制模式 if self._settings['ftp']['cwd']: try: ftp.cwd(self._settings['ftp']['cwd']) # 进入用户指定目录 except ftplib.error_perm as e: if show_err: err_print(self._sn, 'FTP狀態', '進入指定FTP目錄時出錯: ' + str(e), status=1) if bangumi_tag: # 番剧分类 try: ftp.cwd(bangumi_tag) except ftplib.error_perm: try: ftp.mkd(bangumi_tag) ftp.cwd(bangumi_tag) except ftplib.error_perm as e: if show_err: err_print(self._sn, 'FTP狀態', '創建目錄番劇目錄時發生異常, 你可能沒有權限創建目錄: ' + str(e), status=1) # 归类番剧 ftp_bangumi_dir = Config.legalize_filename( self._bangumi_name) # 保证合法 try: ftp.cwd(ftp_bangumi_dir) except ftplib.error_perm: try: ftp.mkd(ftp_bangumi_dir) ftp.cwd(ftp_bangumi_dir) except ftplib.error_perm as e: if show_err: detail = '你可能沒有權限創建目錄(用於分類番劇), 視頻文件將會直接上傳, 收到異常: ' + str( e) err_print(self._sn, 'FTP狀態', detail, status=1) # 删除旧的临时文件夹 nonlocal first_connect if first_connect: # 首次连接 remove_dir(tmp_dir) first_connect = False # 标记第一次连接已完成 # 创建新的临时文件夹 # 创建临时文件夹是因为 pure-ftpd 在续传时会将文件名更改成不可预测的名字 # 正常中斷传输会把名字改回来, 但是意外掉线不会, 为了处理这种情况 # 需要获取 pure-ftpd 未知文件名的续传缓存文件, 为了不和其他视频的缓存文件混淆, 故建立一个临时文件夹 try: ftp.cwd(tmp_dir) except ftplib.error_perm: ftp.mkd(tmp_dir) ftp.cwd(tmp_dir) return connect_flag def exit_ftp(show_err=True): try: ftp.quit() except BaseException as e: if show_err and self._settings['ftp']['show_error_detail']: err_print(self._sn, 'FTP狀態', '將强制關閉FTP連接, 因爲在退出時收到異常: ' + str(e)) ftp.close() def remove_dir(dir_name): try: ftp.rmd(dir_name) except ftplib.error_perm as e: if 'Directory not empty' in str(e): # 如果目录非空, 则删除内部文件 ftp.cwd(dir_name) del_all_files() ftp.cwd('..') ftp.rmd(dir_name) # 删完内部文件, 删除文件夹 elif 'No such file or directory' in str(e): pass else: # 其他非空目录报错 raise e def del_all_files(): try: for file_need_del in ftp.nlst(): if not re.match(r'^(\.|\.\.)$', file_need_del): ftp.delete(file_need_del) # print('删除了文件: ' + file_need_del) except ftplib.error_perm as resp: if not str(resp) == "550 No files found": raise if not connect_ftp(): # 连接 FTP return self.upload_succeed_flag # 如果连接失败 err_print(self._sn, '正在上傳', self._video_filename + ' title=' + self._title + '……') try_counter = 0 video_filename = self._video_filename # video_filename 将可能会储存 pure-ftpd 缓存文件名 max_try_num = self._settings['ftp']['max_retry_num'] local_size = os.path.getsize(self.local_video_path) # 本地文件大小 while try_counter <= max_try_num: try: if try_counter > 0: # 传输遭中断后处理 detail = self._video_filename + ' 发生异常, 重連FTP, 續傳文件, 將重試最多' + str( max_try_num) + '次……' err_print(self._sn, '上傳狀態', detail, status=1) if not connect_ftp(): # 重连 return self.upload_succeed_flag # 解决操蛋的 Pure-Ftpd 续传一次就改名导致不能再续传问题. # 一般正常关闭文件传输 Pure-Ftpd 会把名字改回来, 但是遇到网络意外中断, 那么就不会改回文件名, 留着临时文件名 # 本段就是处理这种情况 try: for i in ftp.nlst(): if 'pureftpd-upload' in i: # 找到 pure-ftpd 缓存, 直接抓缓存来续传 video_filename = i except ftplib.error_perm as resp: if not str(resp ) == "550 No files found": # 非文件不存在错误, 抛出异常 raise # 断点续传 try: # 需要 FTP Server 支持续传 ftp_binary_size = ftp.size(video_filename) # 远程文件字节数 except ftplib.error_perm: # 如果不存在文件 ftp_binary_size = 0 except OSError: try_counter = try_counter + 1 continue ftp.voidcmd('TYPE I') # 二进制模式 conn = ftp.transfercmd('STOR ' + video_filename, ftp_binary_size) # ftp服务器文件名和offset偏移地址 with open(self.local_video_path, 'rb') as f: f.seek(ftp_binary_size) # 从断点处开始读取 while True: block = f.read(1048576) # 读取1M conn.sendall(block) # 送出 block if not block: time.sleep(3) # 等待一下, 让sendall()完成 break conn.close() err_print(self._sn, '上傳狀態', '檢查遠端文件大小是否與本地一致……') exit_ftp(False) connect_ftp(False) # 不重连的话, 下面查询远程文件大小会返回 None, 懵逼... # sendall()没有完成将会 500 Unknown command err_counter = 0 remote_size = 0 while err_counter < 3: try: remote_size = ftp.size(video_filename) # 远程文件大小 break except ftplib.error_perm as e1: err_print(self._sn, 'FTP狀態', 'ftplib.error_perm: ' + str(e1)) remote_size = 0 break except OSError as e2: err_print(self._sn, 'FTP狀態', 'OSError: ' + str(e2)) remote_size = 0 connect_ftp(False) # 掉线重连 err_counter = err_counter + 1 if remote_size is None: err_print(self._sn, 'FTP狀態', 'remote_size is None') remote_size = 0 # 远程文件大小获取失败, 可能文件不存在或者抽风 # 那上面获取远程字节数将会是0, 导致重新下载, 那么此时应该清空缓存目录下的文件 # 避免后续找错文件续传 if remote_size == 0: del_all_files() if remote_size != local_size: # 如果远程文件大小与本地不一致 # print('remote_size='+str(remote_size)) # print('local_size ='+str(local_size)) detail = self._video_filename + ' 在遠端為' + str( round(remote_size / float(1024 * 1024), 2)) + 'MB' + ' 與本地' + str( round(local_size / float(1024 * 1024), 2) ) + 'MB 不一致! 將重試最多' + str(max_try_num) + '次' err_print(self._sn, '上傳狀態', detail, status=1) try_counter = try_counter + 1 continue # 续传 # 顺利上传完后 ftp.cwd('..') # 返回上级目录, 即退出临时目录 try: # 如果同名文件存在, 则删除 ftp.size(self._video_filename) ftp.delete(self._video_filename) except ftplib.error_perm: pass ftp.rename(tmp_dir + '/' + video_filename, self._video_filename) # 将视频从临时文件移出, 顺便重命名 remove_dir(tmp_dir) # 删除临时目录 self.upload_succeed_flag = True # 标记上传成功 break except ConnectionResetError as e: if self._settings['ftp']['show_error_detail']: detail = self._video_filename + ' 在上傳過程中網絡被重置, 將重試最多' + str( max_try_num) + '次' + ', 收到異常: ' + str(e) err_print(self._sn, '上傳狀態', detail, status=1) try_counter = try_counter + 1 except TimeoutError as e: if self._settings['ftp']['show_error_detail']: detail = self._video_filename + ' 在上傳過程中超時, 將重試最多' + str( max_try_num) + '次, 收到異常: ' + str(e) err_print(self._sn, '上傳狀態', detail, status=1) try_counter = try_counter + 1 except socket.timeout as e: if self._settings['ftp']['show_error_detail']: detail = self._video_filename + ' 在上傳過程socket超時, 將重試最多' + str( max_try_num) + '次, 收到異常: ' + str(e) err_print(self._sn, '上傳狀態', detail, status=1) try_counter = try_counter + 1 if not self.upload_succeed_flag: err_print(self._sn, '上傳失敗', self._video_filename + ' 放棄上傳!', status=1) exit_ftp() return self.upload_succeed_flag err_print(self._sn, '上傳完成', self._video_filename, status=2) exit_ftp() # 登出 FTP return self.upload_succeed_flag
# Download oldest & newest image for image in images_to_upload: print 'Downloading %s...' % image ftps.retrbinary('RETR %s' % image, open(image, 'wb').write) # Check if directory for old images exists, if not create it if 'old' not in directory_list: print 'Creating dir "old"...' ftps.mkd('old') # Move checked images to old for image in image_list: #date_str = image.split('_')[1].split('.')[0] #img_date = datetime.strptime(date_str, '%Y%m%d-%H%M%S') print 'Moving %s to "old"...' % image ftps.rename(image, 'old/%s' % image) # Disconnect from FTP server ftps.quit() def upload_image(filename): image_file = {'file': open(filename, 'rb')} print 'Uploading %s to GroupMe image service...' % filename r = requests.post('%s?access_token=%s' % (GROUPME_IMAGE_URL, TOKEN), files=image_file) # Get URL of image as given by GroupMe image service return r.json()['payload']['url'] # Upload images to GroupMe image service for image in images_to_upload: image_url = upload_image(image) # Delete image from local storage
ftp_username = "******" ftp_password = "******" # remote server paths dest_dir = "/production/deploy-" + timestamp backup_dir = "/production/archive/www-" + timestamp production_dir = "/production/www" if __name__ == '__main__': os.chdir(os.path.dirname(__file__)) print "Connecting to FTP host " + ftp_host ftp = FTP_TLS(ftp_host, ftp_username, ftp_password) wmsg = FTP_TLS.getwelcome(ftp) print wmsg #Upload the local dir excluding any directories or files as listed print "Starting directory transfer of " + local_dir senddir(local_dir, dest_dir, ftp, exclude_dirs, exclude_files) #Archive the current production directory print "Renaming " + production_dir + " to " + backup_dir ftp.rename(production_dir, backup_dir) #Rename the uploaded directory to production dirrectory print "Renaming " + dest_dir + " to " + production_dir ftp.rename(dest_dir, production_dir) print "Transfer complete"
# Download oldest & newest image for image in images_to_upload: print 'Downloading %s...' % image ftps.retrbinary('RETR %s' % image, open(image, 'wb').write) # Check if directory for old images exists, if not create it if 'old' not in directory_list: print 'Creating dir "old"...' ftps.mkd('old') # Move checked images to old for image in image_list: #date_str = image.split('_')[1].split('.')[0] #img_date = datetime.strptime(date_str, '%Y%m%d-%H%M%S') print 'Moving %s to "old"...' % image ftps.rename(image, 'old/%s' % image) # Disconnect from FTP server ftps.quit() def upload_image(filename): image_file = {'file': open(filename, 'rb')} print 'Uploading %s to GroupMe image service...' % filename r = requests.post('%s?access_token=%s' % (GROUPME_IMAGE_URL, TOKEN), files=image_file) # Get URL of image as given by GroupMe image service return r.json()['payload']['url'] # Upload images to GroupMe image service