class ftpAccess():
def __init__(self, url):
self.ftp = FTP_TLS(url)
self.ftp.login(
"anonymous", "anonymous",
secure=False) # login anonymously before securing control channel
def GetFtpFile(self, InPath, filename, downloaddir):
outfile = downloaddir + filename
ensure_dir(downloaddir)
self.ftp.cwd(InPath)
self.ftp.retrbinary("RETR " + filename, open(outfile, 'wb').write)
def GetFileListFtp(self, pathname):
self.ftp.cwd(pathname)
ret = list()
out = self.ftp.retrlines('LIST',
addline) # list directory content securely
# search json files temp = entry.split(' ')
for filename in filenamelist:
try:
a = self.ftp.size(filename)
print("{} - {}".format(filename, a))
ret.append([filename, a])
except:
print("{} - xxx".format(filename))
pass
return ret
def download_all_in_one_path(targetdir,resultdir,check = True,num = 50):
if(os.path.exists(resultdir) == False):
os.makedirs(resultdir)
ftp = FTP('129.164.179.23')
ftp.login()
ftp.prot_p()
ftp.cwd(targetdir)
files = ftp.nlst()
target = 'https://heasarc.gsfc.nasa.gov/FTP' + targetdir
c = None
if(check):
c = []
data1 = []
ftp.voidcmd('TYPE I')
print('正在获取校验信息........')
for i in files:
#print(i)
data = os.path.join(target,i)
print(data)
data1.append(data)
if(check):
c.append(ftp.size(i))
ftp.quit()
if(check == False):
print('忽略数据大小校验。')
print('正在校验...............')
down(data1,resultdir,check=c,threadnum = num)
print('\n任务下载完成！！！')
def _download_worker_cddis(url, filename):
"""
Download the URL from gdc.cddis.eosdis.nasa.gov via FTP-SSL and save it to a file.
"""
# Attempt to download the data
print("Downloading %s" % url)
## Login
ftps = FTP_TLS("gdc.cddis.eosdis.nasa.gov",
timeout=DOWN_CONFIG.get('timeout'))
status = ftps.login("anonymous", "*****@*****.**")
if not status.startswith("230"):
ftps.close()
return False
## Secure
status = ftps.prot_p()
if not status.startswith("200"):
ftps.close()
return False
## Download
remote_path = url.split("gdc.cddis.eosdis.nasa.gov", 1)[1]
try:
remote_size = ftps.size(remote_path)
except FTP_ERROR:
ftps.close()
return False
with _CACHE_DIR.open(filename, 'wb') as fh:
pbar = DownloadBar(max=remote_size)
def write(data):
fh.write(data)
pbar.inc(len(data))
sys.stdout.write(pbar.show() + '\r')
sys.stdout.flush()
status = ftps.retrbinary('RETR %s' % remote_path,
write,
blocksize=DOWN_CONFIG.get('block_size'))
sys.stdout.write(pbar.show() + '\n')
sys.stdout.flush()
if not status.startswith("226"):
_CACHE_DIR.remove(filename)
ftps.close()
return False
## Further processing, if needed
if os.path.splitext(filename)[1] == '.Z':
## Save it to a regular gzip'd file after uncompressing it.
_convert_to_gzip(filename)
# Done
ftps.close()
return True
def connect(velkost_ftp,port):
ftp=FTP_TLS(server,meno2,ps,port)
ftp.prot_p()
ftp.cwd(my_list[2])
print "Posielam subor. Cakajte prosim."
obsah=open(file_to_send, 'rb')
obsah.close()
ftp.storbinary('STOR %s' % file_to_send, open(file_to_send, 'rb'))
obsah.close()
print "Subor odoslany [OK]"
print "Obsah adresara na serveri:"
ftp.retrlines("LIST")
size_ftp=ftp.nlst()
pocet=len(size_ftp)
velkost_ftp_subor=size_ftp[pocet-1] #berie posledne pridany subor zo zoznamu
ftp.sendcmd("TYPE i")
velkost_ftp=ftp.size(velkost_ftp_subor)
ftp.close()
return velkost_ftp
def upload(self, bangumi_tag='', debug_file=''):
first_connect = True # 标记是否是第一次连接, 第一次连接会删除临时缓存目录
tmp_dir = str(self._sn) + '-uploading-by-aniGamerPlus'
if debug_file:
self.local_video_path = debug_file
if not os.path.exists(self.local_video_path): # 如果文件不存在,直接返回失败
return self.upload_succeed_flag
if not self._video_filename: # 用于仅上传, 将文件名提取出来
self._video_filename = os.path.split(self.local_video_path)[-1]
socket.setdefaulttimeout(20) # 超时时间20s
if self._settings['ftp']['tls']:
ftp = FTP_TLS() # FTP over TLS
else:
ftp = FTP()
def connect_ftp(show_err=True):
ftp.encoding = 'utf-8' # 解决中文乱码
err_counter = 0
connect_flag = False
while err_counter <= 3:
try:
ftp.connect(self._settings['ftp']['server'],
self._settings['ftp']['port']) # 连接 FTP
ftp.login(self._settings['ftp']['user'],
self._settings['ftp']['pwd']) # 登陆
connect_flag = True
break
except ftplib.error_temp as e:
if show_err:
if 'Too many connections' in str(e):
detail = self._video_filename + ' 当前FTP連接數過多, 5分鐘后重試, 最多重試三次: ' + str(
e)
err_print(self._sn, 'FTP狀態', detail, status=1)
else:
detail = self._video_filename + ' 連接FTP時發生錯誤, 5分鐘后重試, 最多重試三次: ' + str(
e)
err_print(self._sn, 'FTP狀態', detail, status=1)
err_counter = err_counter + 1
for i in range(5 * 60):
time.sleep(1)
except BaseException as e:
if show_err:
detail = self._video_filename + ' 在連接FTP時發生無法處理的異常:' + str(
e)
err_print(self._sn, 'FTP狀態', detail, status=1)
break
if not connect_flag:
err_print(self._sn, '上傳失败', self._video_filename, status=1)
return connect_flag # 如果连接失败, 直接放弃
ftp.voidcmd('TYPE I') # 二进制模式
if self._settings['ftp']['cwd']:
try:
ftp.cwd(self._settings['ftp']['cwd']) # 进入用户指定目录
except ftplib.error_perm as e:
if show_err:
err_print(self._sn,
'FTP狀態',
'進入指定FTP目錄時出錯: ' + str(e),
status=1)
if bangumi_tag: # 番剧分类
try:
ftp.cwd(bangumi_tag)
except ftplib.error_perm:
try:
ftp.mkd(bangumi_tag)
ftp.cwd(bangumi_tag)
except ftplib.error_perm as e:
if show_err:
err_print(self._sn,
'FTP狀態',
'創建目錄番劇目錄時發生異常, 你可能沒有權限創建目錄: ' + str(e),
status=1)
# 归类番剧
ftp_bangumi_dir = Config.legalize_filename(
self._bangumi_name) # 保证合法
try:
ftp.cwd(ftp_bangumi_dir)
except ftplib.error_perm:
try:
ftp.mkd(ftp_bangumi_dir)
ftp.cwd(ftp_bangumi_dir)
except ftplib.error_perm as e:
if show_err:
detail = '你可能沒有權限創建目錄(用於分類番劇), 視頻文件將會直接上傳, 收到異常: ' + str(
e)
err_print(self._sn, 'FTP狀態', detail, status=1)
# 删除旧的临时文件夹
nonlocal first_connect
if first_connect: # 首次连接
remove_dir(tmp_dir)
first_connect = False # 标记第一次连接已完成
# 创建新的临时文件夹
# 创建临时文件夹是因为 pure-ftpd 在续传时会将文件名更改成不可预测的名字
# 正常中斷传输会把名字改回来, 但是意外掉线不会, 为了处理这种情况
# 需要获取 pure-ftpd 未知文件名的续传缓存文件, 为了不和其他视频的缓存文件混淆, 故建立一个临时文件夹
try:
ftp.cwd(tmp_dir)
except ftplib.error_perm:
ftp.mkd(tmp_dir)
ftp.cwd(tmp_dir)
return connect_flag
def exit_ftp(show_err=True):
try:
ftp.quit()
except BaseException as e:
if show_err and self._settings['ftp']['show_error_detail']:
err_print(self._sn, 'FTP狀態',
'將强制關閉FTP連接, 因爲在退出時收到異常: ' + str(e))
ftp.close()
def remove_dir(dir_name):
try:
ftp.rmd(dir_name)
except ftplib.error_perm as e:
if 'Directory not empty' in str(e):
# 如果目录非空, 则删除内部文件
ftp.cwd(dir_name)
del_all_files()
ftp.cwd('..')
ftp.rmd(dir_name) # 删完内部文件, 删除文件夹
elif 'No such file or directory' in str(e):
pass
else:
# 其他非空目录报错
raise e
def del_all_files():
try:
for file_need_del in ftp.nlst():
if not re.match(r'^(\.|\.\.)$', file_need_del):
ftp.delete(file_need_del)
# print('删除了文件: ' + file_need_del)
except ftplib.error_perm as resp:
if not str(resp) == "550 No files found":
raise
if not connect_ftp(): # 连接 FTP
return self.upload_succeed_flag # 如果连接失败
err_print(self._sn, '正在上傳',
self._video_filename + ' title=' + self._title + '……')
try_counter = 0
video_filename = self._video_filename # video_filename 将可能会储存 pure-ftpd 缓存文件名
max_try_num = self._settings['ftp']['max_retry_num']
local_size = os.path.getsize(self.local_video_path) # 本地文件大小
while try_counter <= max_try_num:
try:
if try_counter > 0:
# 传输遭中断后处理
detail = self._video_filename + ' 发生异常, 重連FTP, 續傳文件, 將重試最多' + str(
max_try_num) + '次……'
err_print(self._sn, '上傳狀態', detail, status=1)
if not connect_ftp(): # 重连
return self.upload_succeed_flag
# 解决操蛋的 Pure-Ftpd 续传一次就改名导致不能再续传问题.
# 一般正常关闭文件传输 Pure-Ftpd 会把名字改回来, 但是遇到网络意外中断, 那么就不会改回文件名, 留着临时文件名
# 本段就是处理这种情况
try:
for i in ftp.nlst():
if 'pureftpd-upload' in i:
# 找到 pure-ftpd 缓存, 直接抓缓存来续传
video_filename = i
except ftplib.error_perm as resp:
if not str(resp
) == "550 No files found": # 非文件不存在错误, 抛出异常
raise
# 断点续传
try:
# 需要 FTP Server 支持续传
ftp_binary_size = ftp.size(video_filename) # 远程文件字节数
except ftplib.error_perm:
# 如果不存在文件
ftp_binary_size = 0
except OSError:
try_counter = try_counter + 1
continue
ftp.voidcmd('TYPE I') # 二进制模式
conn = ftp.transfercmd('STOR ' + video_filename,
ftp_binary_size) # ftp服务器文件名和offset偏移地址
with open(self.local_video_path, 'rb') as f:
f.seek(ftp_binary_size) # 从断点处开始读取
while True:
block = f.read(1048576) # 读取1M
conn.sendall(block) # 送出 block
if not block:
time.sleep(3) # 等待一下, 让sendall()完成
break
conn.close()
err_print(self._sn, '上傳狀態', '檢查遠端文件大小是否與本地一致……')
exit_ftp(False)
connect_ftp(False)
# 不重连的话, 下面查询远程文件大小会返回 None, 懵逼...
# sendall()没有完成将会 500 Unknown command
err_counter = 0
remote_size = 0
while err_counter < 3:
try:
remote_size = ftp.size(video_filename) # 远程文件大小
break
except ftplib.error_perm as e1:
err_print(self._sn, 'FTP狀態',
'ftplib.error_perm: ' + str(e1))
remote_size = 0
break
except OSError as e2:
err_print(self._sn, 'FTP狀態', 'OSError: ' + str(e2))
remote_size = 0
connect_ftp(False) # 掉线重连
err_counter = err_counter + 1
if remote_size is None:
err_print(self._sn, 'FTP狀態', 'remote_size is None')
remote_size = 0
# 远程文件大小获取失败, 可能文件不存在或者抽风
# 那上面获取远程字节数将会是0, 导致重新下载, 那么此时应该清空缓存目录下的文件
# 避免后续找错文件续传
if remote_size == 0:
del_all_files()
if remote_size != local_size:
# 如果远程文件大小与本地不一致
# print('remote_size='+str(remote_size))
# print('local_size ='+str(local_size))
detail = self._video_filename + ' 在遠端為' + str(
round(remote_size / float(1024 * 1024),
2)) + 'MB' + ' 與本地' + str(
round(local_size / float(1024 * 1024), 2)
) + 'MB 不一致! 將重試最多' + str(max_try_num) + '次'
err_print(self._sn, '上傳狀態', detail, status=1)
try_counter = try_counter + 1
continue # 续传
# 顺利上传完后
ftp.cwd('..') # 返回上级目录, 即退出临时目录
try:
# 如果同名文件存在, 则删除
ftp.size(self._video_filename)
ftp.delete(self._video_filename)
except ftplib.error_perm:
pass
ftp.rename(tmp_dir + '/' + video_filename,
self._video_filename) # 将视频从临时文件移出, 顺便重命名
remove_dir(tmp_dir) # 删除临时目录
self.upload_succeed_flag = True # 标记上传成功
break
except ConnectionResetError as e:
if self._settings['ftp']['show_error_detail']:
detail = self._video_filename + ' 在上傳過程中網絡被重置, 將重試最多' + str(
max_try_num) + '次' + ', 收到異常: ' + str(e)
err_print(self._sn, '上傳狀態', detail, status=1)
try_counter = try_counter + 1
except TimeoutError as e:
if self._settings['ftp']['show_error_detail']:
detail = self._video_filename + ' 在上傳過程中超時, 將重試最多' + str(
max_try_num) + '次, 收到異常: ' + str(e)
err_print(self._sn, '上傳狀態', detail, status=1)
try_counter = try_counter + 1
except socket.timeout as e:
if self._settings['ftp']['show_error_detail']:
detail = self._video_filename + ' 在上傳過程socket超時, 將重試最多' + str(
max_try_num) + '次, 收到異常: ' + str(e)
err_print(self._sn, '上傳狀態', detail, status=1)
try_counter = try_counter + 1
if not self.upload_succeed_flag:
err_print(self._sn,
'上傳失敗',
self._video_filename + ' 放棄上傳!',
status=1)
exit_ftp()
return self.upload_succeed_flag
err_print(self._sn, '上傳完成', self._video_filename, status=2)
exit_ftp() # 登出 FTP
return self.upload_succeed_flag
def DownloadFile(self, dst_filename, local_filename=None):
res = ''
if local_filename is None:
local_filename = dst_filename
with open(local_filename, 'w+b') as f:
self.ptr = f.tell()
@setInterval(self.monitor_interval)
def monitor():
if not self.waiting:
i = f.tell()
if self.ptr < i:
logging.debug("%d - %0.1f Kb/s" %
(i, (i - self.ptr) /
(1024 * self.monitor_interval)))
self.ptr = i
os.system('clear')
print(
str(int((float(i) / float(dst_filesize)) * 100)) +
'%')
else:
ftp.close()
def connect():
ftp.connect(self.host, self.port)
ftp.login(self.login, self.passwd)
ftp.prot_p()
if self.directory != None:
ftp.cwd(self.directory)
# optimize socket params for download task
ftp.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
ftp.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL,
75)
ftp.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE,
60)
ftp = FTP_TLS()
ftp.set_pasv(True)
connect()
ftp.voidcmd('TYPE I')
dst_filesize = ftp.size(dst_filename)
mon = monitor()
while dst_filesize > f.tell():
try:
connect()
self.waiting = False
# retrieve file from position where we were disconnected
res = ftp.retrbinary('RETR %s' % dst_filename, f.write) if f.tell() == 0 else \
ftp.retrbinary('RETR %s' % dst_filename, f.write, rest=f.tell())
except:
self.max_attempts -= 1
if self.max_attempts == 0:
mon.set()
logging.exception('')
raise
self.waiting = True
logging.info('waiting 30 sec...')
time.sleep(30)
logging.info('reconnect')
mon.set() #stop monitor
ftp.close()
if not res.startswith('226 Transfer complete'):
logging.error(
'Downloaded file {0} is not full.'.format(dst_filename))
os.remove(local_filename)
return None
return 1
