def get_ftp_data(self, cr, uid, ids, context={}):
for chain in self.browse(cr, uid, ids, context=context):
config_obj = chain.ftp_config_id
try:
conn = FTP_TLS(host=config_obj.host,
user=config_obj.username,
passwd=config_obj.passwd)
conn.prot_p()
except:
conn = FTP(host=config_obj.host,
user=config_obj.username,
passwd=config_obj.passwd)
filenames = conn.nlst()
for filename in filenames:
input_file = StringIO()
conn.retrbinary('RETR %s' % filename,
lambda data: input_file.write(data))
input_string = input_file.getvalue()
input_file.close()
csv_reader = unicode_csv_reader(
StringIO(input_string),
delimiter=str(chain.separator),
quoting=(not chain.delimiter and csv.QUOTE_NONE)
or csv.QUOTE_MINIMAL,
quotechar=chain.delimiter and str(chain.delimiter) or None,
charset=chain.charset)
self.import_to_db(cr,
uid,
ids,
csv_reader=csv_reader,
context=context)
conn.delete(filename)
conn.quit()
return True
def check_ftps(hostname, temp_name, username, password, verbose):
ftps_services_failed = []
if verbose:
print("-" * 60)
if verbose:
print(temp_name)
ftps = FTP_TLS(hostname)
ftps.login(username, password)
ftps.prot_p()
#ftps.retrlines('LIST')
if verbose:
ftps.set_debuglevel(2)
# Upload the file
if verbose:
print("FTPS: Uploading the file.")
try:
ftps.storbinary('STOR {0}'.format('ftps.txt'), open(temp_name, 'rb'))
except:
if verbose:
print("FTPS: Uploading the file failed.")
ftps_services_failed.append('ftps_upload')
else:
if verbose:
print("FTPS: Uploaded file successfully.")
pass
# Download the file
if verbose:
print("FTPS: Downloading the file.")
try:
myfile = open('/tmp/ftps.txt', 'wb')
ftps.retrbinary('RETR {0}'.format('ftps.txt'), myfile.write)
except:
if verbose:
print("FTPS: Downloading the uploaded file failed.")
ftps_services_failed.append('ftps_download')
else:
if verbose:
print("FTPS: Downloaded the uploaded file successfully.")
# Delete the file from remote system
try:
ftps.delete('ftps.txt')
except:
if verbose:
print("FTPS: Deleting uploaded file failed.")
ftps_services_failed.append('ftps_delete')
else:
if verbose:
print("FTPS: Deleted the uploaded file successfully.")
pass
# Close the ftps connection.
ftps.close()
# Detel the file which is downloaded
delete_temp_file('/tmp/ftps.txt', verbose)
return ftps_services_failed
def downloadSarZIP(value, ftpParams):
ftps = FTP_TLS()
#ftps.set_debuglevel(2)
ftps.connect(ftpParams.host)
ftps.sendcmd('USER ' + ftpParams.user)
ftps.sendcmd('PASS ' + ftpParams.password)
list_of_files = []
checkFile = []
ftps.retrlines("NLST", list_of_files.append)
# ...or use the existing helper
# list_of_files = ftps.nlst()
dest_dir = "./"
# download files from ftp
for name in list_of_files:
if fnmatch.fnmatch(name, value + "_*"):
checkFile.append(name)
with open(os.path.join(dest_dir, name), "wb") as f:
ftps.retrbinary("RETR {}".format(name), f.write)
#delete files from ftp
for name in list_of_files:
if fnmatch.fnmatch(name, value + "_*"):
ftps.delete(name)
ftps.quit()
return checkFile
def DeleteTextFile():
filename = 'test.txt'
session = FTP_TLS('files.000webhost.com', 'vulcuns-webserver',
'AKPJJDT2lol')
session.cwd('public_html')
session.delete(filename)
session.quit()
def get_ftp_data(self, cr, uid, ids, context={}):
for chain in self.browse(cr, uid, ids, context=context):
config_obj = chain.ftp_config_id
try:
conn = FTP_TLS(host=config_obj.host, user=config_obj.username, passwd=config_obj.passwd)
conn.prot_p()
except:
conn = FTP(host=config_obj.host, user=config_obj.username, passwd=config_obj.passwd)
filenames = conn.nlst()
for filename in filenames:
input_file = StringIO()
conn.retrbinary('RETR %s' % filename, lambda data: input_file.write(data))
input_string = input_file.getvalue()
input_file.close()
csv_reader = unicode_csv_reader(StringIO(input_string), delimiter=str(chain.separator), quoting=(not chain.delimiter and csv.QUOTE_NONE) or csv.QUOTE_MINIMAL, quotechar=chain.delimiter and str(chain.delimiter) or None, charset=chain.charset)
self.import_to_db(cr, uid, ids, csv_reader=csv_reader, context=context)
conn.delete(filename)
conn.quit()
return True
import sys,json server = json.loads(sys.argv[1]) try: if server['secure'] == True: from ftplib import FTP_TLS ftp = FTP_TLS() else: from ftplib import FTP ftp = FTP() ftp.connect(server['host'], server['port']) ftp.login(server['user'], server['pass']) ftp.delete(sys.argv[2]) ftp.quit() except: sys.exit(1)
def upload(self, bangumi_tag='', debug_file=''):
first_connect = True # 标记是否是第一次连接, 第一次连接会删除临时缓存目录
tmp_dir = str(self._sn) + '-uploading-by-aniGamerPlus'
if debug_file:
self.local_video_path = debug_file
if not os.path.exists(self.local_video_path): # 如果文件不存在,直接返回失败
return self.upload_succeed_flag
if not self._video_filename: # 用于仅上传, 将文件名提取出来
self._video_filename = os.path.split(self.local_video_path)[-1]
socket.setdefaulttimeout(20) # 超时时间20s
if self._settings['ftp']['tls']:
ftp = FTP_TLS() # FTP over TLS
else:
ftp = FTP()
def connect_ftp(show_err=True):
ftp.encoding = 'utf-8' # 解决中文乱码
err_counter = 0
connect_flag = False
while err_counter <= 3:
try:
ftp.connect(self._settings['ftp']['server'],
self._settings['ftp']['port']) # 连接 FTP
ftp.login(self._settings['ftp']['user'],
self._settings['ftp']['pwd']) # 登陆
connect_flag = True
break
except ftplib.error_temp as e:
if show_err:
if 'Too many connections' in str(e):
detail = self._video_filename + ' 当前FTP連接數過多, 5分鐘后重試, 最多重試三次: ' + str(
e)
err_print(self._sn, 'FTP狀態', detail, status=1)
else:
detail = self._video_filename + ' 連接FTP時發生錯誤, 5分鐘后重試, 最多重試三次: ' + str(
e)
err_print(self._sn, 'FTP狀態', detail, status=1)
err_counter = err_counter + 1
for i in range(5 * 60):
time.sleep(1)
except BaseException as e:
if show_err:
detail = self._video_filename + ' 在連接FTP時發生無法處理的異常:' + str(
e)
err_print(self._sn, 'FTP狀態', detail, status=1)
break
if not connect_flag:
err_print(self._sn, '上傳失败', self._video_filename, status=1)
return connect_flag # 如果连接失败, 直接放弃
ftp.voidcmd('TYPE I') # 二进制模式
if self._settings['ftp']['cwd']:
try:
ftp.cwd(self._settings['ftp']['cwd']) # 进入用户指定目录
except ftplib.error_perm as e:
if show_err:
err_print(self._sn,
'FTP狀態',
'進入指定FTP目錄時出錯: ' + str(e),
status=1)
if bangumi_tag: # 番剧分类
try:
ftp.cwd(bangumi_tag)
except ftplib.error_perm:
try:
ftp.mkd(bangumi_tag)
ftp.cwd(bangumi_tag)
except ftplib.error_perm as e:
if show_err:
err_print(self._sn,
'FTP狀態',
'創建目錄番劇目錄時發生異常, 你可能沒有權限創建目錄: ' + str(e),
status=1)
# 归类番剧
ftp_bangumi_dir = Config.legalize_filename(
self._bangumi_name) # 保证合法
try:
ftp.cwd(ftp_bangumi_dir)
except ftplib.error_perm:
try:
ftp.mkd(ftp_bangumi_dir)
ftp.cwd(ftp_bangumi_dir)
except ftplib.error_perm as e:
if show_err:
detail = '你可能沒有權限創建目錄(用於分類番劇), 視頻文件將會直接上傳, 收到異常: ' + str(
e)
err_print(self._sn, 'FTP狀態', detail, status=1)
# 删除旧的临时文件夹
nonlocal first_connect
if first_connect: # 首次连接
remove_dir(tmp_dir)
first_connect = False # 标记第一次连接已完成
# 创建新的临时文件夹
# 创建临时文件夹是因为 pure-ftpd 在续传时会将文件名更改成不可预测的名字
# 正常中斷传输会把名字改回来, 但是意外掉线不会, 为了处理这种情况
# 需要获取 pure-ftpd 未知文件名的续传缓存文件, 为了不和其他视频的缓存文件混淆, 故建立一个临时文件夹
try:
ftp.cwd(tmp_dir)
except ftplib.error_perm:
ftp.mkd(tmp_dir)
ftp.cwd(tmp_dir)
return connect_flag
def exit_ftp(show_err=True):
try:
ftp.quit()
except BaseException as e:
if show_err and self._settings['ftp']['show_error_detail']:
err_print(self._sn, 'FTP狀態',
'將强制關閉FTP連接, 因爲在退出時收到異常: ' + str(e))
ftp.close()
def remove_dir(dir_name):
try:
ftp.rmd(dir_name)
except ftplib.error_perm as e:
if 'Directory not empty' in str(e):
# 如果目录非空, 则删除内部文件
ftp.cwd(dir_name)
del_all_files()
ftp.cwd('..')
ftp.rmd(dir_name) # 删完内部文件, 删除文件夹
elif 'No such file or directory' in str(e):
pass
else:
# 其他非空目录报错
raise e
def del_all_files():
try:
for file_need_del in ftp.nlst():
if not re.match(r'^(\.|\.\.)$', file_need_del):
ftp.delete(file_need_del)
# print('删除了文件: ' + file_need_del)
except ftplib.error_perm as resp:
if not str(resp) == "550 No files found":
raise
if not connect_ftp(): # 连接 FTP
return self.upload_succeed_flag # 如果连接失败
err_print(self._sn, '正在上傳',
self._video_filename + ' title=' + self._title + '……')
try_counter = 0
video_filename = self._video_filename # video_filename 将可能会储存 pure-ftpd 缓存文件名
max_try_num = self._settings['ftp']['max_retry_num']
local_size = os.path.getsize(self.local_video_path) # 本地文件大小
while try_counter <= max_try_num:
try:
if try_counter > 0:
# 传输遭中断后处理
detail = self._video_filename + ' 发生异常, 重連FTP, 續傳文件, 將重試最多' + str(
max_try_num) + '次……'
err_print(self._sn, '上傳狀態', detail, status=1)
if not connect_ftp(): # 重连
return self.upload_succeed_flag
# 解决操蛋的 Pure-Ftpd 续传一次就改名导致不能再续传问题.
# 一般正常关闭文件传输 Pure-Ftpd 会把名字改回来, 但是遇到网络意外中断, 那么就不会改回文件名, 留着临时文件名
# 本段就是处理这种情况
try:
for i in ftp.nlst():
if 'pureftpd-upload' in i:
# 找到 pure-ftpd 缓存, 直接抓缓存来续传
video_filename = i
except ftplib.error_perm as resp:
if not str(resp
) == "550 No files found": # 非文件不存在错误, 抛出异常
raise
# 断点续传
try:
# 需要 FTP Server 支持续传
ftp_binary_size = ftp.size(video_filename) # 远程文件字节数
except ftplib.error_perm:
# 如果不存在文件
ftp_binary_size = 0
except OSError:
try_counter = try_counter + 1
continue
ftp.voidcmd('TYPE I') # 二进制模式
conn = ftp.transfercmd('STOR ' + video_filename,
ftp_binary_size) # ftp服务器文件名和offset偏移地址
with open(self.local_video_path, 'rb') as f:
f.seek(ftp_binary_size) # 从断点处开始读取
while True:
block = f.read(1048576) # 读取1M
conn.sendall(block) # 送出 block
if not block:
time.sleep(3) # 等待一下, 让sendall()完成
break
conn.close()
err_print(self._sn, '上傳狀態', '檢查遠端文件大小是否與本地一致……')
exit_ftp(False)
connect_ftp(False)
# 不重连的话, 下面查询远程文件大小会返回 None, 懵逼...
# sendall()没有完成将会 500 Unknown command
err_counter = 0
remote_size = 0
while err_counter < 3:
try:
remote_size = ftp.size(video_filename) # 远程文件大小
break
except ftplib.error_perm as e1:
err_print(self._sn, 'FTP狀態',
'ftplib.error_perm: ' + str(e1))
remote_size = 0
break
except OSError as e2:
err_print(self._sn, 'FTP狀態', 'OSError: ' + str(e2))
remote_size = 0
connect_ftp(False) # 掉线重连
err_counter = err_counter + 1
if remote_size is None:
err_print(self._sn, 'FTP狀態', 'remote_size is None')
remote_size = 0
# 远程文件大小获取失败, 可能文件不存在或者抽风
# 那上面获取远程字节数将会是0, 导致重新下载, 那么此时应该清空缓存目录下的文件
# 避免后续找错文件续传
if remote_size == 0:
del_all_files()
if remote_size != local_size:
# 如果远程文件大小与本地不一致
# print('remote_size='+str(remote_size))
# print('local_size ='+str(local_size))
detail = self._video_filename + ' 在遠端為' + str(
round(remote_size / float(1024 * 1024),
2)) + 'MB' + ' 與本地' + str(
round(local_size / float(1024 * 1024), 2)
) + 'MB 不一致! 將重試最多' + str(max_try_num) + '次'
err_print(self._sn, '上傳狀態', detail, status=1)
try_counter = try_counter + 1
continue # 续传
# 顺利上传完后
ftp.cwd('..') # 返回上级目录, 即退出临时目录
try:
# 如果同名文件存在, 则删除
ftp.size(self._video_filename)
ftp.delete(self._video_filename)
except ftplib.error_perm:
pass
ftp.rename(tmp_dir + '/' + video_filename,
self._video_filename) # 将视频从临时文件移出, 顺便重命名
remove_dir(tmp_dir) # 删除临时目录
self.upload_succeed_flag = True # 标记上传成功
break
except ConnectionResetError as e:
if self._settings['ftp']['show_error_detail']:
detail = self._video_filename + ' 在上傳過程中網絡被重置, 將重試最多' + str(
max_try_num) + '次' + ', 收到異常: ' + str(e)
err_print(self._sn, '上傳狀態', detail, status=1)
try_counter = try_counter + 1
except TimeoutError as e:
if self._settings['ftp']['show_error_detail']:
detail = self._video_filename + ' 在上傳過程中超時, 將重試最多' + str(
max_try_num) + '次, 收到異常: ' + str(e)
err_print(self._sn, '上傳狀態', detail, status=1)
try_counter = try_counter + 1
except socket.timeout as e:
if self._settings['ftp']['show_error_detail']:
detail = self._video_filename + ' 在上傳過程socket超時, 將重試最多' + str(
max_try_num) + '次, 收到異常: ' + str(e)
err_print(self._sn, '上傳狀態', detail, status=1)
try_counter = try_counter + 1
if not self.upload_succeed_flag:
err_print(self._sn,
'上傳失敗',
self._video_filename + ' 放棄上傳!',
status=1)
exit_ftp()
return self.upload_succeed_flag
err_print(self._sn, '上傳完成', self._video_filename, status=2)
exit_ftp() # 登出 FTP
return self.upload_succeed_flag
class FTPClient(object):
"""Class FTPClient
"""
_mh = None
_client = None
_secured = None
_host = None
_port = None
_user = None
_passw = None
_path = None
_verbose = None
_is_connected = None
def __init__(self, secured=False, verbose=False):
"""Class constructor
Called when the object is initialized
Args:
secured (bool): secured FTP
verbose (bool): verbose mode
"""
self._mh = MasterHead.get_head()
self._secured = secured
if (not self._secured):
self._client = FTP()
else:
if (not(version_info[0] == 2 and version_info[1] == 6)):
self._client = FTP_TLS()
else:
raise NotImplementedError(
'Secured mode is not supported for Python 2.6')
self._verbose = verbose
if (self._verbose):
self._client.set_debuglevel(2)
@property
def client(self):
""" FTP client property getter """
return self._client
@property
def secured(self):
""" secured protocol mode property getter """
return self._secured
@property
def host(self):
""" server host property getter """
return self._host
@property
def port(self):
""" server port property getter """
return self._port
@property
def user(self):
""" username property getter """
return self._user
@property
def passw(self):
""" user password property getter """
return self._passw
@property
def path(self):
""" remote path property getter """
return self._path
@property
def verbose(self):
""" verbose mode property getter """
return self._verbose
@property
def is_connected(self):
""" is_connected property getter """
return self._is_connected
def connect(self, host, port=21, user=None, passw=None, path='/', timeout=10):
"""Method connects to server
Args:
host (str): server host
port (int): server port, default protocol port
user (str): username
passw (str): password
path (str): server path
timeout (int): timeout
Returns:
bool: result
Raises:
event: ftp_before_connect
event: ftp_after_connect
"""
try:
message = '{0}/{1}@{2}:{3}{4} timeout:{5}'.format(
user, passw, host, port, path, timeout)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_connecting', message), self._mh.fromhere())
ev = event.Event(
'ftp_before_connect', host, port, user, passw, path, timeout)
if (self._mh.fire_event(ev) > 0):
host = ev.argv(0)
port = ev.argv(1)
user = ev.argv(2)
passw = ev.argv(3)
path = ev.argv(4)
timeout = ev.argv(5)
self._host = host
self._port = port
self._user = user
self._passw = passw
if (ev.will_run_default()):
self._client.connect(self._host, self._port, timeout=timeout)
if (self._user != None):
self._client.login(self._user, self._passw)
if (self._secured):
self._client.prot_p()
self._is_connected = True
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_connected'), self._mh.fromhere())
if (path != None):
self.change_dir(path)
ev = event.Event('ftp_after_connect')
self._mh.fire_event(ev)
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def disconnect(self):
"""Method disconnects from server
Args:
none
Returns:
bool: result
"""
try:
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
else:
self._client.quit()
self._is_connected = False
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_disconnected'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def list_dir(self):
"""Method lists remote working directory
Args:
none
Returns:
list: names
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_list_dir', self._path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
names = self._client.nlst()
if ('.' in names):
names.remove('.')
if ('..' in names):
names.remove('..')
return names
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return None
def change_dir(self, path):
"""Method changes remote working directory
Args:
path (str): new remote path
Returns:
bool: result
Raises:
event: ftp_before_change_dir
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_change_dir', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_change_dir', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.cwd(path)
self._path = self._client.pwd()
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_cur_dir', self._path), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def download_file(self, remote_path, local_path=None):
"""Method downloads file from server
Args:
remote_path (str): remote path
local_path (str): local path, default ./filename
Returns:
bool: result
Raises:
event: ftp_before_download_file
event: ftp_after_download_file
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_downloading_file', remote_path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event(
'ftp_before_download_file', remote_path, local_path)
if (self._mh.fire_event(ev) > 0):
remote_path = ev.argv(0)
local_path = ev.argv(1)
if (local_path != None and not path.exists(local_path)):
self._mh.demsg('htk_on_error', self._mh._trn.msg(
'htk_ftp_unknown_dir', local_path), self._mh.fromhere())
return False
filename = remote_path.split('/')[-1]
lpath = filename if (local_path == None) else path.join(
local_path, filename)
if (ev.will_run_default()):
with open(lpath, 'wb') as f:
self._client.retrbinary('RETR ' + remote_path, f.write)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_file_downloaded'), self._mh.fromhere())
ev = event.Event('ftp_after_download_file')
self._mh.fire_event(ev)
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
if (path.exists(lpath)):
remove(lpath)
return False
def upload_file(self, local_path, remote_path=None):
"""Method uploads file to server
Args:
local_path (str): local path
remote_path (str): remote path, default ./filename
Returns:
bool: result
Raises:
event: ftp_before_upload_file
event: ftp_after_upload_file
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_uploading_file', local_path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_upload_file', local_path, remote_path)
if (self._mh.fire_event(ev) > 0):
local_path = ev.argv(0)
remote_path = ev.argv(1)
if (not(path.exists(local_path) or path.exists(path.relpath(local_path)))):
self._mh.demsg('htk_on_error', self._mh._trn.msg(
'htk_ftp_unknown_file', local_path), self._mh.fromhere())
return False
filename = local_path.split('/')[-1]
rpath = filename if (remote_path == None) else path.join(
remote_path, filename)
if (ev.will_run_default()):
with open(local_path, 'rb') as f:
self._client.storbinary('STOR ' + rpath, f)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_file_uploaded'), self._mh.fromhere())
ev = event.Event('ftp_after_upload_file')
self._mh.fire_event(ev)
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def delete_file(self, path):
"""Method deletes file from server
Args:
path (str): remote path
Returns:
bool: result
Raises:
event: ftp_before_delete_file
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_deleting_file', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_delete_file', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.delete(path)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_file_deleted'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def make_dir(self, path):
"""Method makes directory on server
Args:
path (str): remote path
Returns:
bool: result
Raises:
event: ftp_before_make_dir
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_making_dir', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_make_dir', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.mkd(path)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_dir_made'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
def remove_dir(self, path):
"""Method removes directory from server
Args:
path (str): remote path
Returns:
bool: result
Raises:
event: ftp_before_remove_dir
"""
try:
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_removing_dir', path), self._mh.fromhere())
if (not self._is_connected):
self._mh.demsg('htk_on_warning', self._mh._trn.msg(
'htk_ftp_not_connected'), self._mh.fromhere())
return False
ev = event.Event('ftp_before_remove_dir', path)
if (self._mh.fire_event(ev) > 0):
path = ev.argv(0)
if (ev.will_run_default()):
self._client.rmd(path)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_dir_removed'), self._mh.fromhere())
return True
except all_errors as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False
class ServerWatcher(Watcher):
downloadProgress = Signal((
int,
int,
))
uploadProgress = Signal((
int,
int,
))
# Si added:
textStatus = Signal((str, ))
fileEvent = Signal((str, ))
fileEventCompleted = Signal()
loginCompleted = Signal((
bool,
str,
))
badFilenameFound = Signal((str, ))
LOCATION = 'server'
TEST_FILE = 'iqbox.test'
def __init__(self, host, ssl, parent=None):
"""
Initializes parent class and attributes. Decides
whether to use `FTP_TLS` or `FTP` based on the `ssl` param.
:param host: Location of the FTP server
:param ssl: Tells whether the FTP needs to support TLS or not
:param parent: Reference to a `QObject` instance a parent
"""
super(ServerWatcher, self).__init__(parent)
self.interval = 5000
self.localdir = ''
self.deleteQueue = []
self.downloadQueue = []
self.uploadQueue = []
self.warnedNames = []
self.ftp = None
self.useSSL = ssl
self.host = host
self.preemptiveCheck = False
self.preemptiveActions = []
self.testFile = 'iqbox.test'
@property
def currentdir(self):
"""Returns the current working directory at the server"""
return self.ftp.pwd()
def setLocalDir(self, localdir):
"""
Sets the local directory used to stored all
downloaded files. Creates the directory if needed.
:param localdir: Absolute path to local directory
"""
self.localdir = localdir
if not os.path.exists(self.localdir):
os.makedirs(self.localdir)
@pause_timer
@Slot()
def checkout(self):
"""
Recursively checks out all files on the server.
Returns a dictionary of files on the server with their last modified date.
:param download: Indicates whether or not the files should be downloaded
"""
# Check `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
# These tasks are done in queues to make sure all FTP commands
# are done sequentially, in the same thread.
self.deleteAll()
self.uploadAll()
self.downloadAll()
# Handy list to keep track of the checkout process.
# This list contain absolute paths only.
checked_dirs = list()
# Sets '/' as initial directory and initializes `downloading_dir`
self.ftp.cwd('/')
downloading_dir = self.currentdir
check_date = dt.utcnow()
sidirlist = list()
root_cached = False
fileC = 0
while True:
# Gets the list of sub directories and files inside the
# current directory `downloading_dir`.
self.textStatus.emit('Remote scan- Downloading folder list of ' +
downloading_dir + '...')
if root_cached and downloading_dir == '/':
dir_subdirs = saved_root_dirs
dirfiles = saved_root_files
else:
dir_subdirs = self.getDirs(downloading_dir)
if downloading_dir == '/':
saved_root_dirs = dir_subdirs
# sidirlist.extend(dir_subdirs)
self.textStatus.emit(
'Remote scan- Downloading files list of ' +
downloading_dir + '...')
dirfiles = self.getFiles(downloading_dir)
if downloading_dir == '/':
saved_root_files = dirfiles
root_cached = True
# Leading '/' in `downloading_dir` breaks the `os.path.join` call
localdir = os.path.join(self.localdir, downloading_dir[1:])
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
for file_ in dirfiles:
# `serverpath` is the absolute path of the file on the server,
# download it only if it hasn't been already downloaded
serverpath = os.path.join(downloading_dir, file_)
serverpath = QDir.fromNativeSeparators(serverpath)
server_file = File.fromPath(serverpath)
self.textStatus.emit('Scanning remote file... ' + serverpath +
'...')
# How do we know if we should check this server file?
# We see if the date last checked is the check start time.
if server_file.last_checked_server != check_date:
# Do this process only once per file
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 1 == 2:
self.textStatus.emit(
'Scanning remote files for changes, ' +
str(fileC) + ' files scanned.')
# STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
just_added = not server_file.inserver
# STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
# Example: If it's a temporary file, or a Unix file with a name we don't support.
if just_added:
filename = os.path.basename(serverpath)
if platform.system() == 'Windows':
badName = False
for chr in [
'\\', '/', ':', '?', '"', '<', '>', '|'
]:
if chr in filename:
badName = True
break
if badName:
if filename not in self.warnedNames:
self.warnedNames.append(filename)
self.badFilenameFound.emit(filename)
continue
# STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
lastmdate = server_file.servermdate
# STEP: SAVE THE MOD DATE TO A VARIABLE
# Now we get the last mod time.
# We expect this to work fine since this file
# was found on the server
servermdate = self.lastModified(serverpath)
# STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
server_file.inserver = True
# STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
server_file.last_checked_server = check_date
# STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
server_file.servermdate = servermdate
# STEP: SAVE THIS CHANGE TO THE DATABASE
server_file.session.commit()
delta = 0
if server_file.inlocal:
delta = server_file.timeDiff()
# Emit the signals after the attributes has been set and committed
if just_added is True:
self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
self.fileChanged.emit(ServerWatcher.LOCATION,
serverpath, False)
#END FOR
self.textStatus.emit('Remote scan- Finding next folder...')
dir_ready = True
for dir_ in dir_subdirs:
# `dirpath` is the absolute path of the subdirectory on the server,
dirpath = QDir.fromNativeSeparators(
os.path.join(downloading_dir, dir_))
# `downloading_dir` is ready only when all its subdirectory are on the
# `checked_dirs` list.
if dirpath not in checked_dirs:
# Found one subdirectory that is not on `checked_dirs`,
# will process it in the next iteration.
downloading_dir = dirpath
dir_ready = False
break
if dir_ready is True:
# All subdirectories of `downloading_dir` are already in `checked_dirs`
if downloading_dir == '/':
# All directories ready and at '/', means checkout is complete
# So, exit the main While loop!!
break
else:
# Not at '/'. Current directory is ready so is appended to `checked_dirs`
# Back one directory to find directories that are not in `checked_dirs`
checked_dirs.append(downloading_dir)
downloading_dir = os.path.dirname(downloading_dir)
self.textStatus.emit('Remote scan- Found Folder...')
##### END OF WHILE ################
###################################################################
# Deleted files are the ones whose `last_checked_server` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(
File.last_checked_server < check_date).filter(
File.inserver == True)
for file_ in deleted:
self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)
# Wraps up the checkout process, commits to the database.
session.commit()
@Slot()
def onLogin(self, username, passwd):
ok = True
msg = ''
error_msg = 'Login failed.'
try:
if not self.ftp:
self.ftp = FTP_TLS(self.host) if self.useSSL is True else FTP(
self.host)
loginResponse = self.ftp.login(username, passwd)
except socket.gaierror:
self.ftp = None
ok = False
msg = 'Server address could not be found.'
except (error_perm, error_reply):
info = traceback.format_exception(*sys.exc_info())
for i in info:
sys.stderr.write(i)
ok = False
msg = error_msg
else:
if '230' in loginResponse:
ok = True
else:
ok = False
msg = error_msg
if ok:
# Logged in. Now let's do compability tests.
if not self.testPermissions():
# User doesn't have write permissions, don't bother doing next test.
ok = False
msg = 'It seems like you do not have write access to this server.'
else:
# Permissions test passed, now let's test MFMT for timestamp modification.
if not self.testMFMT():
ok = False
msg = 'This server does not support timestamp modification\n \
need by this application.'
self.loginCompleted.emit(ok, msg)
def getFiles(self, path):
"""
This method simply wraps the `nlst` method with an exception handler,
and returns an empty list in case an exception is caught.
:param path: Relative or absolute path on the server
"""
try:
nlst = self.ftp.nlst(path)
dirs = self.getDirs(path)
# Files are items in nlst that are not in dirs
files = [
item for item in nlst if os.path.basename(item) not in dirs
]
return files
except:
print 'Exception in ServerWatcher.getDirs'
info = traceback.format_exception(*sys.exc_info())
for i in info:
sys.stderr.write(i)
return []
def getDirs(self, path):
"""
Retrieves a list of the directories inside `path`,
uses `retrlines` and the LIST command to retrieve the items.
:param path: Relative or absolute path on the server
"""
dirs = list()
def handleLine(line):
"""
Recieves a line from the LIST command.
This function is meant to be used as callback for the `retrlines` method.
:params line: Line from the LIST command
"""
if line.startswith('d'):
# Only lines starting with 'd' are directories
# Parse the directory out of the line; lines look like:
# 'drwxrwxrwx 1 user group 0 Jun 15 2012 dirname'
dirname = line[55:].strip()
if dirname != '.' and dirname != '..':
# Ignoring '.' and '..' entries
dirs.append(dirname)
try:
self.ftp.retrlines('LIST %s' % path, handleLine)
return dirs
except:
print 'Exception in ServerWatcher.getDirs'
info = traceback.format_exception(*sys.exc_info())
for i in info:
sys.stderr.write(i)
return []
@upload_test
def testPermissions(self):
# For interface purposes. upload_test takes care of everything.
return True
@upload_test
def testMFMT(self):
# Absurd date to test whether the change really happened.
time = dt.utcfromtimestamp(100000000)
try:
self.setLastModified(self.testFile, time)
otherTime = self.lastModified(self.testFile)
diff = (time - otherTime).total_seconds()
if abs(diff) < 2:
# Let's give it a 2 seconds tolerance.
mdtm = True
else:
mdtm = False
except (ValueError, error_reply, error_perm):
info = traceback.format_exception(*sys.exc_info())
for i in info:
sys.stderr.write(i)
mdtm = False
return mdtm
@Slot(str)
def onDelete(self, filename):
self.deleteQueue.append(filename)
def deleteNext(self):
if len(self.deleteQueue) > 0:
next = self.deleteQueue.pop(0)
self.deleteFile(next)
def deleteAll(self):
for filename in self.deleteQueue:
self.deleteFile(filename)
self.deleteQueue = []
@Slot(str)
def deleteFile(self, filename):
"""
Deletes the file `filename` to the server
:param filename: Absolute or relative path to the file
"""
try:
print 'Deleting %s' % filename
self.ftp.delete(filename)
return True
except (error_reply, error_perm):
print 'Error deleting %s' % filename
return False
self.fileEventCompleted.emit()
@Slot(str)
def onDownload(self, filename):
self.downloadQueue.append(filename)
def downloadNext(self):
if len(self.downloadQueue) > 0:
next = self.downloadQueue.pop(0)
self.downloadFile(next)
def downloadAll(self):
for filename in self.downloadQueue:
self.downloadFile(filename)
self.downloadQueue = []
@Slot(str, str)
def downloadFile(self, filename, localpath=None):
"""
Performs a binary download to the file `filename` located on the server.
`filename` parameter can be either absolute or relative, though it can
fail for relative paths if the current directory is not appropiate.
:param filename: Relative or absolute path to the file
:param localpath: Absolute local path where the file will be saved
"""
def handleChunk(chunk):
"""
Receives chuncks of data downloaded from the server.
This function is meant to be used as callback for the `retrbinary` method.
:params chunk: Chunk of downloaded bytes to be written into the file
"""
# Simply writes the received data into the file `self.downloading`
self.downloading.write(chunk)
self.download_progress += len(chunk)
self.downloadProgress.emit(self.download_size,
self.download_progress)
if localpath is None:
localpath = self.localFromServer(filename)
localdir = os.path.dirname(localpath)
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
print 'Downloading: %s to %s' % (filename, localpath)
try:
with open(localpath, 'wb') as f:
# Opens the file at `localname` which will hold the downloaded file.
# Object attributes regarding download status are updated accordingly.
self.fileEvent.emit(filename)
self.downloading = f
self.download_progress = 0
self.download_size = int(
self.ftp.sendcmd('SIZE %s' % filename).split(' ')[-1])
self.ftp.retrbinary('RETR %s' % filename, handleChunk)
print 'Download finished'
# Let's set the same modified time to that on the server.
with File.fromPath(filename) as downloadedfile:
mdate = LocalWatcher.lastModified(localpath)
downloadedfile.localmdate = mdate
downloadedfile.servermdate = mdate
self.setLastModified(filename, mdate)
downloaded = True
except (IOError, OSError):
downloaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm) as ftperr:
print 'Error downloading %s, %s' % (filename, ftperr)
downloaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return downloaded
@Slot(str)
def onUpload(self, filename):
self.uploadQueue.append(filename)
def uploadNext(self):
if len(self.uploadQueue) > 0:
next = self.uploadQueue.pop(0)
self.uploadFile(next)
def uploadAll(self):
for filename in self.uploadQueue:
self.uploadFile(filename)
self.uploadQueue = []
@Slot(str)
def uploadFile(self, filename):
"""
Uploads the file `filename` to the server, creating
the needed directories.
:param filename: Absolute or relative path to the file
"""
def handle(buf):
"""This function is meant to be used as callback for the `storbinary` method."""
self.upload_progress += 1024
self.uploadProgress.emit(self.upload_size, self.upload_progress)
# Creates the directory where the file will be uploaded to
self.mkpath(os.path.dirname(filename))
localpath = self.localFromServer(filename)
print 'Uploading %s to %s' % (localpath, filename)
try:
# Uploads file and updates its modified date in the server
# to match the date in the local filesystem.
self.upload_progress = 0
self.upload_size = os.path.getsize(localpath)
self.fileEvent.emit(localpath)
self.ftp.storbinary('STOR %s' % filename, open(localpath, 'rb'),
1024, handle)
print 'Upload finished'
with File.fromPath(filename) as uploaded:
modified = uploaded.localmdate
uploaded.servermdate = modified
self.setLastModified(filename, modified)
uploaded = True
except (IOError, OSError):
uploaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm, OSError) as err:
print 'Error uploading %s, %s' % (filename, err)
uploaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return uploaded
def lastModified(self, filename):
"""
Uses the MDTM FTP command to find the last modified timestamp
of the file `filename`.
Returns a `datetime.datetime` object in UTC representing the file's
last modified date and time.
:param filename: Relative or absolute path to the file
"""
timestamp = self.ftp.sendcmd('MDTM %s' % filename)
if '213 ' not in timestamp:
# Second chance was found to be needed in some cases.
timestamp = self.ftp.sendcmd('MDTM %s' % filename)
timestamp = timestamp.split(' ')[-1]
dateformat = '%Y%m%d%H%M%S.%f' if '.' in timestamp else '%Y%m%d%H%M%S'
try:
mtime = dt.strptime(timestamp, dateformat)
except ValueError:
mtime = dt.utcnow()
return mtime
def setLastModified(self, serverpath, newtime):
"""
Uses the MFMT or MDTM FTP commands to set `newtime` as the modified timestamp of the
file `serverpath` on the server.
:param serverpath: Relative or absolute path to the file
:param newtime: datedatime object holding the required time
"""
cmds = ['MFMT', 'MDTM']
for cmd in cmds:
try:
self.ftp.sendcmd(
'%s %s %s' %
(cmd, newtime.strftime('%Y%m%d%H%M%S'), serverpath))
return
except (error_perm, error_reply) as e:
if cmd == cmds[len(cmds) - 1]:
# If is the last comand, re-raise the exception, else
# keep trying.
raise e
else:
continue
def mkpath(self, path):
"""
Creates the path `path` on the server by recursively
created folders, if needed.
:param path: Absolute path on the server to be created
"""
try:
self.ftp.cwd(path)
except error_perm:
# `cwd` call failed. Need to create some folders
make_dir = '/'
steps = path.split('/')
for step in steps:
if len(step) == 0:
continue
make_dir += '%s/' % step
try:
self.ftp.mkd(make_dir)
except error_perm:
# Probably already exists
continue
else:
# `cwd` call succeed. No need to create
# any folders
self.ftp.cwd('/')
return
@Slot(str, str)
def added(self, location, serverpath):
super(ServerWatcher, self).added(location, serverpath)
def actionFromPath(serverpath):
f = File()
fileExistsOnServer = True
try:
f.servermdate = self.lastModified(serverpath)
except error_perm:
fileExistsOnServer = False
f.servermdate = 0
f.localmdate = LocalWatcher.lastModified(
self.localFromServer(serverpath))
diff = f.timeDiff()
action = None
if abs(diff) > Watcher.TOLERANCE:
if not fileExistsOnServer or diff > 0:
action = FileAction(serverpath, FileAction.UPLOAD,
ServerWatcher.LOCATION)
else:
action = FileAction(serverpath, FileAction.DOWNLOAD,
LocalWatcher.LOCATION)
return action
if self.preemptiveCheck:
if location == ServerWatcher.LOCATION:
localpath = self.localFromServer(serverpath)
if not os.path.exists(localpath):
action = FileAction(serverpath, FileAction.DOWNLOAD,
ServerWatcher.LOCATION)
self.preemptiveActions.append(action)
else:
action = actionFromPath(serverpath)
if action is not None:
self.preemptiveActions.append(action)
elif location == LocalWatcher.LOCATION:
try:
self.ftp.sendcmd('SIZE %s' % serverpath)
except (error_reply, error_perm):
exists = False
else:
exists = True
if not exists:
action = FileAction(serverpath, FileAction.UPLOAD,
LocalWatcher.LOCATION)
self.preemptiveActions.append(action)
else:
action = actionFromPath(serverpath)
if action is not None:
self.preemptiveActions.append(action)
@Slot(str, str)
def changed(self, location, serverpath):
super(ServerWatcher, self).changed(location, serverpath)
@Slot(str, str)
def deleted(self, location, serverpath):
super(ServerWatcher, self).deleted(location, serverpath)
with File.fromPath(serverpath) as deleted:
deleted.inserver = False
class ServerWatcher(Watcher):
downloadProgress = Signal((int, int,))
uploadProgress = Signal((int, int,))
# Si added:
textStatus = Signal((str,))
fileEvent = Signal((str,))
fileEventCompleted = Signal()
loginCompleted = Signal((bool, str,))
badFilenameFound = Signal((str,))
LOCATION = 'server'
TEST_FILE = 'iqbox.test'
def __init__(self, host, ssl, parent=None):
"""
Initializes parent class and attributes. Decides
whether to use `FTP_TLS` or `FTP` based on the `ssl` param.
:param host: Location of the FTP server
:param ssl: Tells whether the FTP needs to support TLS or not
:param parent: Reference to a `QObject` instance a parent
"""
super(ServerWatcher, self).__init__(parent)
self.interval = 5000
self.localdir = ''
self.deleteQueue = []
self.downloadQueue = []
self.uploadQueue = []
self.warnedNames = []
self.ftp = None
self.useSSL = ssl
self.host = host
self.preemptiveCheck = False
self.preemptiveActions = []
self.testFile = 'iqbox.test'
@property
def currentdir(self):
"""Returns the current working directory at the server"""
return self.ftp.pwd()
def setLocalDir(self, localdir):
"""
Sets the local directory used to stored all
downloaded files. Creates the directory if needed.
:param localdir: Absolute path to local directory
"""
self.localdir = localdir
if not os.path.exists(self.localdir):
os.makedirs(self.localdir)
@pause_timer
@Slot()
def checkout(self):
"""
Recursively checks out all files on the server.
Returns a dictionary of files on the server with their last modified date.
:param download: Indicates whether or not the files should be downloaded
"""
# Check `self.deleteQueue`, `self.uploadQueue` and `self.downloadQueue` queues.
# These tasks are done in queues to make sure all FTP commands
# are done sequentially, in the same thread.
self.deleteAll()
self.uploadAll()
self.downloadAll()
# Handy list to keep track of the checkout process.
# This list contain absolute paths only.
checked_dirs = list()
# Sets '/' as initial directory and initializes `downloading_dir`
self.ftp.cwd('/')
downloading_dir = self.currentdir
check_date = dt.utcnow()
sidirlist = list()
root_cached = False
fileC = 0
while True:
# Gets the list of sub directories and files inside the
# current directory `downloading_dir`.
self.textStatus.emit('Remote scan- Downloading folder list of '+downloading_dir+'...')
if root_cached and downloading_dir == '/':
dir_subdirs = saved_root_dirs
dirfiles = saved_root_files
else:
dir_subdirs = self.getDirs(downloading_dir)
if downloading_dir == '/':
saved_root_dirs = dir_subdirs
# sidirlist.extend(dir_subdirs)
self.textStatus.emit('Remote scan- Downloading files list of '+downloading_dir+'...')
dirfiles = self.getFiles(downloading_dir)
if downloading_dir == '/':
saved_root_files = dirfiles
root_cached = True
# Leading '/' in `downloading_dir` breaks the `os.path.join` call
localdir = os.path.join(self.localdir, downloading_dir[1:])
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
for file_ in dirfiles:
# `serverpath` is the absolute path of the file on the server,
# download it only if it hasn't been already downloaded
serverpath = os.path.join(downloading_dir, file_)
serverpath = QDir.fromNativeSeparators(serverpath)
server_file = File.fromPath(serverpath)
self.textStatus.emit('Scanning remote file... '+serverpath+'...')
# How do we know if we should check this server file?
# We see if the date last checked is the check start time.
if server_file.last_checked_server != check_date:
# Do this process only once per file
# Added by Simon
# Give feedback on scanning of files.
fileC += 1
if fileC % 1 == 2:
self.textStatus.emit('Scanning remote files for changes, '+str(fileC)+' files scanned.')
# STEP: IS THIS THE FIRST TIME WE SAW THE FILE, OR WAS IT ALREADY IN OUR DB?
just_added = not server_file.inserver
# STEP: IF ITS A NEW FILE, ENSURE WE DONT WANT TO SKIP IT
# Example: If it's a temporary file, or a Unix file with a name we don't support.
if just_added:
filename = os.path.basename(serverpath)
if platform.system() == 'Windows':
badName = False
for chr in ['\\', '/', ':', '?', '"', '<', '>', '|']:
if chr in filename:
badName = True
break
if badName:
if filename not in self.warnedNames:
self.warnedNames.append(filename)
self.badFilenameFound.emit(filename)
continue
# STEP: ASSUMING THE FILE DID EXIST IN OUR DB, LETS SAVE THE LAST MODIFICATION DATE
lastmdate = server_file.servermdate
# STEP: SAVE THE MOD DATE TO A VARIABLE
# Now we get the last mod time.
# We expect this to work fine since this file
# was found on the server
servermdate = self.lastModified(serverpath)
# STEP: SET BOOL SHOWING THAT IT WAS ON THE SERVER, SINCE WE KNOW IT IS.
server_file.inserver = True
# STEP: SET THE TIME THE FILE WAS LAST CHECKED TO THE SCAN START TIME
server_file.last_checked_server = check_date
# STEP: SET THE MOD DATE IN THE DATABASE TO THE ONE WE JUST GOT
server_file.servermdate = servermdate
# STEP: SAVE THIS CHANGE TO THE DATABASE
server_file.session.commit()
delta = 0
if server_file.inlocal:
delta = server_file.timeDiff()
# Emit the signals after the attributes has been set and committed
if just_added is True:
self.fileAdded.emit(ServerWatcher.LOCATION, serverpath)
elif server_file.servermdate > lastmdate or delta < -Watcher.TOLERANCE:
self.fileChanged.emit(ServerWatcher.LOCATION, serverpath, False)
#END FOR
self.textStatus.emit('Remote scan- Finding next folder...')
dir_ready = True
for dir_ in dir_subdirs:
# `dirpath` is the absolute path of the subdirectory on the server,
dirpath = QDir.fromNativeSeparators(os.path.join(downloading_dir, dir_))
# `downloading_dir` is ready only when all its subdirectory are on the
# `checked_dirs` list.
if dirpath not in checked_dirs:
# Found one subdirectory that is not on `checked_dirs`,
# will process it in the next iteration.
downloading_dir = dirpath
dir_ready = False
break
if dir_ready is True:
# All subdirectories of `downloading_dir` are already in `checked_dirs`
if downloading_dir == '/':
# All directories ready and at '/', means checkout is complete
# So, exit the main While loop!!
break
else:
# Not at '/'. Current directory is ready so is appended to `checked_dirs`
# Back one directory to find directories that are not in `checked_dirs`
checked_dirs.append(downloading_dir)
downloading_dir = os.path.dirname(downloading_dir)
self.textStatus.emit('Remote scan- Found Folder...')
##### END OF WHILE ################
###################################################################
# Deleted files are the ones whose `last_checked_server` attribute
# didn't get updated in the recursive run.
session = Session()
deleted = session.query(File).filter(File.last_checked_server < check_date).filter(File.inserver == True)
for file_ in deleted:
self.fileDeleted.emit(ServerWatcher.LOCATION, file_.path)
# Wraps up the checkout process, commits to the database.
session.commit()
@Slot()
def onLogin(self, username, passwd):
ok = True
msg = ''
error_msg = 'Login failed.'
try:
if not self.ftp:
self.ftp = FTP_TLS(self.host) if self.useSSL is True else FTP(self.host)
loginResponse = self.ftp.login(username, passwd)
except socket.gaierror:
self.ftp = None
ok = False
msg = 'Server address could not be found.'
except (error_perm, error_reply):
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
ok = False
msg = error_msg
else:
if '230' in loginResponse:
ok = True
else:
ok = False
msg = error_msg
if ok:
# Logged in. Now let's do compability tests.
if not self.testPermissions():
# User doesn't have write permissions, don't bother doing next test.
ok = False
msg = 'It seems like you do not have write access to this server.'
else:
# Permissions test passed, now let's test MFMT for timestamp modification.
if not self.testMFMT():
ok = False
msg = 'This server does not support timestamp modification\n \
need by this application.'
self.loginCompleted.emit(ok, msg)
def getFiles(self, path):
"""
This method simply wraps the `nlst` method with an exception handler,
and returns an empty list in case an exception is caught.
:param path: Relative or absolute path on the server
"""
try:
nlst = self.ftp.nlst(path)
dirs = self.getDirs(path)
# Files are items in nlst that are not in dirs
files = [item for item in nlst if os.path.basename(item) not in dirs]
return files
except:
print 'Exception in ServerWatcher.getDirs'
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
return []
def getDirs(self, path):
"""
Retrieves a list of the directories inside `path`,
uses `retrlines` and the LIST command to retrieve the items.
:param path: Relative or absolute path on the server
"""
dirs = list()
def handleLine(line):
"""
Recieves a line from the LIST command.
This function is meant to be used as callback for the `retrlines` method.
:params line: Line from the LIST command
"""
if line.startswith('d'):
# Only lines starting with 'd' are directories
# Parse the directory out of the line; lines look like:
# 'drwxrwxrwx 1 user group 0 Jun 15 2012 dirname'
dirname = line[55:].strip()
if dirname != '.' and dirname != '..':
# Ignoring '.' and '..' entries
dirs.append(dirname)
try:
self.ftp.retrlines('LIST %s' % path, handleLine)
return dirs
except:
print 'Exception in ServerWatcher.getDirs'
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
return []
@upload_test
def testPermissions(self):
# For interface purposes. upload_test takes care of everything.
return True
@upload_test
def testMFMT(self):
# Absurd date to test whether the change really happened.
time = dt.utcfromtimestamp(100000000)
try:
self.setLastModified(self.testFile, time)
otherTime = self.lastModified(self.testFile)
diff = (time - otherTime).total_seconds()
if abs(diff) < 2:
# Let's give it a 2 seconds tolerance.
mdtm = True
else:
mdtm = False
except (ValueError, error_reply, error_perm):
info = traceback.format_exception(*sys.exc_info())
for i in info: sys.stderr.write(i)
mdtm = False
return mdtm
@Slot(str)
def onDelete(self, filename):
self.deleteQueue.append(filename)
def deleteNext(self):
if len(self.deleteQueue) > 0:
next = self.deleteQueue.pop(0)
self.deleteFile(next)
def deleteAll(self):
for filename in self.deleteQueue:
self.deleteFile(filename)
self.deleteQueue = []
@Slot(str)
def deleteFile(self, filename):
"""
Deletes the file `filename` to the server
:param filename: Absolute or relative path to the file
"""
try:
print 'Deleting %s' % filename
self.ftp.delete(filename)
return True
except (error_reply, error_perm):
print 'Error deleting %s' % filename
return False
self.fileEventCompleted.emit()
@Slot(str)
def onDownload(self, filename):
self.downloadQueue.append(filename)
def downloadNext(self):
if len(self.downloadQueue) > 0:
next = self.downloadQueue.pop(0)
self.downloadFile(next)
def downloadAll(self):
for filename in self.downloadQueue:
self.downloadFile(filename)
self.downloadQueue = []
@Slot(str, str)
def downloadFile(self, filename, localpath=None):
"""
Performs a binary download to the file `filename` located on the server.
`filename` parameter can be either absolute or relative, though it can
fail for relative paths if the current directory is not appropiate.
:param filename: Relative or absolute path to the file
:param localpath: Absolute local path where the file will be saved
"""
def handleChunk(chunk):
"""
Receives chuncks of data downloaded from the server.
This function is meant to be used as callback for the `retrbinary` method.
:params chunk: Chunk of downloaded bytes to be written into the file
"""
# Simply writes the received data into the file `self.downloading`
self.downloading.write(chunk)
self.download_progress += len(chunk)
self.downloadProgress.emit(self.download_size, self.download_progress)
if localpath is None:
localpath = self.localFromServer(filename)
localdir = os.path.dirname(localpath)
if not os.path.exists(localdir):
# Creates the directory if it doesn't already exists.
os.makedirs(localdir)
print 'Downloading: %s to %s' % (filename, localpath)
try:
with open(localpath, 'wb') as f:
# Opens the file at `localname` which will hold the downloaded file.
# Object attributes regarding download status are updated accordingly.
self.fileEvent.emit(filename)
self.downloading = f
self.download_progress = 0
self.download_size = int(self.ftp.sendcmd('SIZE %s' % filename).split(' ')[-1])
self.ftp.retrbinary('RETR %s' % filename, handleChunk)
print 'Download finished'
# Let's set the same modified time to that on the server.
with File.fromPath(filename) as downloadedfile:
mdate = LocalWatcher.lastModified(localpath)
downloadedfile.localmdate = mdate
downloadedfile.servermdate = mdate
self.setLastModified(filename, mdate)
downloaded = True
except (IOError, OSError):
downloaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm) as ftperr:
print 'Error downloading %s, %s' % (filename, ftperr)
downloaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return downloaded
@Slot(str)
def onUpload(self, filename):
self.uploadQueue.append(filename)
def uploadNext(self):
if len(self.uploadQueue) > 0:
next = self.uploadQueue.pop(0)
self.uploadFile(next)
def uploadAll(self):
for filename in self.uploadQueue:
self.uploadFile(filename)
self.uploadQueue = []
@Slot(str)
def uploadFile(self, filename):
"""
Uploads the file `filename` to the server, creating
the needed directories.
:param filename: Absolute or relative path to the file
"""
def handle(buf):
"""This function is meant to be used as callback for the `storbinary` method."""
self.upload_progress += 1024
self.uploadProgress.emit(self.upload_size, self.upload_progress)
# Creates the directory where the file will be uploaded to
self.mkpath(os.path.dirname(filename))
localpath = self.localFromServer(filename)
print 'Uploading %s to %s' % (localpath, filename)
try:
# Uploads file and updates its modified date in the server
# to match the date in the local filesystem.
self.upload_progress = 0
self.upload_size = os.path.getsize(localpath)
self.fileEvent.emit(localpath)
self.ftp.storbinary('STOR %s' % filename,
open(localpath, 'rb'),
1024,
handle)
print 'Upload finished'
with File.fromPath(filename) as uploaded:
modified = uploaded.localmdate
uploaded.servermdate = modified
self.setLastModified(filename, modified)
uploaded = True
except (IOError, OSError):
uploaded = False
self.ioError.emit(localpath)
except (error_reply, error_perm, OSError) as err:
print 'Error uploading %s, %s' % (filename, err)
uploaded = False
# TODO: Sometimes the file doesn't complete properly.
# in that case we maybe shouldn't call this?
self.fileEventCompleted.emit()
return uploaded
def lastModified(self, filename):
"""
Uses the MDTM FTP command to find the last modified timestamp
of the file `filename`.
Returns a `datetime.datetime` object in UTC representing the file's
last modified date and time.
:param filename: Relative or absolute path to the file
"""
timestamp = self.ftp.sendcmd('MDTM %s' % filename)
if '213 ' not in timestamp:
# Second chance was found to be needed in some cases.
timestamp = self.ftp.sendcmd('MDTM %s' % filename)
timestamp = timestamp.split(' ')[-1]
dateformat = '%Y%m%d%H%M%S.%f' if '.' in timestamp else '%Y%m%d%H%M%S'
try:
mtime = dt.strptime(timestamp, dateformat)
except ValueError:
mtime = dt.utcnow()
return mtime
def setLastModified(self, serverpath, newtime):
"""
Uses the MFMT or MDTM FTP commands to set `newtime` as the modified timestamp of the
file `serverpath` on the server.
:param serverpath: Relative or absolute path to the file
:param newtime: datedatime object holding the required time
"""
cmds = ['MFMT', 'MDTM']
for cmd in cmds:
try:
self.ftp.sendcmd(
'%s %s %s' % (cmd, newtime.strftime('%Y%m%d%H%M%S'), serverpath))
return
except (error_perm, error_reply) as e:
if cmd == cmds[len(cmds) - 1]:
# If is the last comand, re-raise the exception, else
# keep trying.
raise e
else:
continue
def mkpath(self, path):
"""
Creates the path `path` on the server by recursively
created folders, if needed.
:param path: Absolute path on the server to be created
"""
try:
self.ftp.cwd(path)
except error_perm:
# `cwd` call failed. Need to create some folders
make_dir = '/'
steps = path.split('/')
for step in steps:
if len(step) == 0:
continue
make_dir += '%s/' % step
try:
self.ftp.mkd(make_dir)
except error_perm:
# Probably already exists
continue
else:
# `cwd` call succeed. No need to create
# any folders
self.ftp.cwd('/')
return
@Slot(str, str)
def added(self, location, serverpath):
super(ServerWatcher, self).added(location, serverpath)
def actionFromPath(serverpath):
f = File()
fileExistsOnServer = True
try:
f.servermdate = self.lastModified(serverpath)
except error_perm:
fileExistsOnServer = False
f.servermdate = 0
f.localmdate = LocalWatcher.lastModified(self.localFromServer(serverpath))
diff = f.timeDiff()
action = None
if abs(diff) > Watcher.TOLERANCE:
if not fileExistsOnServer or diff > 0:
action = FileAction(serverpath, FileAction.UPLOAD, ServerWatcher.LOCATION)
else:
action = FileAction(serverpath, FileAction.DOWNLOAD, LocalWatcher.LOCATION)
return action
if self.preemptiveCheck:
if location == ServerWatcher.LOCATION:
localpath = self.localFromServer(serverpath)
if not os.path.exists(localpath):
action = FileAction(serverpath, FileAction.DOWNLOAD, ServerWatcher.LOCATION)
self.preemptiveActions.append(action)
else:
action = actionFromPath(serverpath)
if action is not None:
self.preemptiveActions.append(action)
elif location == LocalWatcher.LOCATION:
try:
self.ftp.sendcmd('SIZE %s' % serverpath)
except (error_reply, error_perm):
exists = False
else:
exists = True
if not exists:
action = FileAction(serverpath, FileAction.UPLOAD, LocalWatcher.LOCATION)
self.preemptiveActions.append(action)
else:
action = actionFromPath(serverpath)
if action is not None:
self.preemptiveActions.append(action)
@Slot(str, str)
def changed(self, location, serverpath):
super(ServerWatcher, self).changed(location, serverpath)
@Slot(str, str)
def deleted(self, location, serverpath):
super(ServerWatcher, self).deleted(location, serverpath)
with File.fromPath(serverpath) as deleted:
deleted.inserver = False
