def get_file_number(dirname):
global DIRECT_DOWNLOAD_TOTAL
global CONN2
#print("DEBUG get_file_number, dirname:'%s'"%dirname)
try:
CONN2.cwd(dirname)
CONN2.voidcmd('TYPE I')
except Exception as e:
printl('ERROR: cannot cd to "%s" due to %s' % (dirname, e))
return None
else:
#bug filelines not match with filelines_bk order
filelines = []
filelines_bk = []
filelines_bk = CONN2.nlst()
CONN2.dir(filelines.append)
if len(filelines_bk) != len(filelines):
printl("DEBUG ERROR number don't match\n")
i = 0
for file in filelines:
if '<DIR>' in file or file.startswith('d'):
if filelines_bk[i] in file:
get_file_number(filelines_bk[i])
else:
#print("disorderrrrrrrrrrrrrrrrr!")
#print ("DEBUG file=",file)
get_file_number(file.split()[-1])
CONN2.cwd('..')
else:
DIRECT_DOWNLOAD_TOTAL += 1
#print("DEBUG counting: ",DIRECT_DOWNLOAD_TOTAL)
i += 1
def start_monitor_download(self):
global MONITOR_STOP
if MONITOR_STOP:
MONITOR_STOP = False
self.button_monitor.config(text="Monitoring...",
bg='orange',
relief='sunken',
state='normal')
t_monitor = threading.Thread(target=self.monitor_ftpinfo)
t_download = threading.Thread(target=self.download_ftpinfo)
#for terminating purpose
MONITOR_THREADS.append(t_monitor)
MONITOR_THREADS.append(t_download)
t_monitor.start()
t_download.start()
print("DEBUG threads monitor {} and download {} start".format(
t_monitor, t_download))
#Here to start another consumer thread to trigger those ftp
else:
MONITOR_STOP = True
self.button_monitor.config(text="Stopping..",
bg='orange',
relief='sunken',
state='disable')
terminate_threads(MONITOR_THREADS)
printl("Monitor is terminated")
self.button_monitor.config(text="Start monitor",
bg='white',
relief='raised',
state='normal')
def start_direct_download(self):
global DIRECT_DOWNLOAD_STOP
global DIRECT_DOWNLOAD_THREADS
global PROGRESS_BAR
global PROGRESS_LBL
if DIRECT_DOWNLOAD_STOP:
DIRECT_DOWNLOAD_STOP = False
self.button_direct.config(text="Downloading...",
bg='orange',
relief='sunken',
state='normal')
t = threading.Thread(target=self.direct_download)
DIRECT_DOWNLOAD_THREADS.append(t)
t.start()
else:
DIRECT_DOWNLOAD_STOP = True
self.button_direct.config(text="Stopping..",
bg='orange',
relief='sunken',
state='disable')
terminate_threads(DIRECT_DOWNLOAD_THREADS)
printl("Direct Download is terminated")
self.button_direct.config(text="Direct download",
bg='white',
relief='raised',
state='normal')
PROGRESS_BAR.pack_forget()
PROGRESS_LBL.pack_forget()
def my_upload(host, port, acc, pwd, file_path, remote_path):
global CONN
if not ftp_conn(host, port, acc, pwd, 3):
return False
if not ftp_upload_file(file_path, remote_path):
printl('Upload error, exited')
return False
else:
printl("Upload file {} successfully!".format(remote_path))
return True
def new_conn_get_file_number(host, port, acc, pwd, download_dir):
global CONN2
print('start the conn2 for file number, host:{0}, port:{1}, acc:{2}, pwd:{3}'\
.format(host,port,acc,pwd))
try:
CONN2 = ftplib.FTP()
#set a 3 seconds timeout
CONN2.connect(host, port, 30000)
except (socket.error, socket.gaierror), e:
printl('ERROR: cannot reach host "%s", exited.' % host)
return False
def choose_dir(self, ev=None):
global SAVE_DIR
p = askdirectory(parent=self.ftp_top, mustexist=1) # 返回目录路径
print("choose to save in directory:", p)
if 'unicode' in str(type(p)):
p = p.encode('utf-8') #.decode('gb2312')
elif 'str' in str(type(p)):
p = p.decode('utf-8')
print(p)
if p:
SAVE_DIR = p
printl("Updated save dir: %s" % SAVE_DIR)
self.v_savein.set(p)
def ftp_conn(host, port, acc, pwd, timeout=None):
global CONN
printl('ftp_conn start, host:{0}, port:{1}, acc:{2}, pwd:{3}'\
.format(host,port,acc,pwd))
try:
CONN = ftplib.FTP()
#set a 3 seconds timeout
if not timeout:
CONN.connect(host, port, timeout)
else:
CONN.connect(host, port)
except (socket.error, socket.gaierror), e:
printl('ERROR: cannot reach host "%s", exited.' % host)
return False
def ftp_upload_file(file_path, remote_path):
global CONN
printl('Ftp upload start, file_name:{0}, destination:{1}'\
.format(file_path,remote_path))
bufsize = 1024
fp = open(file_path, 'rb')
try:
CONN.storbinary('STOR ' + remote_path, fp, bufsize)
except Exception as e:
printl("error when uploading, e={}".format(e))
return False
fp.close()
return True
def retrive_bak():
global HOST
global PORT
global ACC
global PWD
global DOWNLOAD_DIR
global MONITOR_INTERVAL
global L_RESERVED_FTP
global EXSERVER
global MAIL_ADD
global AD4_ACC
global AD4_PWD
printl("'Welcome to Mail Monitor v2.1'")
try:
data_bak = pickle.load(open(DATA_BAK_FILE, "rb"))
#printl("Retrive data_bak:{}".format(data_bak))
HOST = data_bak.ftp_bak.host
PORT = data_bak.ftp_bak.port
ACC = data_bak.ftp_bak.user
PWD = data_bak.ftp_bak.pwd
DOWNLOAD_DIR = data_bak.ftp_bak.target_dir
MONITOR_INTERVAL = data_bak.ftp_bak.interval
L_RESERVED_FTP = data_bak.ftp_bak.l_reserved
EXSERVER = data_bak.ol_bak.server
MAIL_ADD = data_bak.ol_bak.mail
AD4_ACC = data_bak.ol_bak.user
AD4_PWD = data_bak.ol_bak.pwd
except Exception as e:
#printl("ERROR occure, e= %s" %e)
if not L_RESERVED_FTP:
retrive_reserved_ftp()
pass
else:
#printl("Retrive success!\n")
return data_bak
return None
def my_download(host, port, acc, pwd, save_dir, download_dir):
global CONN
global DIRECT_DOWNLOAD_TOTAL
global DIRECT_DOWNLOAD_COUNT
global PROGRESS_COST_SEC
global PROGRESS_TOTAL_BYTES
global PROGRESS_STRVAR
global PROGRESS_BAR
global PROGRESS_LBL
down_name = os.path.basename(download_dir)
printl("my_download starts,download_dir: '%s'" % (download_dir))
#if this file had been downloaded, quit
printl("Check exists of dir: %s" % (os.path.join(save_dir, down_name)))
if os.path.exists(os.path.join(save_dir, down_name)):
printl("Dir already exsit:%s, quit download." %
os.path.join(save_dir, down_name))
return None
os.chdir(save_dir)
if not ftp_conn(host, port, acc, pwd, 30000):
return None
printl("Begin to download, calculating files number...")
DIRECT_DOWNLOAD_TOTAL = 0
DIRECT_DOWNLOAD_COUNT = 0
start_get_file_number(host, port, acc, pwd, download_dir)
#CONN.set_debuglevel(1)
#get_file_number(download_dir)
if DIRECT_DOWNLOAD_TOTAL == 0:
pass
#return None
#printl("Total %d files" % (DIRECT_DOWNLOAD_TOTAL))
PROGRESS_COST_SEC = 0
PROGRESS_TOTAL_BYTES = 0
PROGRESS_BAR.pack(side=LEFT)
PROGRESS_LBL.pack(side=LEFT)
download_success = ftp_download_dir(download_dir)
CONN.quit()
PROGRESS_BAR.pack_forget()
PROGRESS_LBL.pack_forget()
if not download_success:
return None
return os.path.join(save_dir, os.path.basename(download_dir))
def find_mail(self, mail_subject_keyword):
'''
check the inbox and return
a mail list that contains new mails subjects
which received time is > the specific time
'''
print("Start to check new mail received after time: %s" %
str(self.utc_time_point))
#mail_list = []
my_inbox = self.my_account.inbox
my_inbox.refresh()
mail_number = my_inbox.total_count
latest_mail = my_inbox.all().order_by('-datetime_received')[:1].next()
latest_mail_time = latest_mail.datetime_received + timedelta(hours=8)
if self.utc_time_point >= latest_mail_time:
print("No new mail..")
yield None
re_rule = re.compile(mail_subject_keyword, re.I)
latest_ten = my_inbox.all().order_by('-datetime_received')[:10]
for item in latest_ten:
d_rec = item.datetime_received + timedelta(hours=8)
if self.utc_time_point >= d_rec:
break
subject = item.subject
if subject == None:
subject = ''
if re_rule.search(subject):
printl("\nDetect a new mail, Date:[%s], subject:[%s]" %
(str(d_rec), subject))
yield item
#update time to the checked latest mail's
self.utc_time_point = latest_mail_time
def periodical_check(self):
global AUTOANA_ENABLE
if self.v_chk.get() == 1:
printl("Enable periodical monitor")
self.entry_mail_k.config(state='normal')
self.button_monitor.config(state='normal')
self.label_mail_k.config(state='normal')
self.spin_interval.config(state='normal')
self.label_interval.config(state='normal')
AUTOANA_ENABLE = True
self.label_exserver.config(state='normal')
self.entry_exserver.config(state='normal')
self.label_mail_add.config(state='normal')
self.entry_mail_add.config(state='normal')
self.label_csl.config(state='normal')
self.entry_csl.config(state='normal')
self.label_cip.config(state='normal')
self.entry_cip.config(state='normal')
else:
#printl("Periodical monitor disabled")
self.entry_mail_k.config(state='disable')
self.button_monitor.config(state='disable')
self.label_mail_k.config(state='disable')
self.spin_interval.config(state='disable')
self.label_interval.config(state='disable')
AUTOANA_ENABLE = False
self.label_exserver.config(state='disable')
self.entry_exserver.config(state='disable')
self.label_mail_add.config(state='disable')
self.entry_mail_add.config(state='disable')
self.label_csl.config(state='disable')
self.entry_csl.config(state='disable')
self.label_cip.config(state='disable')
self.entry_cip.config(state='disable')
def direct_download(self):
global HOST
global PORT
global ACC
global PWD
global SAVE_DIR
global DOWNLOAD_DIR
global DIRECT_DOWNLOAD_STOP
global L_RESERVED_FTP
global PROGRESS_TOTAL_BYTES
global PROGRESS_COST_SEC
printl("Direct_download starts")
#extract ftp info
if self.v_host.get():
ftp_info = self.extract_ftp_info(self.v_host.get(),
from_mail=False)
#FTP_INFO = collections.namedtuple("FTP_INFO", "HOST PORT ACC PWD DIRNAME")
if ftp_info:
self.v_host.set(ftp_info.HOST)
self.v_port.set(ftp_info.PORT)
self.v_user.set(ftp_info.ACC)
self.v_pwd.set(ftp_info.PWD)
self.v_ddirname.set(ftp_info.DIRNAME)
if self.v_host.get():
HOST = self.v_host.get()
if self.v_port.get():
PORT = self.v_port.get()
if self.v_user.get():
ACC = self.v_user.get()
if self.v_pwd.get():
PWD = self.v_pwd.get()
if self.v_ddirname.get():
#get rid of the last '/' if had
self.v_ddirname.set(self.v_ddirname.get().strip())
if self.v_ddirname.get()[-1] == '/':
self.v_ddirname.set(self.v_ddirname.get()[:-1])
DOWNLOAD_DIR = self.v_ddirname.get()
#set host to display as ip:port format
self.v_host.set(HOST + ':' + PORT)
file_saved = ''
start_t = time.clock()
try:
file_saved = my_download(HOST, PORT, ACC, PWD, SAVE_DIR,
DOWNLOAD_DIR)
except Exception as e:
print("my_download error", e)
file_saved = None
end_t = time.clock()
interval_t = time.clock() - start_t
print("DEBUG file_saved=", file_saved)
if not file_saved:
print("DEBUG Download failed, time used: %.1f seconds" %
interval_t)
#crash
#showerror(title='Ftp Connect Error', message="Cannot accesst to %s" % HOST)
else:
self.l_savein.bell()
printl("Download completed in: %s, total time: %.1f seconds" %
(file_saved, interval_t))
print("DEBUG total bytes:{} total sec:{}".format(
PROGRESS_TOTAL_BYTES, PROGRESS_COST_SEC))
#update MM's knowledge about ftp info
s = r"ftp://" + ACC + ":" + PWD + "@" + HOST + ":" + PORT
if s not in L_RESERVED_FTP:
L_RESERVED_FTP.append(s)
printl("New ftp address learned:", s)
if AUTOANA_ENABLE:
#send to queue for other processing, e.g.,auto search in listdir.py
print("DEBUG my_ftp.py send %s to auto analyse" % file_saved)
FTP_FILE_QUE.put(file_saved)
DIRECT_DOWNLOAD_STOP = True
self.button_direct.config(text="Direct download",
bg='white',
relief='raised',
state='normal')
return file_saved
def ask_quit(self, ftp_top):
global HOST
global PORT
global ACC
global PWD
global DOWNLOAD_DIR
global MAIL_KEYWORD
global MONITOR_INTERVAL
global EXSERVER
global MAIL_ADD
global AD4_ACC
global AD4_PWD
global FTQ
res = askyesnocancel("Mail Monitor",
"(Recommended)Save current configurations?",
parent=ftp_top)
if res:
#save
HOST = self.v_host.get()
PORT = self.v_port.get()
ACC = self.v_user.get()
PWD = self.v_pwd.get()
DOWNLOAD_DIR = self.v_ddirname.get()
MAIL_KEYWORD = self.v_mail_k.get()
MONITOR_INTERVAL = self.v_interval.get()
EXSERVER = self.v_exserver.get()
MAIL_ADD = self.v_mail_k_add.get()
AD4_ACC = self.v_csl.get()
#do not save password
#self.v_cip.set('')
AD4_PWD = self.v_cip.get()
save_bak()
elif res == False:
pass
else:
#'cancel' nothing to do
return
printl("'Mail Monitor exited'\n")
self.running = False
FTP_FILE_QUE.put('ftp quit')
#terminate progress threads
#crash don't know why
#terminate_threads(PROGRESS_THREADS)
if __name__ == '__main__':
#quit() all windows and parent window to be closed
ftp_top.quit()
else:
#important remember to do these stuffs before destruction!
#Even though the window is destoyed, thease globals still in ram
FTQ.queue.clear()
#derstroy only this window
ftp_top.destroy()
l_thr = []
l_thr.extend(MONITOR_THREADS)
l_thr.extend(DIRECT_DOWNLOAD_THREADS)
l_thr.extend(PROGRESS_THREADS)
terminate_threads(l_thr)
for l in l_thr:
if l.is_alive():
l._Thread__stop()
'''
def monitor_ftpinfo(self):
'''
get ftp info extracted from mail body
and the mail comes from read exchange server by read_exchange.py
'''
global MONITOR_STOP
global DIRECT_DOWNLOAD_STOP
global L_RESERVED_FTP
global FTP_INFO_HISTORY
global MUTEXT
interval_count = 0
find_folder = "inbox"
try:
acc = self.v_csl.get()
pwd = self.v_cip.get()
ser = self.v_exserver.get()
mail = self.v_mail_k_add.get()
my_ol = read_exchange.MY_OUTLOOK(acc, pwd, ser, mail)
except Exception as e:
printl("Error exchanger account: {} connecting failed: {}".format(
mail, e))
else:
printl("Start monitoring exchange account %s" % mail)
del_html = re.compile(r'<[^>]+>', re.S)
while 1:
if MONITOR_STOP or (not self.running):
break
for mail_item in my_ol.find_mail(self.v_mail_k.get()):
if not mail_item:
break
else:
#sound a bell
self.l_savein.bell()
#Extract ftp info and then start to download
plain_body = del_html.sub('', mail_item.body)
#print("DEBUG plain_body:",plain_body)
ftp_info = self.extract_ftp_info(plain_body,
from_mail=True)
if (ftp_info != None) and (ftp_info
not in FTP_INFO_HISTORY):
FTP_INFO_HISTORY.append(ftp_info)
else:
print("DBUEG already handled ftp_info")
continue
#only dirname case:
try_ftp_list = []
if ftp_info != None and ftp_info.HOST == '' and ftp_info.ACC == '' and ftp_info.DIRNAME != '':
#use reserved ftp info to combine this dirname to try:
for ftp_addr in L_RESERVED_FTP:
#here is it necessary to add the '\r' ending?
s = ftp_addr + ftp_info.DIRNAME + '\r'
if 'unicode' in str(type(s)):
s = s.encode('utf-8')
try_ftp_list.append(s)
#is here necessary to do the mutex?yes
self.ftp_queue.append(try_ftp_list[:])
#for the full ftp case
elif ftp_info != None and ftp_info.HOST != '' and ftp_info.ACC != '':
printl("Detected ftp_info:{}".format(ftp_info))
#FTP_INFO = collections.namedtuple("FTP_INFO", "HOST PORT ACC PWD DIRNAME")
#must add '\\r' because extract_ftp_info use this as an end
s = r"ftp://"+ftp_info.ACC+":"+ftp_info.PWD+"@"+ftp_info.HOST+":"+\
ftp_info.PORT+ftp_info.DIRNAME + '\r'
if 'unicode' in str(type(s)):
s = s.encode('utf-8')
self.ftp_queue.append([s])
else:
print("DEBUG error, should no such case")
pass
self.v_ftp_number.set(str(len(self.ftp_queue)))
sub = mail_item.subject
printl("Detect a ftp_info: %s from mail: %s" %
(str(ftp_info), sub))
#end for
time.sleep(int(self.v_interval.get()))
interval_count += 1
print("Monitor ftp>>>>, count %d"\
% (interval_count))
#end while
MONITOR_STOP = True
self.button_monitor.config(text="Start monitor",
bg='white',
relief='raised',
state='normal')
def download_ftpinfo(self):
global DIRECT_DOWNLOAD_STOP
print("DEBUG start download_ftpinfo circle")
interval_count = 0
while 1:
if MONITOR_STOP or (not self.running):
printl("download_ftpinfo thread stopped")
break
if self.ftp_queue:
#pop out the first ftp_info_list
ftp_info_list = self.ftp_queue[0][:]
#ftp_info_list len >1 only when the case from partial ftp info
#that guessed from many reserved ftp address
#so once one ftp_info was successfully download
#quit the for circle
print("DEBUG start to download ftp_info_list", ftp_info_list)
file_saved = ''
for ftp_info in ftp_info_list:
self.v_host.set(ftp_info)
if DIRECT_DOWNLOAD_STOP:
DIRECT_DOWNLOAD_STOP = False
self.button_direct.config\
(text="Downloading...",bg='orange',relief='sunken',state='normal')
file_saved = self.direct_download()
if file_saved:
self.v_saved_number += 1
self.l_savein.config(
text='%s' %
(str(self.v_saved_number) + ' Saved in: '),
bg='orange')
#quit after downloading successuflly
break
else:
printl("try another reserved ftp address...")
continue
else:
printl(
"Error, another direct download is under processing, you can't start to download this!"
)
self.ftp_queue.insert(0, ftp_info_list)
break
#record all the download info into ftp_log.txt
if not file_saved:
file_saved = "Failed"
printl("ftp_info: '%s' download result: '%s'" %
(str(ftp_info), file_saved))
self.ftp_queue.pop(0)
self.v_ftp_number.set(str(len(self.ftp_queue)))
else:
pass
time.sleep(int(self.v_interval.get()))
interval_count += 1
print(">>>>Download ftp, count %d" % (interval_count))
#while
print("DEBUG download_ftpinfo stopped.")
def ftp_download_dir(dirname):
global CONN
global DIRECT_DOWNLOAD_COUNT
global DIRECT_DOWNLOAD_TOTAL
global DIRECT_DOWNLOAD_BYTES
print("ftp_download_dir, dirname: '%s'" % dirname)
try:
CONN.cwd(dirname)
except ftplib.error_perm:
printl('ERROR: cannot cd to "%s"' % dirname)
return False
else:
new_dir = os.path.basename(dirname)
if not os.path.exists(new_dir):
os.mkdir(new_dir)
os.chdir(new_dir)
filelines = []
filelines_bk = []
CONN.dir(filelines.append)
filelines_bk = CONN.nlst()
if len(filelines_bk) != len(filelines):
print("Error, Number does not match!\n")
i = 0
for file in filelines:
#unmatched file_name
file_name = ''
if filelines_bk[i] in file:
file_name = filelines_bk[i]
else:
#the filelines_bk is not in the same order with filelines
#filelines like:
#'drwxr-xr-x 2 ftpalcatel users 1024 Oct 26 14:46 SQ2DSL02.02C_20171026095345.0'
#There is a bug if the file name contains blanks
file_name = file.split()[-1]
#<DIR> display in widows and drwxr- in linux
#here maybe a bug if only a file with name 'dxxxx'
if '<DIR>' in file or file.startswith('d'):
ftp_download_dir(file_name)
CONN.cwd('..')
os.chdir('..')
else:
try:
#how to speed up?
DIRECT_DOWNLOAD_BYTES = 0
#change transfor mode from ASCII to Binary
#then allowed to get file_size
CONN.voidcmd('TYPE I')
file_size = CONN.size(file_name)
maxblocksize = 2097152
printl("Downloading[%d/%d]: %s"%\
(DIRECT_DOWNLOAD_COUNT+1, DIRECT_DOWNLOAD_TOTAL, file_name))
file_to_write = open(file_name, 'wb').write
CONN.retrbinary('RETR %s' % file_name, \
#use lambda to pass multiple paremeters
#open(filelines_bk[i], 'wb').write,maxblocksize)
lambda block: ftp_write_handle(block,file_size,file_to_write), maxblocksize)
#except ftplib.error_perm:
except Exception as e:
printl('ERROR: cannot download file "%s"' % file_name)
print("error:", e)
try:
os.unlink(file_name)
except Exception as e:
#print("DEBUG os.unlink error:",e)
continue
else:
print("DEBUG os.unlink successed")
continue
else:
DIRECT_DOWNLOAD_COUNT += 1
i += 1
return True
global CONN
printl('ftp_conn start, host:{0}, port:{1}, acc:{2}, pwd:{3}'\
.format(host,port,acc,pwd))
try:
CONN = ftplib.FTP()
#set a 3 seconds timeout
if not timeout:
CONN.connect(host, port, timeout)
else:
CONN.connect(host, port)
except (socket.error, socket.gaierror), e:
printl('ERROR: cannot reach host "%s", exited.' % host)
return False
printl("Successfully connected to host '%s'" % host)
try:
CONN.login(acc, pwd)
except ftplib.error_perm:
printl('ERROR: cannot login as "%s", exited.' % acc)
CONN.quit()
return False
printl("Successfully logged in as '%s'" % acc)
return True
############ftp_conn()#####################
