def get_file_number(dirname): global DIRECT_DOWNLOAD_TOTAL global CONN2 #print("DEBUG get_file_number, dirname:'%s'"%dirname) try: CONN2.cwd(dirname) CONN2.voidcmd('TYPE I') except Exception as e: printl('ERROR: cannot cd to "%s" due to %s' % (dirname, e)) return None else: #bug filelines not match with filelines_bk order filelines = [] filelines_bk = [] filelines_bk = CONN2.nlst() CONN2.dir(filelines.append) if len(filelines_bk) != len(filelines): printl("DEBUG ERROR number don't match\n") i = 0 for file in filelines: if '<DIR>' in file or file.startswith('d'): if filelines_bk[i] in file: get_file_number(filelines_bk[i]) else: #print("disorderrrrrrrrrrrrrrrrr!") #print ("DEBUG file=",file) get_file_number(file.split()[-1]) CONN2.cwd('..') else: DIRECT_DOWNLOAD_TOTAL += 1 #print("DEBUG counting: ",DIRECT_DOWNLOAD_TOTAL) i += 1
def start_monitor_download(self): global MONITOR_STOP if MONITOR_STOP: MONITOR_STOP = False self.button_monitor.config(text="Monitoring...", bg='orange', relief='sunken', state='normal') t_monitor = threading.Thread(target=self.monitor_ftpinfo) t_download = threading.Thread(target=self.download_ftpinfo) #for terminating purpose MONITOR_THREADS.append(t_monitor) MONITOR_THREADS.append(t_download) t_monitor.start() t_download.start() print("DEBUG threads monitor {} and download {} start".format( t_monitor, t_download)) #Here to start another consumer thread to trigger those ftp else: MONITOR_STOP = True self.button_monitor.config(text="Stopping..", bg='orange', relief='sunken', state='disable') terminate_threads(MONITOR_THREADS) printl("Monitor is terminated") self.button_monitor.config(text="Start monitor", bg='white', relief='raised', state='normal')
def start_direct_download(self): global DIRECT_DOWNLOAD_STOP global DIRECT_DOWNLOAD_THREADS global PROGRESS_BAR global PROGRESS_LBL if DIRECT_DOWNLOAD_STOP: DIRECT_DOWNLOAD_STOP = False self.button_direct.config(text="Downloading...", bg='orange', relief='sunken', state='normal') t = threading.Thread(target=self.direct_download) DIRECT_DOWNLOAD_THREADS.append(t) t.start() else: DIRECT_DOWNLOAD_STOP = True self.button_direct.config(text="Stopping..", bg='orange', relief='sunken', state='disable') terminate_threads(DIRECT_DOWNLOAD_THREADS) printl("Direct Download is terminated") self.button_direct.config(text="Direct download", bg='white', relief='raised', state='normal') PROGRESS_BAR.pack_forget() PROGRESS_LBL.pack_forget()
def my_upload(host, port, acc, pwd, file_path, remote_path): global CONN if not ftp_conn(host, port, acc, pwd, 3): return False if not ftp_upload_file(file_path, remote_path): printl('Upload error, exited') return False else: printl("Upload file {} successfully!".format(remote_path)) return True
def new_conn_get_file_number(host, port, acc, pwd, download_dir): global CONN2 print('start the conn2 for file number, host:{0}, port:{1}, acc:{2}, pwd:{3}'\ .format(host,port,acc,pwd)) try: CONN2 = ftplib.FTP() #set a 3 seconds timeout CONN2.connect(host, port, 30000) except (socket.error, socket.gaierror), e: printl('ERROR: cannot reach host "%s", exited.' % host) return False
def choose_dir(self, ev=None): global SAVE_DIR p = askdirectory(parent=self.ftp_top, mustexist=1) # 返回目录路径 print("choose to save in directory:", p) if 'unicode' in str(type(p)): p = p.encode('utf-8') #.decode('gb2312') elif 'str' in str(type(p)): p = p.decode('utf-8') print(p) if p: SAVE_DIR = p printl("Updated save dir: %s" % SAVE_DIR) self.v_savein.set(p)
def ftp_conn(host, port, acc, pwd, timeout=None): global CONN printl('ftp_conn start, host:{0}, port:{1}, acc:{2}, pwd:{3}'\ .format(host,port,acc,pwd)) try: CONN = ftplib.FTP() #set a 3 seconds timeout if not timeout: CONN.connect(host, port, timeout) else: CONN.connect(host, port) except (socket.error, socket.gaierror), e: printl('ERROR: cannot reach host "%s", exited.' % host) return False
def ftp_upload_file(file_path, remote_path): global CONN printl('Ftp upload start, file_name:{0}, destination:{1}'\ .format(file_path,remote_path)) bufsize = 1024 fp = open(file_path, 'rb') try: CONN.storbinary('STOR ' + remote_path, fp, bufsize) except Exception as e: printl("error when uploading, e={}".format(e)) return False fp.close() return True
def retrive_bak(): global HOST global PORT global ACC global PWD global DOWNLOAD_DIR global MONITOR_INTERVAL global L_RESERVED_FTP global EXSERVER global MAIL_ADD global AD4_ACC global AD4_PWD printl("'Welcome to Mail Monitor v2.1'") try: data_bak = pickle.load(open(DATA_BAK_FILE, "rb")) #printl("Retrive data_bak:{}".format(data_bak)) HOST = data_bak.ftp_bak.host PORT = data_bak.ftp_bak.port ACC = data_bak.ftp_bak.user PWD = data_bak.ftp_bak.pwd DOWNLOAD_DIR = data_bak.ftp_bak.target_dir MONITOR_INTERVAL = data_bak.ftp_bak.interval L_RESERVED_FTP = data_bak.ftp_bak.l_reserved EXSERVER = data_bak.ol_bak.server MAIL_ADD = data_bak.ol_bak.mail AD4_ACC = data_bak.ol_bak.user AD4_PWD = data_bak.ol_bak.pwd except Exception as e: #printl("ERROR occure, e= %s" %e) if not L_RESERVED_FTP: retrive_reserved_ftp() pass else: #printl("Retrive success!\n") return data_bak return None
def my_download(host, port, acc, pwd, save_dir, download_dir): global CONN global DIRECT_DOWNLOAD_TOTAL global DIRECT_DOWNLOAD_COUNT global PROGRESS_COST_SEC global PROGRESS_TOTAL_BYTES global PROGRESS_STRVAR global PROGRESS_BAR global PROGRESS_LBL down_name = os.path.basename(download_dir) printl("my_download starts,download_dir: '%s'" % (download_dir)) #if this file had been downloaded, quit printl("Check exists of dir: %s" % (os.path.join(save_dir, down_name))) if os.path.exists(os.path.join(save_dir, down_name)): printl("Dir already exsit:%s, quit download." % os.path.join(save_dir, down_name)) return None os.chdir(save_dir) if not ftp_conn(host, port, acc, pwd, 30000): return None printl("Begin to download, calculating files number...") DIRECT_DOWNLOAD_TOTAL = 0 DIRECT_DOWNLOAD_COUNT = 0 start_get_file_number(host, port, acc, pwd, download_dir) #CONN.set_debuglevel(1) #get_file_number(download_dir) if DIRECT_DOWNLOAD_TOTAL == 0: pass #return None #printl("Total %d files" % (DIRECT_DOWNLOAD_TOTAL)) PROGRESS_COST_SEC = 0 PROGRESS_TOTAL_BYTES = 0 PROGRESS_BAR.pack(side=LEFT) PROGRESS_LBL.pack(side=LEFT) download_success = ftp_download_dir(download_dir) CONN.quit() PROGRESS_BAR.pack_forget() PROGRESS_LBL.pack_forget() if not download_success: return None return os.path.join(save_dir, os.path.basename(download_dir))
def find_mail(self, mail_subject_keyword): ''' check the inbox and return a mail list that contains new mails subjects which received time is > the specific time ''' print("Start to check new mail received after time: %s" % str(self.utc_time_point)) #mail_list = [] my_inbox = self.my_account.inbox my_inbox.refresh() mail_number = my_inbox.total_count latest_mail = my_inbox.all().order_by('-datetime_received')[:1].next() latest_mail_time = latest_mail.datetime_received + timedelta(hours=8) if self.utc_time_point >= latest_mail_time: print("No new mail..") yield None re_rule = re.compile(mail_subject_keyword, re.I) latest_ten = my_inbox.all().order_by('-datetime_received')[:10] for item in latest_ten: d_rec = item.datetime_received + timedelta(hours=8) if self.utc_time_point >= d_rec: break subject = item.subject if subject == None: subject = '' if re_rule.search(subject): printl("\nDetect a new mail, Date:[%s], subject:[%s]" % (str(d_rec), subject)) yield item #update time to the checked latest mail's self.utc_time_point = latest_mail_time
def periodical_check(self): global AUTOANA_ENABLE if self.v_chk.get() == 1: printl("Enable periodical monitor") self.entry_mail_k.config(state='normal') self.button_monitor.config(state='normal') self.label_mail_k.config(state='normal') self.spin_interval.config(state='normal') self.label_interval.config(state='normal') AUTOANA_ENABLE = True self.label_exserver.config(state='normal') self.entry_exserver.config(state='normal') self.label_mail_add.config(state='normal') self.entry_mail_add.config(state='normal') self.label_csl.config(state='normal') self.entry_csl.config(state='normal') self.label_cip.config(state='normal') self.entry_cip.config(state='normal') else: #printl("Periodical monitor disabled") self.entry_mail_k.config(state='disable') self.button_monitor.config(state='disable') self.label_mail_k.config(state='disable') self.spin_interval.config(state='disable') self.label_interval.config(state='disable') AUTOANA_ENABLE = False self.label_exserver.config(state='disable') self.entry_exserver.config(state='disable') self.label_mail_add.config(state='disable') self.entry_mail_add.config(state='disable') self.label_csl.config(state='disable') self.entry_csl.config(state='disable') self.label_cip.config(state='disable') self.entry_cip.config(state='disable')
def direct_download(self): global HOST global PORT global ACC global PWD global SAVE_DIR global DOWNLOAD_DIR global DIRECT_DOWNLOAD_STOP global L_RESERVED_FTP global PROGRESS_TOTAL_BYTES global PROGRESS_COST_SEC printl("Direct_download starts") #extract ftp info if self.v_host.get(): ftp_info = self.extract_ftp_info(self.v_host.get(), from_mail=False) #FTP_INFO = collections.namedtuple("FTP_INFO", "HOST PORT ACC PWD DIRNAME") if ftp_info: self.v_host.set(ftp_info.HOST) self.v_port.set(ftp_info.PORT) self.v_user.set(ftp_info.ACC) self.v_pwd.set(ftp_info.PWD) self.v_ddirname.set(ftp_info.DIRNAME) if self.v_host.get(): HOST = self.v_host.get() if self.v_port.get(): PORT = self.v_port.get() if self.v_user.get(): ACC = self.v_user.get() if self.v_pwd.get(): PWD = self.v_pwd.get() if self.v_ddirname.get(): #get rid of the last '/' if had self.v_ddirname.set(self.v_ddirname.get().strip()) if self.v_ddirname.get()[-1] == '/': self.v_ddirname.set(self.v_ddirname.get()[:-1]) DOWNLOAD_DIR = self.v_ddirname.get() #set host to display as ip:port format self.v_host.set(HOST + ':' + PORT) file_saved = '' start_t = time.clock() try: file_saved = my_download(HOST, PORT, ACC, PWD, SAVE_DIR, DOWNLOAD_DIR) except Exception as e: print("my_download error", e) file_saved = None end_t = time.clock() interval_t = time.clock() - start_t print("DEBUG file_saved=", file_saved) if not file_saved: print("DEBUG Download failed, time used: %.1f seconds" % interval_t) #crash #showerror(title='Ftp Connect Error', message="Cannot accesst to %s" % HOST) else: self.l_savein.bell() printl("Download completed in: %s, total time: %.1f seconds" % (file_saved, interval_t)) print("DEBUG total bytes:{} total sec:{}".format( PROGRESS_TOTAL_BYTES, PROGRESS_COST_SEC)) #update MM's knowledge about ftp info s = r"ftp://" + ACC + ":" + PWD + "@" + HOST + ":" + PORT if s not in L_RESERVED_FTP: L_RESERVED_FTP.append(s) printl("New ftp address learned:", s) if AUTOANA_ENABLE: #send to queue for other processing, e.g.,auto search in listdir.py print("DEBUG my_ftp.py send %s to auto analyse" % file_saved) FTP_FILE_QUE.put(file_saved) DIRECT_DOWNLOAD_STOP = True self.button_direct.config(text="Direct download", bg='white', relief='raised', state='normal') return file_saved
def ask_quit(self, ftp_top): global HOST global PORT global ACC global PWD global DOWNLOAD_DIR global MAIL_KEYWORD global MONITOR_INTERVAL global EXSERVER global MAIL_ADD global AD4_ACC global AD4_PWD global FTQ res = askyesnocancel("Mail Monitor", "(Recommended)Save current configurations?", parent=ftp_top) if res: #save HOST = self.v_host.get() PORT = self.v_port.get() ACC = self.v_user.get() PWD = self.v_pwd.get() DOWNLOAD_DIR = self.v_ddirname.get() MAIL_KEYWORD = self.v_mail_k.get() MONITOR_INTERVAL = self.v_interval.get() EXSERVER = self.v_exserver.get() MAIL_ADD = self.v_mail_k_add.get() AD4_ACC = self.v_csl.get() #do not save password #self.v_cip.set('') AD4_PWD = self.v_cip.get() save_bak() elif res == False: pass else: #'cancel' nothing to do return printl("'Mail Monitor exited'\n") self.running = False FTP_FILE_QUE.put('ftp quit') #terminate progress threads #crash don't know why #terminate_threads(PROGRESS_THREADS) if __name__ == '__main__': #quit() all windows and parent window to be closed ftp_top.quit() else: #important remember to do these stuffs before destruction! #Even though the window is destoyed, thease globals still in ram FTQ.queue.clear() #derstroy only this window ftp_top.destroy() l_thr = [] l_thr.extend(MONITOR_THREADS) l_thr.extend(DIRECT_DOWNLOAD_THREADS) l_thr.extend(PROGRESS_THREADS) terminate_threads(l_thr) for l in l_thr: if l.is_alive(): l._Thread__stop() '''
def monitor_ftpinfo(self): ''' get ftp info extracted from mail body and the mail comes from read exchange server by read_exchange.py ''' global MONITOR_STOP global DIRECT_DOWNLOAD_STOP global L_RESERVED_FTP global FTP_INFO_HISTORY global MUTEXT interval_count = 0 find_folder = "inbox" try: acc = self.v_csl.get() pwd = self.v_cip.get() ser = self.v_exserver.get() mail = self.v_mail_k_add.get() my_ol = read_exchange.MY_OUTLOOK(acc, pwd, ser, mail) except Exception as e: printl("Error exchanger account: {} connecting failed: {}".format( mail, e)) else: printl("Start monitoring exchange account %s" % mail) del_html = re.compile(r'<[^>]+>', re.S) while 1: if MONITOR_STOP or (not self.running): break for mail_item in my_ol.find_mail(self.v_mail_k.get()): if not mail_item: break else: #sound a bell self.l_savein.bell() #Extract ftp info and then start to download plain_body = del_html.sub('', mail_item.body) #print("DEBUG plain_body:",plain_body) ftp_info = self.extract_ftp_info(plain_body, from_mail=True) if (ftp_info != None) and (ftp_info not in FTP_INFO_HISTORY): FTP_INFO_HISTORY.append(ftp_info) else: print("DBUEG already handled ftp_info") continue #only dirname case: try_ftp_list = [] if ftp_info != None and ftp_info.HOST == '' and ftp_info.ACC == '' and ftp_info.DIRNAME != '': #use reserved ftp info to combine this dirname to try: for ftp_addr in L_RESERVED_FTP: #here is it necessary to add the '\r' ending? s = ftp_addr + ftp_info.DIRNAME + '\r' if 'unicode' in str(type(s)): s = s.encode('utf-8') try_ftp_list.append(s) #is here necessary to do the mutex?yes self.ftp_queue.append(try_ftp_list[:]) #for the full ftp case elif ftp_info != None and ftp_info.HOST != '' and ftp_info.ACC != '': printl("Detected ftp_info:{}".format(ftp_info)) #FTP_INFO = collections.namedtuple("FTP_INFO", "HOST PORT ACC PWD DIRNAME") #must add '\\r' because extract_ftp_info use this as an end s = r"ftp://"+ftp_info.ACC+":"+ftp_info.PWD+"@"+ftp_info.HOST+":"+\ ftp_info.PORT+ftp_info.DIRNAME + '\r' if 'unicode' in str(type(s)): s = s.encode('utf-8') self.ftp_queue.append([s]) else: print("DEBUG error, should no such case") pass self.v_ftp_number.set(str(len(self.ftp_queue))) sub = mail_item.subject printl("Detect a ftp_info: %s from mail: %s" % (str(ftp_info), sub)) #end for time.sleep(int(self.v_interval.get())) interval_count += 1 print("Monitor ftp>>>>, count %d"\ % (interval_count)) #end while MONITOR_STOP = True self.button_monitor.config(text="Start monitor", bg='white', relief='raised', state='normal')
def download_ftpinfo(self): global DIRECT_DOWNLOAD_STOP print("DEBUG start download_ftpinfo circle") interval_count = 0 while 1: if MONITOR_STOP or (not self.running): printl("download_ftpinfo thread stopped") break if self.ftp_queue: #pop out the first ftp_info_list ftp_info_list = self.ftp_queue[0][:] #ftp_info_list len >1 only when the case from partial ftp info #that guessed from many reserved ftp address #so once one ftp_info was successfully download #quit the for circle print("DEBUG start to download ftp_info_list", ftp_info_list) file_saved = '' for ftp_info in ftp_info_list: self.v_host.set(ftp_info) if DIRECT_DOWNLOAD_STOP: DIRECT_DOWNLOAD_STOP = False self.button_direct.config\ (text="Downloading...",bg='orange',relief='sunken',state='normal') file_saved = self.direct_download() if file_saved: self.v_saved_number += 1 self.l_savein.config( text='%s' % (str(self.v_saved_number) + ' Saved in: '), bg='orange') #quit after downloading successuflly break else: printl("try another reserved ftp address...") continue else: printl( "Error, another direct download is under processing, you can't start to download this!" ) self.ftp_queue.insert(0, ftp_info_list) break #record all the download info into ftp_log.txt if not file_saved: file_saved = "Failed" printl("ftp_info: '%s' download result: '%s'" % (str(ftp_info), file_saved)) self.ftp_queue.pop(0) self.v_ftp_number.set(str(len(self.ftp_queue))) else: pass time.sleep(int(self.v_interval.get())) interval_count += 1 print(">>>>Download ftp, count %d" % (interval_count)) #while print("DEBUG download_ftpinfo stopped.")
def ftp_download_dir(dirname): global CONN global DIRECT_DOWNLOAD_COUNT global DIRECT_DOWNLOAD_TOTAL global DIRECT_DOWNLOAD_BYTES print("ftp_download_dir, dirname: '%s'" % dirname) try: CONN.cwd(dirname) except ftplib.error_perm: printl('ERROR: cannot cd to "%s"' % dirname) return False else: new_dir = os.path.basename(dirname) if not os.path.exists(new_dir): os.mkdir(new_dir) os.chdir(new_dir) filelines = [] filelines_bk = [] CONN.dir(filelines.append) filelines_bk = CONN.nlst() if len(filelines_bk) != len(filelines): print("Error, Number does not match!\n") i = 0 for file in filelines: #unmatched file_name file_name = '' if filelines_bk[i] in file: file_name = filelines_bk[i] else: #the filelines_bk is not in the same order with filelines #filelines like: #'drwxr-xr-x 2 ftpalcatel users 1024 Oct 26 14:46 SQ2DSL02.02C_20171026095345.0' #There is a bug if the file name contains blanks file_name = file.split()[-1] #<DIR> display in widows and drwxr- in linux #here maybe a bug if only a file with name 'dxxxx' if '<DIR>' in file or file.startswith('d'): ftp_download_dir(file_name) CONN.cwd('..') os.chdir('..') else: try: #how to speed up? DIRECT_DOWNLOAD_BYTES = 0 #change transfor mode from ASCII to Binary #then allowed to get file_size CONN.voidcmd('TYPE I') file_size = CONN.size(file_name) maxblocksize = 2097152 printl("Downloading[%d/%d]: %s"%\ (DIRECT_DOWNLOAD_COUNT+1, DIRECT_DOWNLOAD_TOTAL, file_name)) file_to_write = open(file_name, 'wb').write CONN.retrbinary('RETR %s' % file_name, \ #use lambda to pass multiple paremeters #open(filelines_bk[i], 'wb').write,maxblocksize) lambda block: ftp_write_handle(block,file_size,file_to_write), maxblocksize) #except ftplib.error_perm: except Exception as e: printl('ERROR: cannot download file "%s"' % file_name) print("error:", e) try: os.unlink(file_name) except Exception as e: #print("DEBUG os.unlink error:",e) continue else: print("DEBUG os.unlink successed") continue else: DIRECT_DOWNLOAD_COUNT += 1 i += 1 return True
global CONN printl('ftp_conn start, host:{0}, port:{1}, acc:{2}, pwd:{3}'\ .format(host,port,acc,pwd)) try: CONN = ftplib.FTP() #set a 3 seconds timeout if not timeout: CONN.connect(host, port, timeout) else: CONN.connect(host, port) except (socket.error, socket.gaierror), e: printl('ERROR: cannot reach host "%s", exited.' % host) return False printl("Successfully connected to host '%s'" % host) try: CONN.login(acc, pwd) except ftplib.error_perm: printl('ERROR: cannot login as "%s", exited.' % acc) CONN.quit() return False printl("Successfully logged in as '%s'" % acc) return True ############ftp_conn()#####################