def process_localfiles(pmailslst, pmutex, pservfileprocessedlst, pfrompath, ptopath): #1.move file to his dir #2.update maillst status by uid&flag fileslst = os.listdir(pfrompath) fileslst.sort() for i in range(0, len(fileslst), 1): if getidxkeywordinlst( pservfileprocessedlst, get2betw13(fileslst[i], public.GFLAG_UID, public.GSEPTOR_FILE)) > -1: #move to ptopath logaq('moving file to ' + ptopath) shutil.move(pfrompath + fileslst[i], ptopath + fileslst[i]) updateflagmsg_byuidflag( pmailslst, pmutex, get2betw13(fileslst[i], public.GFLAG_UID, public.GSEPTOR_FILE), public.GSTATUS_SQLDATA_EXPORTED, public.GSTATUS_SQLDATA_DOWNLOADED, '') else: #local file exist,serverfile no exist tmpidx = getidxbyuidflag( get2betw13(fileslst[i], public.GFLAG_UID, public.GSEPTOR_FILE), public.GSTATUS_SQLDATA_EXPORTED, pmailslst) if tmpidx > -1: #os.rename(pfrompath+fileslst[i],(pfrompath+fileslst[i]).replace(public.GSUFFIX_FILEMAIL,'.drr')) logaq('moving file to send') shutil.move(pfrompath + fileslst[i], public.gdir_atachmt_willsend) updateflagmsg_byuidflag( pmailslst, pmutex, get2betw13(fileslst[i], public.GFLAG_UID, public.GSEPTOR_FILE), public.GSTATUS_SQLDATA_EXPORTED, public.GSTATUS_ERROR, PG_MSG2)
def process_exceptmaillst(pmaillst, pmutex): #1.reply flag=E mail #2.reply flag=B&Spent>30Min mail #3.reply flag=G&Stayfor>2H idx = 0 while idx < len(pmaillst): if pmaillst[idx]['ftime'] and pmaillst[idx][ 'flag'] == public.GSTATUS_SQLDATA_BEFEXPORT and ( time.time() - pmaillst[idx]['ftime']) > public.gmmaxqtime + 60: updateflagmsg_byuidflag( pmaillst, pmutex, pmaillst[idx]['uid'], public.GSTATUS_SQLDATA_BEFEXPORT, public.GSTATUS_ERROR, MS_MSG2.replace('MAX_QTIME', str(public.gmmaxqtime / 60))) if pmaillst[idx]['ftime'] and pmaillst[idx][ 'flag'] == public.GSTATUS_SQLDATA_DOWNLOADED and ( time.time() - pmaillst[idx]['ftime']) > public.gmmaxqtime * 4: updateflagmsg_byuidflag( pmaillst, pmutex, pmaillst[idx]['uid'], public.GSTATUS_SQLDATA_DOWNLOADED, public.GSTATUS_ERROR, MS_MSG3.replace('MAX_QTIME', str(public.gmmaxqtime / 60))) if pmaillst[idx]['flag'] == public.GSTATUS_ERROR: logaq('sendmail uid:' + pmaillst[idx]['uid']) send_mail(public.gmusr, pmaillst[idx]['from'] + ',' + pmaillst[idx]['to'], pmaillst[idx]['cc'], 'Re:' + pmaillst[idx]['subject'], MS_MSGH + pmaillst[idx]['msg'] + MS_MSGT, '', '') logaq('deleting mailmeta by uid') deldict_fromlist(pmaillst, pmutex, 'uid', pmaillst[idx]['uid']) idx = idx - 1 idx = idx + 1
def put_main(pmailslst, pparaslst, pmutex): fileslst = os.listdir(public.gdir_atachmt_downloaded) fileslst.sort() curmuid = -1 predbdusr = '' curdbdusr = '' for i in range(0, len(fileslst), 1): print(fileslst[i]) path = os.path.join(public.gdir_atachmt_downloaded, fileslst[i]) #process file context # curdbdusr = getdbtnsdbusr(get2betw13(fileslst[i], 'UID', '_'), public.GSTATUS_ATACHMT_DOWNLOADED, pmailslst) print(curdbdusr) if not curdbdusr: os.rename(path, path.replace(public.GSUFFIX_FILEMAIL, '.urr')) logaq('moving file to senddir') shutil.move(path.replace(public.GSUFFIX_FILEMAIL, '.urr'), public.gdir_atachmt_willsend) updateflagmsg_byuidflag(pmailslst, pmutex, get2betw13(fileslst[i], 'UID', '_'), public.GSTATUS_ATACHMT_DOWNLOADED, public.GSTATUS_ERROR, PG_MSG1) if i > 0: updateflagmsg_byuidflag( pmailslst, pmutex, get2betw13(fileslst[i - 1], 'UID', '_'), public.GSTATUS_ATACHMT_DOWNLOADED, public.GSTATUS_ATACHMT_UPLOADED, '') elif predbdusr != curdbdusr: #getoinfo if predbdusr: transport.close() loip, loprt, lousr, lopd = getoinforbytnsusr(curdbdusr, pparaslst) print(loip + ',' + loprt + ',' + lousr + ',' + lopd) transport = paramiko.Transport((loip, int(loprt))) transport.connect(username=lousr, password=lopd) sftp = paramiko.SFTPClient.from_transport(transport) if os.path.isfile(path): sftp.put(path, '/home/test/' + fileslst[i]) shutil.move(path, public.gdir_atachmt_uploaded) #updatedict_inlist(pmailslst,pmutex,pg_dictname_uid,get2betw13(fileslst[i],'UID','_'),pg_dictname_flag,public.GSTATUS_ATACHMT_DOWNLOADED,public.GSTATUS_ATACHMT_UPLOADED) if (predbdusr and predbdusr != curdbdusr) or (i == len(fileslst) - 1): idxlast = i - 1 if i == len(fileslst) - 1: idxlast = len(fileslst) - 1 print(get2betw13(fileslst[idxlast], 'UID', '_')) updateflagmsg_byuidflag(pmailslst, pmutex, get2betw13(fileslst[idxlast], 'UID', '_'), public.GSTATUS_ATACHMT_DOWNLOADED, public.GSTATUS_ATACHMT_UPLOADED, '') if i == len(fileslst) - 1: transport.close() #if fileslst[i].find('DW')>-1:#fileuid not match mailslst uid #print('----\n'+public.gdir_atachmt_uploaded+fileslst[i]+'\n'+public.gdir_atachmt_uploaded+'err.err'+'----\n') #os.rename(public.gdir_atachmt_uploaded+fileslst[i],public.gdir_atachmt_uploaded+'err.err') predbdusr = curdbdusr
def getdel_servfiles(ptransport, pserloginhome, plocalpath, pfilelstremove): sftp = paramiko.SFTPClient.from_transport(ptransport) try: tmpfiles = sftp.listdir() logaq('sftp filelst:' + str(tmpfiles), 'i') tmpfiles.sort() for tfile in tmpfiles: if tfile.find('UID') > -1 and tfile.find('tar.gz') > 0: logaq('get&del:' + tfile, 'i') sftp.get(pserloginhome + tfile, plocalpath + tfile) pfilelstremove.append(tfile) sftp.remove(pserloginhome + tfile) finally: sftp.close()
def send_mail(pfrom, pto, pcc, psub, pcontt, pdiratta, plstatta): #mailheader global ms_serv message = MIMEMultipart() message['From'] = pfrom message['To'] = pto message['Cc'] = pcc message['Subject'] = psub #mailbody message.attach(MIMEText(pcontt, 'plain', 'utf-8')) #plain or html #attachment if pdiratta and plstatta: for filen in plstatta.split(','): logaq('uploading mail attachment:' + filen, 'i') atta = MIMEApplication(open(pdiratta + '\\' + filen, 'rb').read()) atta.add_header('Content-Disposition', 'attachment', filename=filen) message.attach(atta) try: lreceivers = pto + ',' + pcc logaq('receivers:' + lreceivers) ms_serv.sendmail(pfrom, lreceivers.split(','), message.as_string()) except Exception as e: logaq('mailss sendmail Error: %s' % e, 'e') ms_serv = smtplib.SMTP_SSL(public.gshst, public.gsprt) ms_serv.login(public.gmusr, getpdbyen(public.gmenpd)) logaq('smtp server relogin ok', 'i') ms_serv.sendmail(pfrom, lreceivers.split(','), message.as_string())
def process_mailheader(pmsg): global ma_from global ma_to global ma_cc global ma_sub logaq('--from||to||cc||subject--', 'i') ma_sub = clearctrlchar(decode_str(pmsg.get('subject'))) ma_from = clearctrlchar(email.utils.parseaddr(pmsg.get('from'))[1]) ma_to = clearctrlchar(pmsg.get('to')) ma_to = format_tocc(ma_to) cc = pmsg.get('cc') if cc: ma_cc = clearctrlchar(cc) ma_cc = format_tocc(ma_cc) logaq(ma_from + '||' + ma_to + '||' + ma_cc + '||' + ma_sub, 'i')
def processfile(pmailslst, pmutex, ppath, pfilename): #process file content for creating view #Format: #old file:select ...... from ......where ......;select ...... from ...... where ......; #new file:create view filename_seq1V as select ...... from ......where ......;create view filename_seq2V as select ...... from ...... where ......; global s2_charset ltmpuid = get2betw13(pfilename, public.GPREFIX_FILEUID, public.GSEPTOR_FILE) tmpfd = open(ppath, 'rb') try: tmpdata = tmpfd.read() finally: tmpfd.close() s2_charset = chardet.detect(tmpdata)['encoding'] if s2_charset[ 0: 3] == 'UTF': #AMERICAN_AMERICA.ZHS16GBK AL32UTF8 ZHS16CGB231280 os.environ["NLS_LANG"] = public.GNLS_LANG_UTF8 else: os.environ["NLS_LANG"] = public.GNLS_LANG_GBK if check_fileformat(tmpdata.decode(s2_charset)): lviewname = public.GFLAG_UID + get2betw13( pfilename, public.GPREFIX_FILEUID, public.GSUFFIX_FILEMAIL) tmpfiletext = replace_oldkeyword( tmpdata.decode(s2_charset), S2_SELECT, S2_CREATEORREP + lviewname + S2_SUFFIX, S2_SEGSEPTOR, 1, lviewname + '_') tmperr, tmpfiletext = remove_riskfactors(tmpfiletext) if tmperr: updateflagmsg_byuidflag(pmailslst, pmutex, ltmpuid, public.GSTATUS_ATACHMT_DOWNLOADED, public.GSTATUS_ERROR, S2_RFACTORHINT + tmperr + S2_TAIL) logaq(pfilename + ':' + S2_RFACTORHINT + tmperr, 'e') if os.path.exists(public.gdir_history + pfilename): os.remove(public.gdir_history + pfilename) logaq('moving file to his') shutil.move(ppath, public.gdir_history) return False tmpdata = tmpfiletext.encode(s2_charset) tmpfd = open(ppath, 'wb') try: tmpfd.write(tmpdata) finally: tmpfd.close() return True else: updateflagmsg_byuidflag(pmailslst, pmutex, ltmpuid, public.GSTATUS_ATACHMT_DOWNLOADED, public.GSTATUS_ERROR, S2_MSG4) logaq(pfilename + ':' + S2_MSG4, 'e') if os.path.exists(public.gdir_history + pfilename): os.remove(public.gdir_history + pfilename) logaq('moving file to his') shutil.move(ppath, public.gdir_history) return False
def callprocedure(pmailslst, pparaslst, pmutex, ppath, pfilename, pproclst, pmutexp, pdbconlst): #add pdbconlst #!!!!!!!!!!this procedure maybe spend a long time!!!!!!!!!!! #BUG FINDED 20181123 ltmpuid = get2betw13(pfilename, public.GPREFIX_FILEUID, public.GSEPTOR_FILE) updateflagmsg_byuidflag(pmailslst, pmutex, ltmpuid, public.GSTATUS_ATACHMT_DOWNLOADED, public.GSTATUS_SQLDATA_BEFEXPORT, '', time.time()) ltmpviewlike = public.GFLAG_UID + get2betw13( pfilename, public.GPREFIX_FILEUID, public.GSUFFIX_FILEMAIL) + '%' ltmpowner = getdbusrbyuid(ltmpuid, pmailslst).strip() ltmpdbtns = getdbtnsbyuid(ltmpuid, pmailslst) if len(ltmpowner) == 0: ltmpowner = getdbusrbydbtns(ltmpdbtns, pparaslst) logaq('dbtns:' + ltmpdbtns) logaq('dbusr:'******'t exit) #ltmpprocname=S2_PROC_PRODUCTDATA.replace('POWNER',ltmpowner.upper()) #ltmpprocname=ltmpprocname.replace('PVIEWLIKE',ltmpviewlike.upper()) #lrst=execprocedure(getsqlpluscon(ltmpdbtns,ltmpowner,pparaslst),ltmpprocname,pproclst,pmutexp) lsqlerr = '' for lerr in S2_ERROR: if lrst.lower().find(lerr.lower()) > -1: lsqlerr = lrst if lsqlerr: #procedure exec error lsqlerr = clearblankchar(lsqlerr) updateflagmsg_byuidflag(pmailslst, pmutex, ltmpuid, public.GSTATUS_SQLDATA_BEFEXPORT, public.GSTATUS_ERROR, lsqlerr) logaq(pfilename + ':' + lsqlerr, 'e') if os.path.exists(public.gdir_history + pfilename): os.remove(public.gdir_history + pfilename) logaq('moving file to his') shutil.move(ppath, public.gdir_history) else: #cannot updateflag to exported until all UID attachments process over. # updateflagmsg_byuidflag(pmailslst,pmutex,pmailslst[pidxuid]['uid'],public.GSTATUS_SQLDATA_BEFEXPORT,public.GSTATUS_SQLDATA_EXPORTED,'') if os.path.exists(public.gdir_history + pfilename): os.remove(public.gdir_history + pfilename) logaq('moving file to his') shutil.move(ppath, public.gdir_history)
def relogin_mail(ppretime, pcon): try: if time.time() - ppretime > public.gmcycle: try: pcon.close() pcon.logout() logaq('mailsa conclosed(>cycle)', 'w') except Exception as e: logaq('mailsa conclose: %s' % e, 'e') tmpcon = imaplib.IMAP4_SSL(port=public.giprt, host=public.gihst) logaq('connect ok', 'i') tmpcon.login(public.gmusr, getpdbyen(public.gmenpd)) logaq('login ok', 'i') return tmpcon else: return pcon except Exception as e: logaq('mailsa ini cycle: %s' % e, 'e')
def targzipsplit_servfiles(ptransport, puidstringlst): #ssh = paramiko.SSHClient() #ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) #ssh.connect(hostname='ip', port=22, username='******', password='******') #tmploginhome='' ssh = paramiko.SSHClient() ssh._transport = ptransport try: #stdin, stdout, stderr = ssh.exec_command('pwd') #res,err = stdout.read(),stderr.read() #result = res if res else err #tmploginhome=result.decode() #logaq(tmploginhome) ''' stdin, stdout, stderr = ssh.exec_command(command=PG_CMD_TARGZ01)#,environment=PG_ENV_BASH res,err = stdout.read(),stderr.read() result = res if res else err logaq('echo tar&gzip01>*.sh:'+result.decode()) stdin, stdout, stderr = ssh.exec_command(command=PG_CMD_TARGZ02.replace('UIDLST',puidstringlst)) res,err = stdout.read(),stderr.read() result = res if res else err logaq('echo tar&gzip02>>*.sh:'+result.decode()) stdin, stdout, stderr = ssh.exec_command(command=PG_CMD_TARGZ03) res,err = stdout.read(),stderr.read() result = res if res else err logaq('echo tar&gzip03>>*.sh:'+result.decode()) #remarked for solaris split -d {} bug stdin, stdout, stderr = ssh.exec_command(command=PG_CMD_SPLIT50M) res,err = stdout.read(),stderr.read() result = res if res else err logaq('echo split nM file>>*.sh:'+result.decode()) stdin, stdout, stderr = ssh.exec_command(command=PG_CMD_RMGZBEFSPLIT)#window copy /b /path/UID* /path/UID.zip; res,err = stdout.read(),stderr.read() result = res if res else err logaq('echo rm -rf *tar.gz(exist *.gz.00)>>*.sh:'+result.decode()) ''' stdin, stdout, stderr = ssh.exec_command(PG_CMD_SH) res, err = stdout.read(), stderr.read() result = res if res else err logaq('sh *.sh:' + result.decode()) #return tmploginhome finally: #ssh.close() #Error SSH session not active pass
def process_mailslst(pmailslst, pmutex, pservfileprocessedlst, pdbtns): idxtmp = 0 while idxtmp < len(pmailslst): luid = pmailslst[idxtmp]['uid'] if getidxkeywordinlst(pservfileprocessedlst, pmailslst[idxtmp]['uid']) > -1: updateflagmsg_byuidflag(pmailslst, pmutex, pmailslst[idxtmp]['uid'], public.GSTATUS_SQLDATA_EXPORTED, public.GSTATUS_SQLDATA_DOWNLOADED, '') elif pmailslst[idxtmp]['dbtns'] == pdbtns and pmailslst[idxtmp][ 'flag'] == public.GSTATUS_SQLDATA_EXPORTED: updateflagmsg_byuidflag(pmailslst, pmutex, pmailslst[idxtmp]['uid'], public.GSTATUS_SQLDATA_EXPORTED, public.GSTATUS_ERROR, PG_MSG3) if luid != pmailslst[idxtmp]['uid']: logaq( '!!!!!!multithreads data conflict(maillst idx which being used is deleted)', 'e') idxtmp = idxtmp + 1
def relogin_mail(ppretime): global ms_serv try: if time.time() - ppretime > public.gmcycle: try: ms_serv.quit() logaq('mailss servquit(>cycle)', 'w') except Exception as e: logaq('mailss conquit: %s' % e, 'e') ms_serv = smtplib.SMTP_SSL(public.gshst, public.gsprt) ms_serv.login(public.gmusr, getpdbyen(public.gmenpd)) logaq('smtp server login ok', 'i') except Exception as e: logaq('mailss ini cycle: %s' % e, 'e')
def create_s2fthread(pmaillst, pparalst, pmutex, pthreadlst, pproclst, pmutexp, pdbconlst): #this is a bottleneck,so must create 3 thread at least #script execute must be stop in GMAXSECS_RUN seconds #del pthreadlst[:]#copy:tmplst=lst[:],thread can not use deepcopy while not breaktime(): try: check_processthread(pproclst, pthreadlst, pmutexp, pdbconlst) if len(pthreadlst) < public.gmaxcnt_bnthread: setdirtns = gettnsset_indir(public.gdir_atachmt_downloaded, public.GPREFIX_FILEUID) for st in setdirtns: if not exist_tnsthread(pthreadlst, st): logaq( 'create s2fthread:' + public.GPREFIX_BNTHREAD + st, 'i') t_bn = threading.Thread(target=s2file_main, args=(pmaillst, pparalst, pmutex, st, pproclst, pmutexp, pdbconlst)) t_bn.setName(public.GPREFIX_BNTHREAD + st) t_bn.setDaemon(False) t_bn.start() pthreadlst.append({ 'thread': t_bn, 'ctime': time.time() }) if len(pthreadlst) >= public.gmaxcnt_bnthread: break logaq('create_s2fthread sleep ' + str(public.gsleep) + 's,refreshini...') time.sleep(public.gsleep) except Exception as e: time.sleep(3) logaq('start while Error: %s' % e, 'e') finally: pass #refresh_ini()#call external program,maybe will produce error which casing sys exit if breaktime(): check_processthread(pproclst, pthreadlst, pmutexp, pdbconlst, True)
def checkfilestatus(pmailslst, ppath, pfilename): rst = False tmpidxuid = getidxbyuidflag( get2betw13(pfilename, public.GPREFIX_FILEUID, public.GSEPTOR_FILE), public.GSTATUS_SQLDATA_BEFEXPORT, pmailslst) if tmpidxuid > -1: logaq(pfilename + ':' + S2_MSG2, 'w') #shutil.move(ppath,public.gdir_history) else: tmpidxuid = getidxbyuidflag( get2betw13(pfilename, public.GPREFIX_FILEUID, public.GSEPTOR_FILE), public.GSTATUS_ATACHMT_DOWNLOADED, pmailslst) if tmpidxuid < 0: #write error file to his,move this file to his logaq(pfilename + ':' + S2_MSG1, 'e') #writefile(os.path.join(public.gdir_history,pfilename),S2_MSG1,'.err') if os.path.exists(public.gdir_history + pfilename): os.remove(public.gdir_history + pfilename) logaq('moving file to his') shutil.move(ppath, public.gdir_history) else: rst = True return rst
def mails_main(pmaillst, pmutex, pdirattachment): if breaktime(): exit() try: global ms_serv global ms_msg1 lpretime = time.time() ms_serv = smtplib.SMTP_SSL(public.gshst, public.gsprt) ms_serv.login(public.gmusr, getpdbyen(public.gmenpd)) logaq('smtp server login ok', 'i') while not breaktime(): while not getfilepgok(public.gmutex_filepg) and not breaktime(): logaq('waiting for filepg ' + str(public.gsleep) + 's...', 'i') time.sleep(public.gsleep) logaq('process exception mail') process_exceptmaillst(pmaillst, pmutex) continue pretmpmuid = '' tmpmuid = '' existsplitf = False relogin_mail(lpretime) lpretime = time.time() try: lexistnewfile = False fileslst = os.listdir(pdirattachment) fileslst.sort() for i in range(0, len(fileslst), 1): tmplstfile = fileslst[i] logaq('file:' + tmplstfile, 'i') lpath = os.path.join(pdirattachment, tmplstfile) if time.time() - os.path.getmtime(lpath) <= 1: logaq('mtime<=1:' + lpath) lexistnewfile = True continue tmpmuid = get2betw13(tmplstfile, public.GFLAG_UID, public.GSUFFIX_TARGZFILEMAIL) #--split file process if tmpmuid == pretmpmuid: existsplitf = True if (existsplitf and (i<len(fileslst)-1) and get2betw13(fileslst[i+1],public.GFLAG_UID,'_')!=tmpmuid) \ or (existsplitf and i==len(fileslst)-1): tmpfilelike = get2betw13( tmplstfile, '', public.GSUFFIX_TARGZFILEMAIL ) + public.GSUFFIX_TARGZFILEMAIL writefile( pdirattachment + 'UID' + pretmpmuid, ms_cmd_merge.replace('FILENAME', tmpfilelike), '.cmd') tmplstfile = tmplstfile + ',UID' + pretmpmuid + '.cmd' existsplitf = False #--end split file process idxmail = getidxbyuidflag( tmpmuid, public.GSTATUS_SQLDATA_DOWNLOADED, pmaillst) ltmpuid = pmaillst[idxmail]['uid'] #get mailheader info if idxmail > -1: logaq('sendmail uid:' + tmpmuid) ms_msg1 = ms_msg1.replace('STOPTIME', public.gstop_time) ms_msg1 = ms_msg1.replace('MAX_QTIME', str(public.gmmaxqtime / 60)) send_mail( public.gmusr, pmaillst[idxmail]['from'] + ',' + pmaillst[idxmail]['to'], pmaillst[idxmail]['cc'], 'Re:' + pmaillst[idxmail]['subject'], ms_msg1, public.gdir_atachmt_willsend, tmplstfile) if pmaillst[idxmail]['uid'] != ltmpuid: logaq('!!!!!!multithreads data conflict!!!!!!', 'e') if os.path.exists(public.gdir_history + fileslst[i]): os.remove(public.gdir_history + fileslst[i]) logaq('moving file to his uid:' + tmpmuid) shutil.move(lpath, public.gdir_history) #--delete pmaillst if pretmpmuid and tmpmuid != pretmpmuid: logaq('deleting mailmeta by preuid(cur<>pre)') deldict_fromlist(pmaillst, pmutex, 'uid', pretmpmuid) if i == (len(fileslst) - 1): logaq('deleting mailmeta by uid(lastone)') deldict_fromlist(pmaillst, pmutex, 'uid', tmpmuid) #--end delete pretmpmuid = tmpmuid process_exceptmaillst(pmaillst, pmutex) if not lexistnewfile: setfilepgok(public.gmutex_filepg, False) else: time.sleep(3) logaq(pmaillst) logaq('while end', 'i') except Exception as e: logaq('mailss Error: %s' % e, 'e') time.sleep(3) finally: logaq('while finally', 'i') saveobject(pmaillst, 'maills4') #refresh_ini() finally: try: ms_serv.quit() except Exception as e: logaq('smtp exit exception: %s' % e, 'i')
lpypara = sys.argv[1] if helps(lpypara): exit() try: #1.init; get ini config parameter,init dirs,log,set loglevel by arg passed refresh_ini() init_dirs() init_log(lpypara) #2.refresh parameters of db refresh_paras(gparaslst) #3.prevent script runing again if isrunning(): exit() #4.create thread:mails analyzing&downloading logaq('--starting--', 'i') logaq('create thread Thread_mailsa', 'i') t_mailsa = threading.Thread(target=maila_main, args=(gmailslst, gparaslst, gmutex)) t_mailsa.setName('Thread_mailsa1') t_mailsa.setDaemon(False) t_mailsa.start() #5.create thread:exportfiles tar&zip&put/get logaq('create thread Thread_filepg') t_filepg = threading.Thread(target=get_main, args=(gmailslst, gparaslst, gmutex)) t_filepg.setName('Thread_filepg1') t_filepg.setDaemon(False) t_filepg.start()
def exeprocedure_bycxora(pconstr, pproname, pproparaowner, pproparaviewlike, pdbconlst, pmutex): #instead of execprocedure; solving BUG:mutithread call sqlplus hang rst = 'begin exeprocedure_bycxora' logaq('ConToDB constr:' + get2betw13(pconstr, '', '/') + '/******@' + get2betw13(pconstr, '@', '')) logaq('callproc:' + pproname + '(' + pproparaowner + ',' + pproparaviewlike + ')') try: odb = cx_Oracle.Connection(pconstr) dbcur = odb.cursor() except Exception as e: logaq('DB connect Error: %s' % e, 'e') rst = "Error:DB connect Exception" return rst try: add_dbcon(pdbconlst, odb, dbcur, threading.currentThread().getName(), pmutex) logaq('dblst size(AaddedBcall):' + str(len(pdbconlst))) dbcur.callproc(pproname, [pproparaowner, pproparaviewlike]) logaq('remove dbfromlst') remove_dbcon(pdbconlst, threading.currentThread().getName(), pmutex) logaq('dblst size(After call&removed):' + str(len(pdbconlst))) except Exception as e: logaq('DB Error: %s' % e, 'e') rst = S2_MSG5 logaq('remove dbfromlst') remove_dbcon(pdbconlst, threading.currentThread().getName(), pmutex) logaq('dblst size(After call&removed):' + str(len(pdbconlst))) finally: try: dbcur.close() except Exception as e: logaq('dbcur close Error: %s' % e, 'e') try: odb.close() except Exception as e: logaq('dbcon close Error: %s' % e, 'e') return rst
def execsfile(pconstr, pfile): global s2_charset logaq('NLS_LANG=' + os.environ["NLS_LANG"]) logaq(S2_SQLAPP + ' constr:' + get2betw13(pconstr, '', '/') + '/******@' + get2betw13(pconstr, '@', '') + ' file:' + pfile) #/xlzhu@ procf = Popen([S2_SQLAPP, '-S', pconstr], stdout=PIPE, stdin=PIPE, stderr=PIPE) logaq('opensqlplus id:' + str(id(procf))) logaq('begin create views...') #procf.stdin.write(('@'+pfile).encode())#exist bug try: (outf, errf) = procf.communicate(input=('@' + pfile).encode(s2_charset), timeout=60) logaq('create views end') except TimeoutExpired: logaq('sqlplus timeout', 'w') procf.terminate() (outf, errf) = procf.communicate() except Exception as e: logaq('sqlplus Exception: %s' % e, 'w') (outf, errf) = procf.communicate() if procf.returncode != 0: returnmsgf = errf.decode() logaq( 'returncode:' + str(procf.returncode) + 'err:' + returnmsgf + 'out:' + outf.decode(), 'e') if len(returnmsgf) == 0: returnmsgf = outf.decode() if len(returnmsgf) == 0: returnmsgf = S2_MSG5 #sys.exit(procf.returncode) else: returnmsgf = outf.decode() return returnmsgf
def createviews(pmailslst, pparaslst, pmutex, ppath, pfilename): #login to sqlplus exec sqlfile rst = 'OK' lsqlerr = '' ltmpuid = get2betw13(pfilename, public.GPREFIX_FILEUID, public.GSEPTOR_FILE) lcon = getsqlpluscon(getdbtnsbyuid(ltmpuid, pmailslst), getdbusrbyuid(ltmpuid, pmailslst), pparaslst) if lcon: lrst = execsfile(lcon, ppath) try: logaq('execsfile return') except Exception as e: logaq('log Error: %s' % e, 'e') else: logaq(pfilename + ':' + S2_MSG3, 'e') if os.path.exists(public.gdir_history + pfilename): os.remove(public.gdir_history + pfilename) logaq('moving file to his') shutil.move(ppath, public.gdir_history) rst = '' return rst #logaq('judging returnstr error') for lerr in S2_ERROR: if lrst.lower().find(lerr.lower()) > -1: lsqlerr = lrst #logaq('judging end 20s') time.sleep(20) if lsqlerr: #sqlfile exec error lsqlerr = clearblankchar(lsqlerr) updateflagmsg_byuidflag(pmailslst, pmutex, ltmpuid, public.GSTATUS_ATACHMT_DOWNLOADED, public.GSTATUS_ERROR, lsqlerr) logaq(pfilename + ':' + lsqlerr, 'e') if os.path.exists(public.gdir_history + pfilename): os.remove(public.gdir_history + pfilename) logaq('moving file to his') shutil.move(ppath, public.gdir_history) rst = '' return rst
def get_main(pmailslst, pparaslst, pmutex): ''' #1.get all servers(tnslist) where login to download UID*.csv files #2.ssh to server for tar&biz2&split all UID*.csv files #3.sftp to server for downloading&removing all UID*.gz or UID*.gz.00(00 stand for two bit number) #4.move local file TNS_UID0000000000.sql to his dir #5.update pmailst status to sqldata_downloaded ''' tmpfilelst = [] transport = None while not breaktime(): while not gets2fileok(public.gmutex_s2file) and not breaktime(): logaq('waiting for s2file ' + str(public.gsleep) + 's...', 'i') time.sleep(public.gsleep) continue strlstuid10 = ',' try: for serv in gettnsset_inmailslst(pmailslst, public.GSTATUS_SQLDATA_EXPORTED): tmpip, tmpprt, tmpusr, tmppd = getoinforbytnsusr( serv + getdbusrbydbtns(serv, pparaslst), pparaslst) transport = paramiko.Transport((tmpip, int(tmpprt))) transport.connect(username=tmpusr, password=tmppd) logaq('server connect ok', 'i') while strlstuid10: strlstuid10 = getuidstrlst_inmailslst( pmailslst, serv, public.GSTATUS_SQLDATA_EXPORTED, strlstuid10) if strlstuid10: logaq('process uid:' + strlstuid10, 'i') replaceline_infile(PG_CMD_LPATH, PG_CMD_TEMPLATE, PG_CMD_EXESH, PG_CMD_KEYWORD, strlstuid10) put_shfile(transport, PG_CMD_LPATH + PG_CMD_EXESH, PG_CMD_EXESH) targzipsplit_servfiles(transport, strlstuid10) del tmpfilelst[:] getdel_servfiles(transport, './', public.gdir_atachmt_willsend, tmpfilelst) #process_localfiles(pmailslst,pmutex,tmpfilelst,public.gdir_atachmt_downloaded,public.gdir_history) process_mailslst(pmailslst, pmutex, tmpfilelst, serv) logaq('sleep 3s for nextloop...', 'i') time.sleep(3) logaq('filepg.get_main end', 'i') if not existflag_inmailslst(pmailslst, public.GSTATUS_SQLDATA_EXPORTED): sets2fileok(public.gmutex_s2file, False) setfilepgok(public.gmutex_filepg, True) logaq(pmailslst) except Exception as e: logaq('filespg Error: %s' % e, 'e') time.sleep(3) finally: if transport: try: transport.close() except Exception as e: logaq('transport close exception: %s' % e, 'i') logaq('Dowload datafile of SQLExported over', 'i') saveobject(pmailslst, 'maills3')
def execprocedure(pconstr, pprocname, pproclst, pmutexp): ''' if procp.stdin: procp.stdin.close() if procp.stdout: procp.stdout.close() if procp.stderr: procp.stderr.close() ''' logaq(S2_SQLAPP + ' constr:' + pconstr + ' procedure:' + pprocname) procp = Popen([S2_SQLAPP, '-S', pconstr], stdout=PIPE, stdin=PIPE, stderr=PIPE) logaq('opensqlplus id:' + str(id(procp))) add_process(pproclst, procp, threading.currentThread().getName(), pmutexp) logaq('sqlpluslst size(AaddedBexec):' + str(len(pproclst))) #procp.stdin.write(pprocname.encode())#exist bug try: (outp, errp) = procp.communicate(input=pprocname.encode(), timeout=public.gmmaxqtime) except TimeoutExpired: logaq('sqlplus timeout', 'w') procp.terminate() (outp, errp) = procp.communicate() except Exception as e: logaq('sqlplus Exception: %s' % e, 'w') (outp, errp) = procp.communicate() if procp.returncode != 0: returnmsgp = errp.decode() logaq( 'returncode:' + str(procp.returncode) + 'err:' + returnmsgp + 'out:' + outp.decode(), 'e') if len(returnmsgp) == 0: returnmsgp = outp.decode() if len(returnmsgp) == 0: returnmsgp = S2_MSG5 else: returnmsgp = outp.decode() logaq('remove sqlplusfromlst') remove_process(pproclst, threading.currentThread().getName(), pmutexp) logaq('sqlpluslst size(After exec&removed):' + str(len(pproclst))) return returnmsgp
def maila_main(pmailslst, pparaslst, pmutex): global ma_uid global ma_from global ma_to global ma_cc global ma_sub global ma_flag global ma_ftime global ma_dbusr global ma_dbtns global ma_msg if breaktime(): exit() try: lpretime = time.time() ma_uid = '' conn = imaplib.IMAP4_SSL(port=public.giprt, host=public.gihst) logaq('connect ok', 'i') conn.login(public.gmusr, getpdbyen(public.gmenpd)) logaq('login ok', 'i') while not breaktime(): try: global ma_chset global ma_existatta #del pmailslst[:] conn = relogin_mail(lpretime, conn) conn.select(public.giwkdir) logaq('working in folder:%s' % public.giwkdir, 'i') #type, data = conn.search(None, 'ALL', 'SUBJECT "*review1810221613"')#can not search mail like 'keyword' type, data = conn.search(None, 'ALL') mailbidx = data[0].split() #[b'1',b'2',b'3'] mailcnt = len(mailbidx) logaq('mail cnt:' + str(mailcnt), 'i') for num in range( mailcnt - 1, -1, -1 ): #process from new mail(if delete mail,Don't begin from old,else Error:Nontype for decode) try: #pmailslst.append(newmailmeta()) #get mailuid clearmailmeta() type, data = conn.fetch(mailbidx[num], 'UID') ma_chset = chardet.detect(data[0])['encoding'] logaq('uidencode:' + ma_chset) ma_uid = getmailuid( str( email.message_from_string( data[0].decode(ma_chset)))) logaq('current mail uid:' + ma_uid, 'i') #get maildetailinfo type, data = conn.fetch(mailbidx[num], 'RFC822') logaq('--MailBinaryInfo:') logaq(data) logaq('--EndBinaryInfo') try: ma_chset = chardet.detect(data[0][1])['encoding'] logaq('mailencode:' + ma_chset) except Exception as e: logaq( str(ma_chset) + ' decode error(will default utf-8): %s' % e, 'e') ma_chset = 'utf-8' msg = email.message_from_string( data[0][1].decode(ma_chset)) #.decode('gbk') process_mailheader(msg) if allow_from(ma_from, public.gmfromallow): process_mailbody(pparaslst, msg, num) if ma_msg: ma_flag = public.GSTATUS_ERROR else: if not ma_existatta: ma_msg = MS_MSG3 ma_flag = public.GSTATUS_ERROR else: ma_flag = public.GSTATUS_ATACHMT_DOWNLOADED else: logaq( 'mail will be deleted directly(sender not in allow list),uid:' + ma_uid, 'w') ma_uid = '' except Exception as e: logaq('mailsa Forloop Error: %s' % e, 'e') ma_flag = public.GSTATUS_ERROR ma_msg = str(e) time.sleep(3) addmaillist(pmailslst, pmutex, ma_uid, ma_from, ma_to, ma_cc, ma_sub, ma_flag, ma_ftime, ma_dbusr, ma_dbtns, ma_msg) #--testing please remark this codes logaq('Deleting mail uid:' + ma_uid, 'i') conn.store(mailbidx[num], '+FLAGS', '\\Deleted') #delete mail #--testing remark logaq(pmailslst) setmailsaok(public.gmutex_mailsa, True) logaq('maila_main sleep ' + str(public.gsleep) + 's,refresh_ini...') time.sleep(public.gsleep) refresh_ini() #break;#if delete mail,please mark break except Exception as e: logaq('mailsa while Error: %s' % e, 'e') time.sleep(3) finally: try: conn.close() except Exception as e: logaq('connect close exception: %s' % e, 'i') conn.logout() logaq('logout mailserver', 'i') saveobject(pmailslst, 'maillst') logaq('serializing maillist', 'i')
def s2file_main(pmailslst, pparaslst, pmutex, pdb, pproclst, pmutexp, pdbconlst): #add pdbconlst while not getmailsaok(public.gmutex_mailsa) and not breaktime(): logaq('waiting for mailsa 10s...', 'i') time.sleep(10) continue preuid = '' curuid = '' try: fileslst = os.listdir(public.gdir_atachmt_downloaded) fileslst.sort() logaq('FilesCNT:' + str(len(fileslst))) for i in range(0, len(fileslst), 1): try: lexistnewfile = False #only process file start with pdb logaq('Process idx:' + str(i)) if get2betw13(fileslst[i], '', public.GPREFIX_FILEUID) == pdb: logaq('Curfile:' + fileslst[i]) lpath = os.path.join(public.gdir_atachmt_downloaded, fileslst[i]) curuid = get2betw13(fileslst[i], public.GPREFIX_FILEUID, public.GSEPTOR_FILE) #if exist one attachment error in same mail,will not process others if getflagbyuid(curuid, pmailslst) == public.GSTATUS_ERROR: if os.path.exists(public.gdir_history + fileslst[i]): os.remove(public.gdir_history + fileslst[i]) logaq('moving file to his') shutil.move(lpath, public.gdir_history) continue #if all attachment have been processed in same mail,update flag to GSTATUS_SQLDATA_EXPORTED if preuid and preuid != curuid: updateflagmsg_byuidflag( pmailslst, pmutex, preuid, public.GSTATUS_SQLDATA_BEFEXPORT, public.GSTATUS_SQLDATA_EXPORTED, '') if time.time() - os.path.getmtime(lpath) <= 1: logaq('MTime<=1:' + lpath) lexistnewfile = True continue if not checkfilestatus(pmailslst, lpath, fileslst[i]): continue else: #process file context #1.judging file charset #2.changing select segment to create or replace view ..... as select segment #3.judging whether exist risk factors if not processfile(pmailslst, pmutex, lpath, fileslst[i]): continue #end process file context #login to sqlplus exec sqlfile if not createviews(pmailslst, pparaslst, pmutex, lpath, fileslst[i]): continue else: callprocedure(pmailslst, pparaslst, pmutex, lpath, fileslst[i], pproclst, pmutexp, pdbconlst) preuid = curuid logaq('Curfile end') except Exception as e: logaq('s2file Forloop Error: %s' % e, 'e') time.sleep(3) updateflagmsg_byuidflag(pmailslst, pmutex, curuid, public.GSTATUS_ATACHMT_DOWNLOADED, public.GSTATUS_ERROR, str(e)) updateflagmsg_byuidflag(pmailslst, pmutex, curuid, public.GSTATUS_SQLDATA_BEFEXPORT, public.GSTATUS_ERROR, str(e)) updateflagmsg_byuidflag(pmailslst, pmutex, curuid, public.GSTATUS_SQLDATA_BEFEXPORT, public.GSTATUS_SQLDATA_EXPORTED, '') if not lexistnewfile: setmailsaok(public.gmutex_mailsa, False) else: time.sleep(3) sets2fileok(public.gmutex_s2file, True) logaq(pmailslst) except Exception as e: logaq('s2file Error: %s' % e, 'e') finally: #saveobject(pmailslst,'maills2') logaq('s2file finally')
def process_mailbody(pparaslst, pmsg, pmidx): isfirstpart = 1 idxattach = 0 global ma_existatta global ma_chset global ma_flag global ma_ftime global ma_dbusr global ma_dbtns global ma_msg ma_existatta = False for part in pmsg.walk(): if not part.is_multipart(): try: ma_chset = chardet.detect( part.get_payload(decode=True))['encoding'] logaq('bodyencode:' + str(ma_chset)) #ma_chset='utf-8' part.get_payload(decode=True).decode(ma_chset) #mail content except Exception as e: logaq( str(ma_chset) + ' decode error(will default utf-8): %s' % e, 'e') ma_chset = 'utf-8' part.get_payload(decode=True).decode(ma_chset) if not ma_dbusr and isfirstpart: tmpstr = getkeyword( part.get_payload(decode=True).decode(ma_chset), 'username') if tmpstr: ma_dbusr = tmpstr if not ma_dbtns and isfirstpart: tmpstr = getkeyword( part.get_payload(decode=True).decode(ma_chset), 'database') logaq('database:' + tmpstr, 'i') if tmpstr: ma_dbtns = getdbtnsbymaildb(tmpstr, pparaslst, pmidx) logaq('tns:' + ma_dbtns, 'i') if not ma_dbtns: ma_msg = MS_MSG1 ma_flag = public.GSTATUS_ERROR logaq(MS_MSG2, 'e') break isfirstpart = 0 #################-----------downloading attachments file_name = part.get_filename() contType = part.get_content_type() logaq('attachment:' + str(file_name)) if file_name: ma_existatta = True idxattach = idxattach + 1 '''remark for unifying attachment name format:TNS+'_UID'+UID+'_file'+sequence.sql h = email.header.Header(file_name) dh = email.header.decode_header(h)#decode attachment name filename = dh[0][0] logaq(filename) logaq(dh[0][1]) if dh[0][1]: filename = decode_str(str(filename,dh[0][1])) logaq(filename) ''' data = part.get_payload(decode=True) #begin download lfilename = ma_dbtns + '_UID' + ma_uid + '_file' + str( idxattach) logaq('download data to file:' + public.gdir_atachmt_downloaded + lfilename + public.GSUFFIX_FILEMAIL) att_file = open( public.gdir_atachmt_downloaded + lfilename + public.GSUFFIX_FILEMAIL, 'wb') try: att_file.write(data) #save attachment finally: att_file.close()