def get_main(pmailslst, pparaslst, pmutex):
'''
#1.get all servers(tnslist) where login to download UID*.csv files
#2.ssh to server for tar&biz2&split all UID*.csv files
#3.sftp to server for downloading&removing all UID*.gz or UID*.gz.00(00 stand for two bit number)
#4.move local file TNS_UID0000000000.sql to his dir
#5.update pmailst status to sqldata_downloaded
'''
tmpfilelst = []
transport = None
while not breaktime():
while not gets2fileok(public.gmutex_s2file) and not breaktime():
logaq('waiting for s2file ' + str(public.gsleep) + 's...', 'i')
time.sleep(public.gsleep)
continue
strlstuid10 = ','
try:
for serv in gettnsset_inmailslst(pmailslst,
public.GSTATUS_SQLDATA_EXPORTED):
tmpip, tmpprt, tmpusr, tmppd = getoinforbytnsusr(
serv + getdbusrbydbtns(serv, pparaslst), pparaslst)
transport = paramiko.Transport((tmpip, int(tmpprt)))
transport.connect(username=tmpusr, password=tmppd)
logaq('server connect ok', 'i')
while strlstuid10:
strlstuid10 = getuidstrlst_inmailslst(
pmailslst, serv, public.GSTATUS_SQLDATA_EXPORTED,
strlstuid10)
if strlstuid10:
logaq('process uid:' + strlstuid10, 'i')
replaceline_infile(PG_CMD_LPATH, PG_CMD_TEMPLATE,
PG_CMD_EXESH, PG_CMD_KEYWORD,
strlstuid10)
put_shfile(transport, PG_CMD_LPATH + PG_CMD_EXESH,
PG_CMD_EXESH)
targzipsplit_servfiles(transport, strlstuid10)
del tmpfilelst[:]
getdel_servfiles(transport, './', public.gdir_atachmt_willsend,
tmpfilelst)
#process_localfiles(pmailslst,pmutex,tmpfilelst,public.gdir_atachmt_downloaded,public.gdir_history)
process_mailslst(pmailslst, pmutex, tmpfilelst, serv)
logaq('sleep 3s for nextloop...', 'i')
time.sleep(3)
logaq('filepg.get_main end', 'i')
if not existflag_inmailslst(pmailslst,
public.GSTATUS_SQLDATA_EXPORTED):
sets2fileok(public.gmutex_s2file, False)
setfilepgok(public.gmutex_filepg, True)
logaq(pmailslst)
except Exception as e:
logaq('filespg Error: %s' % e, 'e')
time.sleep(3)
finally:
if transport:
try:
transport.close()
except Exception as e:
logaq('transport close exception: %s' % e, 'i')
logaq('Dowload datafile of SQLExported over', 'i')
saveobject(pmailslst, 'maills3')
def create_s2fthread(pmaillst, pparalst, pmutex, pthreadlst, pproclst, pmutexp,
pdbconlst):
#this is a bottleneck,so must create 3 thread at least
#script execute must be stop in GMAXSECS_RUN seconds
#del pthreadlst[:]#copy:tmplst=lst[:],thread can not use deepcopy
while not breaktime():
try:
check_processthread(pproclst, pthreadlst, pmutexp, pdbconlst)
if len(pthreadlst) < public.gmaxcnt_bnthread:
setdirtns = gettnsset_indir(public.gdir_atachmt_downloaded,
public.GPREFIX_FILEUID)
for st in setdirtns:
if not exist_tnsthread(pthreadlst, st):
logaq(
'create s2fthread:' + public.GPREFIX_BNTHREAD + st,
'i')
t_bn = threading.Thread(target=s2file_main,
args=(pmaillst, pparalst,
pmutex, st, pproclst,
pmutexp, pdbconlst))
t_bn.setName(public.GPREFIX_BNTHREAD + st)
t_bn.setDaemon(False)
t_bn.start()
pthreadlst.append({
'thread': t_bn,
'ctime': time.time()
})
if len(pthreadlst) >= public.gmaxcnt_bnthread:
break
logaq('create_s2fthread sleep ' + str(public.gsleep) +
's,refreshini...')
time.sleep(public.gsleep)
except Exception as e:
time.sleep(3)
logaq('start while Error: %s' % e, 'e')
finally:
pass
#refresh_ini()#call external program,maybe will produce error which casing sys exit
if breaktime():
check_processthread(pproclst, pthreadlst, pmutexp, pdbconlst, True)
def mails_main(pmaillst, pmutex, pdirattachment):
if breaktime():
exit()
try:
global ms_serv
global ms_msg1
lpretime = time.time()
ms_serv = smtplib.SMTP_SSL(public.gshst, public.gsprt)
ms_serv.login(public.gmusr, getpdbyen(public.gmenpd))
logaq('smtp server login ok', 'i')
while not breaktime():
while not getfilepgok(public.gmutex_filepg) and not breaktime():
logaq('waiting for filepg ' + str(public.gsleep) + 's...', 'i')
time.sleep(public.gsleep)
logaq('process exception mail')
process_exceptmaillst(pmaillst, pmutex)
continue
pretmpmuid = ''
tmpmuid = ''
existsplitf = False
relogin_mail(lpretime)
lpretime = time.time()
try:
lexistnewfile = False
fileslst = os.listdir(pdirattachment)
fileslst.sort()
for i in range(0, len(fileslst), 1):
tmplstfile = fileslst[i]
logaq('file:' + tmplstfile, 'i')
lpath = os.path.join(pdirattachment, tmplstfile)
if time.time() - os.path.getmtime(lpath) <= 1:
logaq('mtime<=1:' + lpath)
lexistnewfile = True
continue
tmpmuid = get2betw13(tmplstfile, public.GFLAG_UID,
public.GSUFFIX_TARGZFILEMAIL)
#--split file process
if tmpmuid == pretmpmuid:
existsplitf = True
if (existsplitf and (i<len(fileslst)-1) and get2betw13(fileslst[i+1],public.GFLAG_UID,'_')!=tmpmuid) \
or (existsplitf and i==len(fileslst)-1):
tmpfilelike = get2betw13(
tmplstfile, '', public.GSUFFIX_TARGZFILEMAIL
) + public.GSUFFIX_TARGZFILEMAIL
writefile(
pdirattachment + 'UID' + pretmpmuid,
ms_cmd_merge.replace('FILENAME', tmpfilelike),
'.cmd')
tmplstfile = tmplstfile + ',UID' + pretmpmuid + '.cmd'
existsplitf = False
#--end split file process
idxmail = getidxbyuidflag(
tmpmuid, public.GSTATUS_SQLDATA_DOWNLOADED, pmaillst)
ltmpuid = pmaillst[idxmail]['uid']
#get mailheader info
if idxmail > -1:
logaq('sendmail uid:' + tmpmuid)
ms_msg1 = ms_msg1.replace('STOPTIME',
public.gstop_time)
ms_msg1 = ms_msg1.replace('MAX_QTIME',
str(public.gmmaxqtime / 60))
send_mail(
public.gmusr, pmaillst[idxmail]['from'] + ',' +
pmaillst[idxmail]['to'], pmaillst[idxmail]['cc'],
'Re:' + pmaillst[idxmail]['subject'], ms_msg1,
public.gdir_atachmt_willsend, tmplstfile)
if pmaillst[idxmail]['uid'] != ltmpuid:
logaq('!!!!!!multithreads data conflict!!!!!!', 'e')
if os.path.exists(public.gdir_history + fileslst[i]):
os.remove(public.gdir_history + fileslst[i])
logaq('moving file to his uid:' + tmpmuid)
shutil.move(lpath, public.gdir_history)
#--delete pmaillst
if pretmpmuid and tmpmuid != pretmpmuid:
logaq('deleting mailmeta by preuid(cur<>pre)')
deldict_fromlist(pmaillst, pmutex, 'uid', pretmpmuid)
if i == (len(fileslst) - 1):
logaq('deleting mailmeta by uid(lastone)')
deldict_fromlist(pmaillst, pmutex, 'uid', tmpmuid)
#--end delete
pretmpmuid = tmpmuid
process_exceptmaillst(pmaillst, pmutex)
if not lexistnewfile:
setfilepgok(public.gmutex_filepg, False)
else:
time.sleep(3)
logaq(pmaillst)
logaq('while end', 'i')
except Exception as e:
logaq('mailss Error: %s' % e, 'e')
time.sleep(3)
finally:
logaq('while finally', 'i')
saveobject(pmaillst, 'maills4')
#refresh_ini()
finally:
try:
ms_serv.quit()
except Exception as e:
logaq('smtp exit exception: %s' % e, 'i')
#6.create thread:mails sending
logaq('create thread Thread_mailss')
t_mailss = threading.Thread(target=mails_main,
args=(gmailslst, gmutex,
public.gdir_atachmt_willsend))
t_mailss.setName('Thread_mailss1')
t_mailss.setDaemon(False)
t_mailss.start()
#7.create N thread of s2file which export data from database will spend a long time(bottleneck)
create_s2fthread(gmailslst, gparaslst, gmutex, gthreadlst_bn,
gprocesslst_bn, gmutexprocess, gdbconlst_bn)
except Exception as e:
logaq('start Error: %s' % e, 'e')
finally:
logaq('maillist infor:')
logaq(gmailslst)
logaq('paralist infor:')
logaq(gparaslst)
logaq('s2fprocesslist infor:')
logaq(gprocesslst_bn)
logaq('s2fdbconlist infor:')
logaq(gdbconlst_bn)
logaq('s2fhreadlist infor:')
logaq(gthreadlst_bn)
logaq('--startend--', 'i')
if breaktime():
logaq('this script exit by setting of stoptime parameter', 'i')
def s2file_main(pmailslst, pparaslst, pmutex, pdb, pproclst, pmutexp,
pdbconlst): #add pdbconlst
while not getmailsaok(public.gmutex_mailsa) and not breaktime():
logaq('waiting for mailsa 10s...', 'i')
time.sleep(10)
continue
preuid = ''
curuid = ''
try:
fileslst = os.listdir(public.gdir_atachmt_downloaded)
fileslst.sort()
logaq('FilesCNT:' + str(len(fileslst)))
for i in range(0, len(fileslst), 1):
try:
lexistnewfile = False
#only process file start with pdb
logaq('Process idx:' + str(i))
if get2betw13(fileslst[i], '', public.GPREFIX_FILEUID) == pdb:
logaq('Curfile:' + fileslst[i])
lpath = os.path.join(public.gdir_atachmt_downloaded,
fileslst[i])
curuid = get2betw13(fileslst[i], public.GPREFIX_FILEUID,
public.GSEPTOR_FILE)
#if exist one attachment error in same mail,will not process others
if getflagbyuid(curuid, pmailslst) == public.GSTATUS_ERROR:
if os.path.exists(public.gdir_history + fileslst[i]):
os.remove(public.gdir_history + fileslst[i])
logaq('moving file to his')
shutil.move(lpath, public.gdir_history)
continue
#if all attachment have been processed in same mail,update flag to GSTATUS_SQLDATA_EXPORTED
if preuid and preuid != curuid:
updateflagmsg_byuidflag(
pmailslst, pmutex, preuid,
public.GSTATUS_SQLDATA_BEFEXPORT,
public.GSTATUS_SQLDATA_EXPORTED, '')
if time.time() - os.path.getmtime(lpath) <= 1:
logaq('MTime<=1:' + lpath)
lexistnewfile = True
continue
if not checkfilestatus(pmailslst, lpath, fileslst[i]):
continue
else:
#process file context
#1.judging file charset
#2.changing select segment to create or replace view ..... as select segment
#3.judging whether exist risk factors
if not processfile(pmailslst, pmutex, lpath,
fileslst[i]):
continue
#end process file context
#login to sqlplus exec sqlfile
if not createviews(pmailslst, pparaslst, pmutex, lpath,
fileslst[i]):
continue
else:
callprocedure(pmailslst, pparaslst, pmutex, lpath,
fileslst[i], pproclst, pmutexp,
pdbconlst)
preuid = curuid
logaq('Curfile end')
except Exception as e:
logaq('s2file Forloop Error: %s' % e, 'e')
time.sleep(3)
updateflagmsg_byuidflag(pmailslst, pmutex, curuid,
public.GSTATUS_ATACHMT_DOWNLOADED,
public.GSTATUS_ERROR, str(e))
updateflagmsg_byuidflag(pmailslst, pmutex, curuid,
public.GSTATUS_SQLDATA_BEFEXPORT,
public.GSTATUS_ERROR, str(e))
updateflagmsg_byuidflag(pmailslst, pmutex, curuid,
public.GSTATUS_SQLDATA_BEFEXPORT,
public.GSTATUS_SQLDATA_EXPORTED, '')
if not lexistnewfile:
setmailsaok(public.gmutex_mailsa, False)
else:
time.sleep(3)
sets2fileok(public.gmutex_s2file, True)
logaq(pmailslst)
except Exception as e:
logaq('s2file Error: %s' % e, 'e')
finally:
#saveobject(pmailslst,'maills2')
logaq('s2file finally')
def maila_main(pmailslst, pparaslst, pmutex):
global ma_uid
global ma_from
global ma_to
global ma_cc
global ma_sub
global ma_flag
global ma_ftime
global ma_dbusr
global ma_dbtns
global ma_msg
if breaktime():
exit()
try:
lpretime = time.time()
ma_uid = ''
conn = imaplib.IMAP4_SSL(port=public.giprt, host=public.gihst)
logaq('connect ok', 'i')
conn.login(public.gmusr, getpdbyen(public.gmenpd))
logaq('login ok', 'i')
while not breaktime():
try:
global ma_chset
global ma_existatta
#del pmailslst[:]
conn = relogin_mail(lpretime, conn)
conn.select(public.giwkdir)
logaq('working in folder:%s' % public.giwkdir, 'i')
#type, data = conn.search(None, 'ALL', 'SUBJECT "*review1810221613"')#can not search mail like 'keyword'
type, data = conn.search(None, 'ALL')
mailbidx = data[0].split() #[b'1',b'2',b'3']
mailcnt = len(mailbidx)
logaq('mail cnt:' + str(mailcnt), 'i')
for num in range(
mailcnt - 1, -1, -1
): #process from new mail(if delete mail,Don't begin from old,else Error:Nontype for decode)
try:
#pmailslst.append(newmailmeta())
#get mailuid
clearmailmeta()
type, data = conn.fetch(mailbidx[num], 'UID')
ma_chset = chardet.detect(data[0])['encoding']
logaq('uidencode:' + ma_chset)
ma_uid = getmailuid(
str(
email.message_from_string(
data[0].decode(ma_chset))))
logaq('current mail uid:' + ma_uid, 'i')
#get maildetailinfo
type, data = conn.fetch(mailbidx[num], 'RFC822')
logaq('--MailBinaryInfo:')
logaq(data)
logaq('--EndBinaryInfo')
try:
ma_chset = chardet.detect(data[0][1])['encoding']
logaq('mailencode:' + ma_chset)
except Exception as e:
logaq(
str(ma_chset) +
' decode error(will default utf-8): %s' % e,
'e')
ma_chset = 'utf-8'
msg = email.message_from_string(
data[0][1].decode(ma_chset)) #.decode('gbk')
process_mailheader(msg)
if allow_from(ma_from, public.gmfromallow):
process_mailbody(pparaslst, msg, num)
if ma_msg:
ma_flag = public.GSTATUS_ERROR
else:
if not ma_existatta:
ma_msg = MS_MSG3
ma_flag = public.GSTATUS_ERROR
else:
ma_flag = public.GSTATUS_ATACHMT_DOWNLOADED
else:
logaq(
'mail will be deleted directly(sender not in allow list),uid:'
+ ma_uid, 'w')
ma_uid = ''
except Exception as e:
logaq('mailsa Forloop Error: %s' % e, 'e')
ma_flag = public.GSTATUS_ERROR
ma_msg = str(e)
time.sleep(3)
addmaillist(pmailslst, pmutex, ma_uid, ma_from, ma_to,
ma_cc, ma_sub, ma_flag, ma_ftime, ma_dbusr,
ma_dbtns, ma_msg)
#--testing please remark this codes
logaq('Deleting mail uid:' + ma_uid, 'i')
conn.store(mailbidx[num], '+FLAGS',
'\\Deleted') #delete mail
#--testing remark
logaq(pmailslst)
setmailsaok(public.gmutex_mailsa, True)
logaq('maila_main sleep ' + str(public.gsleep) +
's,refresh_ini...')
time.sleep(public.gsleep)
refresh_ini()
#break;#if delete mail,please mark break
except Exception as e:
logaq('mailsa while Error: %s' % e, 'e')
time.sleep(3)
finally:
try:
conn.close()
except Exception as e:
logaq('connect close exception: %s' % e, 'i')
conn.logout()
logaq('logout mailserver', 'i')
saveobject(pmailslst, 'maillst')
logaq('serializing maillist', 'i')