def process_localfiles(pmailslst, pmutex, pservfileprocessedlst, pfrompath,
ptopath):
#1.move file to his dir
#2.update maillst status by uid&flag
fileslst = os.listdir(pfrompath)
fileslst.sort()
for i in range(0, len(fileslst), 1):
if getidxkeywordinlst(
pservfileprocessedlst,
get2betw13(fileslst[i], public.GFLAG_UID,
public.GSEPTOR_FILE)) > -1:
#move to ptopath
logaq('moving file to ' + ptopath)
shutil.move(pfrompath + fileslst[i], ptopath + fileslst[i])
updateflagmsg_byuidflag(
pmailslst, pmutex,
get2betw13(fileslst[i], public.GFLAG_UID, public.GSEPTOR_FILE),
public.GSTATUS_SQLDATA_EXPORTED,
public.GSTATUS_SQLDATA_DOWNLOADED, '')
else:
#local file exist,serverfile no exist
tmpidx = getidxbyuidflag(
get2betw13(fileslst[i], public.GFLAG_UID, public.GSEPTOR_FILE),
public.GSTATUS_SQLDATA_EXPORTED, pmailslst)
if tmpidx > -1:
#os.rename(pfrompath+fileslst[i],(pfrompath+fileslst[i]).replace(public.GSUFFIX_FILEMAIL,'.drr'))
logaq('moving file to send')
shutil.move(pfrompath + fileslst[i],
public.gdir_atachmt_willsend)
updateflagmsg_byuidflag(
pmailslst, pmutex,
get2betw13(fileslst[i], public.GFLAG_UID,
public.GSEPTOR_FILE),
public.GSTATUS_SQLDATA_EXPORTED, public.GSTATUS_ERROR,
PG_MSG2)
def process_exceptmaillst(pmaillst, pmutex):
#1.reply flag=E mail
#2.reply flag=B&Spent>30Min mail
#3.reply flag=G&Stayfor>2H
idx = 0
while idx < len(pmaillst):
if pmaillst[idx]['ftime'] and pmaillst[idx][
'flag'] == public.GSTATUS_SQLDATA_BEFEXPORT and (
time.time() -
pmaillst[idx]['ftime']) > public.gmmaxqtime + 60:
updateflagmsg_byuidflag(
pmaillst, pmutex, pmaillst[idx]['uid'],
public.GSTATUS_SQLDATA_BEFEXPORT, public.GSTATUS_ERROR,
MS_MSG2.replace('MAX_QTIME', str(public.gmmaxqtime / 60)))
if pmaillst[idx]['ftime'] and pmaillst[idx][
'flag'] == public.GSTATUS_SQLDATA_DOWNLOADED and (
time.time() -
pmaillst[idx]['ftime']) > public.gmmaxqtime * 4:
updateflagmsg_byuidflag(
pmaillst, pmutex, pmaillst[idx]['uid'],
public.GSTATUS_SQLDATA_DOWNLOADED, public.GSTATUS_ERROR,
MS_MSG3.replace('MAX_QTIME', str(public.gmmaxqtime / 60)))
if pmaillst[idx]['flag'] == public.GSTATUS_ERROR:
logaq('sendmail uid:' + pmaillst[idx]['uid'])
send_mail(public.gmusr,
pmaillst[idx]['from'] + ',' + pmaillst[idx]['to'],
pmaillst[idx]['cc'], 'Re:' + pmaillst[idx]['subject'],
MS_MSGH + pmaillst[idx]['msg'] + MS_MSGT, '', '')
logaq('deleting mailmeta by uid')
deldict_fromlist(pmaillst, pmutex, 'uid', pmaillst[idx]['uid'])
idx = idx - 1
idx = idx + 1
def put_main(pmailslst, pparaslst, pmutex):
fileslst = os.listdir(public.gdir_atachmt_downloaded)
fileslst.sort()
curmuid = -1
predbdusr = ''
curdbdusr = ''
for i in range(0, len(fileslst), 1):
print(fileslst[i])
path = os.path.join(public.gdir_atachmt_downloaded, fileslst[i])
#process file context
#
curdbdusr = getdbtnsdbusr(get2betw13(fileslst[i], 'UID', '_'),
public.GSTATUS_ATACHMT_DOWNLOADED, pmailslst)
print(curdbdusr)
if not curdbdusr:
os.rename(path, path.replace(public.GSUFFIX_FILEMAIL, '.urr'))
logaq('moving file to senddir')
shutil.move(path.replace(public.GSUFFIX_FILEMAIL, '.urr'),
public.gdir_atachmt_willsend)
updateflagmsg_byuidflag(pmailslst, pmutex,
get2betw13(fileslst[i], 'UID', '_'),
public.GSTATUS_ATACHMT_DOWNLOADED,
public.GSTATUS_ERROR, PG_MSG1)
if i > 0:
updateflagmsg_byuidflag(
pmailslst, pmutex, get2betw13(fileslst[i - 1], 'UID', '_'),
public.GSTATUS_ATACHMT_DOWNLOADED,
public.GSTATUS_ATACHMT_UPLOADED, '')
elif predbdusr != curdbdusr:
#getoinfo
if predbdusr:
transport.close()
loip, loprt, lousr, lopd = getoinforbytnsusr(curdbdusr, pparaslst)
print(loip + ',' + loprt + ',' + lousr + ',' + lopd)
transport = paramiko.Transport((loip, int(loprt)))
transport.connect(username=lousr, password=lopd)
sftp = paramiko.SFTPClient.from_transport(transport)
if os.path.isfile(path):
sftp.put(path, '/home/test/' + fileslst[i])
shutil.move(path, public.gdir_atachmt_uploaded)
#updatedict_inlist(pmailslst,pmutex,pg_dictname_uid,get2betw13(fileslst[i],'UID','_'),pg_dictname_flag,public.GSTATUS_ATACHMT_DOWNLOADED,public.GSTATUS_ATACHMT_UPLOADED)
if (predbdusr and predbdusr != curdbdusr) or (i == len(fileslst) - 1):
idxlast = i - 1
if i == len(fileslst) - 1:
idxlast = len(fileslst) - 1
print(get2betw13(fileslst[idxlast], 'UID', '_'))
updateflagmsg_byuidflag(pmailslst, pmutex,
get2betw13(fileslst[idxlast], 'UID', '_'),
public.GSTATUS_ATACHMT_DOWNLOADED,
public.GSTATUS_ATACHMT_UPLOADED, '')
if i == len(fileslst) - 1:
transport.close()
#if fileslst[i].find('DW')>-1:#fileuid not match mailslst uid
#print('----\n'+public.gdir_atachmt_uploaded+fileslst[i]+'\n'+public.gdir_atachmt_uploaded+'err.err'+'----\n')
#os.rename(public.gdir_atachmt_uploaded+fileslst[i],public.gdir_atachmt_uploaded+'err.err')
predbdusr = curdbdusr
def getdel_servfiles(ptransport, pserloginhome, plocalpath, pfilelstremove):
sftp = paramiko.SFTPClient.from_transport(ptransport)
try:
tmpfiles = sftp.listdir()
logaq('sftp filelst:' + str(tmpfiles), 'i')
tmpfiles.sort()
for tfile in tmpfiles:
if tfile.find('UID') > -1 and tfile.find('tar.gz') > 0:
logaq('get&del:' + tfile, 'i')
sftp.get(pserloginhome + tfile, plocalpath + tfile)
pfilelstremove.append(tfile)
sftp.remove(pserloginhome + tfile)
finally:
sftp.close()
def send_mail(pfrom, pto, pcc, psub, pcontt, pdiratta, plstatta):
#mailheader
global ms_serv
message = MIMEMultipart()
message['From'] = pfrom
message['To'] = pto
message['Cc'] = pcc
message['Subject'] = psub
#mailbody
message.attach(MIMEText(pcontt, 'plain', 'utf-8')) #plain or html
#attachment
if pdiratta and plstatta:
for filen in plstatta.split(','):
logaq('uploading mail attachment:' + filen, 'i')
atta = MIMEApplication(open(pdiratta + '\\' + filen, 'rb').read())
atta.add_header('Content-Disposition',
'attachment',
filename=filen)
message.attach(atta)
try:
lreceivers = pto + ',' + pcc
logaq('receivers:' + lreceivers)
ms_serv.sendmail(pfrom, lreceivers.split(','), message.as_string())
except Exception as e:
logaq('mailss sendmail Error: %s' % e, 'e')
ms_serv = smtplib.SMTP_SSL(public.gshst, public.gsprt)
ms_serv.login(public.gmusr, getpdbyen(public.gmenpd))
logaq('smtp server relogin ok', 'i')
ms_serv.sendmail(pfrom, lreceivers.split(','), message.as_string())
def process_mailheader(pmsg):
global ma_from
global ma_to
global ma_cc
global ma_sub
logaq('--from||to||cc||subject--', 'i')
ma_sub = clearctrlchar(decode_str(pmsg.get('subject')))
ma_from = clearctrlchar(email.utils.parseaddr(pmsg.get('from'))[1])
ma_to = clearctrlchar(pmsg.get('to'))
ma_to = format_tocc(ma_to)
cc = pmsg.get('cc')
if cc:
ma_cc = clearctrlchar(cc)
ma_cc = format_tocc(ma_cc)
logaq(ma_from + '||' + ma_to + '||' + ma_cc + '||' + ma_sub, 'i')
def processfile(pmailslst, pmutex, ppath, pfilename):
#process file content for creating view
#Format:
#old file:select ...... from ......where ......;select ...... from ...... where ......;
#new file:create view filename_seq1V as select ...... from ......where ......;create view filename_seq2V as select ...... from ...... where ......;
global s2_charset
ltmpuid = get2betw13(pfilename, public.GPREFIX_FILEUID,
public.GSEPTOR_FILE)
tmpfd = open(ppath, 'rb')
try:
tmpdata = tmpfd.read()
finally:
tmpfd.close()
s2_charset = chardet.detect(tmpdata)['encoding']
if s2_charset[
0:
3] == 'UTF': #AMERICAN_AMERICA.ZHS16GBK AL32UTF8 ZHS16CGB231280
os.environ["NLS_LANG"] = public.GNLS_LANG_UTF8
else:
os.environ["NLS_LANG"] = public.GNLS_LANG_GBK
if check_fileformat(tmpdata.decode(s2_charset)):
lviewname = public.GFLAG_UID + get2betw13(
pfilename, public.GPREFIX_FILEUID, public.GSUFFIX_FILEMAIL)
tmpfiletext = replace_oldkeyword(
tmpdata.decode(s2_charset), S2_SELECT,
S2_CREATEORREP + lviewname + S2_SUFFIX, S2_SEGSEPTOR, 1,
lviewname + '_')
tmperr, tmpfiletext = remove_riskfactors(tmpfiletext)
if tmperr:
updateflagmsg_byuidflag(pmailslst, pmutex, ltmpuid,
public.GSTATUS_ATACHMT_DOWNLOADED,
public.GSTATUS_ERROR,
S2_RFACTORHINT + tmperr + S2_TAIL)
logaq(pfilename + ':' + S2_RFACTORHINT + tmperr, 'e')
if os.path.exists(public.gdir_history + pfilename):
os.remove(public.gdir_history + pfilename)
logaq('moving file to his')
shutil.move(ppath, public.gdir_history)
return False
tmpdata = tmpfiletext.encode(s2_charset)
tmpfd = open(ppath, 'wb')
try:
tmpfd.write(tmpdata)
finally:
tmpfd.close()
return True
else:
updateflagmsg_byuidflag(pmailslst, pmutex, ltmpuid,
public.GSTATUS_ATACHMT_DOWNLOADED,
public.GSTATUS_ERROR, S2_MSG4)
logaq(pfilename + ':' + S2_MSG4, 'e')
if os.path.exists(public.gdir_history + pfilename):
os.remove(public.gdir_history + pfilename)
logaq('moving file to his')
shutil.move(ppath, public.gdir_history)
return False
def callprocedure(pmailslst, pparaslst, pmutex, ppath, pfilename, pproclst,
pmutexp, pdbconlst): #add pdbconlst
#!!!!!!!!!!this procedure maybe spend a long time!!!!!!!!!!!
#BUG FINDED 20181123
ltmpuid = get2betw13(pfilename, public.GPREFIX_FILEUID,
public.GSEPTOR_FILE)
updateflagmsg_byuidflag(pmailslst, pmutex, ltmpuid,
public.GSTATUS_ATACHMT_DOWNLOADED,
public.GSTATUS_SQLDATA_BEFEXPORT, '', time.time())
ltmpviewlike = public.GFLAG_UID + get2betw13(
pfilename, public.GPREFIX_FILEUID, public.GSUFFIX_FILEMAIL) + '%'
ltmpowner = getdbusrbyuid(ltmpuid, pmailslst).strip()
ltmpdbtns = getdbtnsbyuid(ltmpuid, pmailslst)
if len(ltmpowner) == 0:
ltmpowner = getdbusrbydbtns(ltmpdbtns, pparaslst)
logaq('dbtns:' + ltmpdbtns)
logaq('dbusr:'******'t exit)
#ltmpprocname=S2_PROC_PRODUCTDATA.replace('POWNER',ltmpowner.upper())
#ltmpprocname=ltmpprocname.replace('PVIEWLIKE',ltmpviewlike.upper())
#lrst=execprocedure(getsqlpluscon(ltmpdbtns,ltmpowner,pparaslst),ltmpprocname,pproclst,pmutexp)
lsqlerr = ''
for lerr in S2_ERROR:
if lrst.lower().find(lerr.lower()) > -1:
lsqlerr = lrst
if lsqlerr: #procedure exec error
lsqlerr = clearblankchar(lsqlerr)
updateflagmsg_byuidflag(pmailslst, pmutex, ltmpuid,
public.GSTATUS_SQLDATA_BEFEXPORT,
public.GSTATUS_ERROR, lsqlerr)
logaq(pfilename + ':' + lsqlerr, 'e')
if os.path.exists(public.gdir_history + pfilename):
os.remove(public.gdir_history + pfilename)
logaq('moving file to his')
shutil.move(ppath, public.gdir_history)
else: #cannot updateflag to exported until all UID attachments process over.
# updateflagmsg_byuidflag(pmailslst,pmutex,pmailslst[pidxuid]['uid'],public.GSTATUS_SQLDATA_BEFEXPORT,public.GSTATUS_SQLDATA_EXPORTED,'')
if os.path.exists(public.gdir_history + pfilename):
os.remove(public.gdir_history + pfilename)
logaq('moving file to his')
shutil.move(ppath, public.gdir_history)
def relogin_mail(ppretime, pcon):
try:
if time.time() - ppretime > public.gmcycle:
try:
pcon.close()
pcon.logout()
logaq('mailsa conclosed(>cycle)', 'w')
except Exception as e:
logaq('mailsa conclose: %s' % e, 'e')
tmpcon = imaplib.IMAP4_SSL(port=public.giprt, host=public.gihst)
logaq('connect ok', 'i')
tmpcon.login(public.gmusr, getpdbyen(public.gmenpd))
logaq('login ok', 'i')
return tmpcon
else:
return pcon
except Exception as e:
logaq('mailsa ini cycle: %s' % e, 'e')
def targzipsplit_servfiles(ptransport, puidstringlst):
#ssh = paramiko.SSHClient()
#ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
#ssh.connect(hostname='ip', port=22, username='******', password='******')
#tmploginhome=''
ssh = paramiko.SSHClient()
ssh._transport = ptransport
try:
#stdin, stdout, stderr = ssh.exec_command('pwd')
#res,err = stdout.read(),stderr.read()
#result = res if res else err
#tmploginhome=result.decode()
#logaq(tmploginhome)
'''
stdin, stdout, stderr = ssh.exec_command(command=PG_CMD_TARGZ01)#,environment=PG_ENV_BASH
res,err = stdout.read(),stderr.read()
result = res if res else err
logaq('echo tar&gzip01>*.sh:'+result.decode())
stdin, stdout, stderr = ssh.exec_command(command=PG_CMD_TARGZ02.replace('UIDLST',puidstringlst))
res,err = stdout.read(),stderr.read()
result = res if res else err
logaq('echo tar&gzip02>>*.sh:'+result.decode())
stdin, stdout, stderr = ssh.exec_command(command=PG_CMD_TARGZ03)
res,err = stdout.read(),stderr.read()
result = res if res else err
logaq('echo tar&gzip03>>*.sh:'+result.decode())
#remarked for solaris split -d {} bug
stdin, stdout, stderr = ssh.exec_command(command=PG_CMD_SPLIT50M)
res,err = stdout.read(),stderr.read()
result = res if res else err
logaq('echo split nM file>>*.sh:'+result.decode())
stdin, stdout, stderr = ssh.exec_command(command=PG_CMD_RMGZBEFSPLIT)#window copy /b /path/UID* /path/UID.zip;
res,err = stdout.read(),stderr.read()
result = res if res else err
logaq('echo rm -rf *tar.gz(exist *.gz.00)>>*.sh:'+result.decode())
'''
stdin, stdout, stderr = ssh.exec_command(PG_CMD_SH)
res, err = stdout.read(), stderr.read()
result = res if res else err
logaq('sh *.sh:' + result.decode())
#return tmploginhome
finally:
#ssh.close() #Error SSH session not active
pass
def process_mailslst(pmailslst, pmutex, pservfileprocessedlst, pdbtns):
idxtmp = 0
while idxtmp < len(pmailslst):
luid = pmailslst[idxtmp]['uid']
if getidxkeywordinlst(pservfileprocessedlst,
pmailslst[idxtmp]['uid']) > -1:
updateflagmsg_byuidflag(pmailslst, pmutex,
pmailslst[idxtmp]['uid'],
public.GSTATUS_SQLDATA_EXPORTED,
public.GSTATUS_SQLDATA_DOWNLOADED, '')
elif pmailslst[idxtmp]['dbtns'] == pdbtns and pmailslst[idxtmp][
'flag'] == public.GSTATUS_SQLDATA_EXPORTED:
updateflagmsg_byuidflag(pmailslst, pmutex,
pmailslst[idxtmp]['uid'],
public.GSTATUS_SQLDATA_EXPORTED,
public.GSTATUS_ERROR, PG_MSG3)
if luid != pmailslst[idxtmp]['uid']:
logaq(
'!!!!!!multithreads data conflict(maillst idx which being used is deleted)',
'e')
idxtmp = idxtmp + 1
def relogin_mail(ppretime):
global ms_serv
try:
if time.time() - ppretime > public.gmcycle:
try:
ms_serv.quit()
logaq('mailss servquit(>cycle)', 'w')
except Exception as e:
logaq('mailss conquit: %s' % e, 'e')
ms_serv = smtplib.SMTP_SSL(public.gshst, public.gsprt)
ms_serv.login(public.gmusr, getpdbyen(public.gmenpd))
logaq('smtp server login ok', 'i')
except Exception as e:
logaq('mailss ini cycle: %s' % e, 'e')
def create_s2fthread(pmaillst, pparalst, pmutex, pthreadlst, pproclst, pmutexp,
pdbconlst):
#this is a bottleneck,so must create 3 thread at least
#script execute must be stop in GMAXSECS_RUN seconds
#del pthreadlst[:]#copy:tmplst=lst[:],thread can not use deepcopy
while not breaktime():
try:
check_processthread(pproclst, pthreadlst, pmutexp, pdbconlst)
if len(pthreadlst) < public.gmaxcnt_bnthread:
setdirtns = gettnsset_indir(public.gdir_atachmt_downloaded,
public.GPREFIX_FILEUID)
for st in setdirtns:
if not exist_tnsthread(pthreadlst, st):
logaq(
'create s2fthread:' + public.GPREFIX_BNTHREAD + st,
'i')
t_bn = threading.Thread(target=s2file_main,
args=(pmaillst, pparalst,
pmutex, st, pproclst,
pmutexp, pdbconlst))
t_bn.setName(public.GPREFIX_BNTHREAD + st)
t_bn.setDaemon(False)
t_bn.start()
pthreadlst.append({
'thread': t_bn,
'ctime': time.time()
})
if len(pthreadlst) >= public.gmaxcnt_bnthread:
break
logaq('create_s2fthread sleep ' + str(public.gsleep) +
's,refreshini...')
time.sleep(public.gsleep)
except Exception as e:
time.sleep(3)
logaq('start while Error: %s' % e, 'e')
finally:
pass
#refresh_ini()#call external program,maybe will produce error which casing sys exit
if breaktime():
check_processthread(pproclst, pthreadlst, pmutexp, pdbconlst, True)
def checkfilestatus(pmailslst, ppath, pfilename):
rst = False
tmpidxuid = getidxbyuidflag(
get2betw13(pfilename, public.GPREFIX_FILEUID, public.GSEPTOR_FILE),
public.GSTATUS_SQLDATA_BEFEXPORT, pmailslst)
if tmpidxuid > -1:
logaq(pfilename + ':' + S2_MSG2, 'w')
#shutil.move(ppath,public.gdir_history)
else:
tmpidxuid = getidxbyuidflag(
get2betw13(pfilename, public.GPREFIX_FILEUID, public.GSEPTOR_FILE),
public.GSTATUS_ATACHMT_DOWNLOADED, pmailslst)
if tmpidxuid < 0:
#write error file to his,move this file to his
logaq(pfilename + ':' + S2_MSG1, 'e')
#writefile(os.path.join(public.gdir_history,pfilename),S2_MSG1,'.err')
if os.path.exists(public.gdir_history + pfilename):
os.remove(public.gdir_history + pfilename)
logaq('moving file to his')
shutil.move(ppath, public.gdir_history)
else:
rst = True
return rst
def mails_main(pmaillst, pmutex, pdirattachment):
if breaktime():
exit()
try:
global ms_serv
global ms_msg1
lpretime = time.time()
ms_serv = smtplib.SMTP_SSL(public.gshst, public.gsprt)
ms_serv.login(public.gmusr, getpdbyen(public.gmenpd))
logaq('smtp server login ok', 'i')
while not breaktime():
while not getfilepgok(public.gmutex_filepg) and not breaktime():
logaq('waiting for filepg ' + str(public.gsleep) + 's...', 'i')
time.sleep(public.gsleep)
logaq('process exception mail')
process_exceptmaillst(pmaillst, pmutex)
continue
pretmpmuid = ''
tmpmuid = ''
existsplitf = False
relogin_mail(lpretime)
lpretime = time.time()
try:
lexistnewfile = False
fileslst = os.listdir(pdirattachment)
fileslst.sort()
for i in range(0, len(fileslst), 1):
tmplstfile = fileslst[i]
logaq('file:' + tmplstfile, 'i')
lpath = os.path.join(pdirattachment, tmplstfile)
if time.time() - os.path.getmtime(lpath) <= 1:
logaq('mtime<=1:' + lpath)
lexistnewfile = True
continue
tmpmuid = get2betw13(tmplstfile, public.GFLAG_UID,
public.GSUFFIX_TARGZFILEMAIL)
#--split file process
if tmpmuid == pretmpmuid:
existsplitf = True
if (existsplitf and (i<len(fileslst)-1) and get2betw13(fileslst[i+1],public.GFLAG_UID,'_')!=tmpmuid) \
or (existsplitf and i==len(fileslst)-1):
tmpfilelike = get2betw13(
tmplstfile, '', public.GSUFFIX_TARGZFILEMAIL
) + public.GSUFFIX_TARGZFILEMAIL
writefile(
pdirattachment + 'UID' + pretmpmuid,
ms_cmd_merge.replace('FILENAME', tmpfilelike),
'.cmd')
tmplstfile = tmplstfile + ',UID' + pretmpmuid + '.cmd'
existsplitf = False
#--end split file process
idxmail = getidxbyuidflag(
tmpmuid, public.GSTATUS_SQLDATA_DOWNLOADED, pmaillst)
ltmpuid = pmaillst[idxmail]['uid']
#get mailheader info
if idxmail > -1:
logaq('sendmail uid:' + tmpmuid)
ms_msg1 = ms_msg1.replace('STOPTIME',
public.gstop_time)
ms_msg1 = ms_msg1.replace('MAX_QTIME',
str(public.gmmaxqtime / 60))
send_mail(
public.gmusr, pmaillst[idxmail]['from'] + ',' +
pmaillst[idxmail]['to'], pmaillst[idxmail]['cc'],
'Re:' + pmaillst[idxmail]['subject'], ms_msg1,
public.gdir_atachmt_willsend, tmplstfile)
if pmaillst[idxmail]['uid'] != ltmpuid:
logaq('!!!!!!multithreads data conflict!!!!!!', 'e')
if os.path.exists(public.gdir_history + fileslst[i]):
os.remove(public.gdir_history + fileslst[i])
logaq('moving file to his uid:' + tmpmuid)
shutil.move(lpath, public.gdir_history)
#--delete pmaillst
if pretmpmuid and tmpmuid != pretmpmuid:
logaq('deleting mailmeta by preuid(cur<>pre)')
deldict_fromlist(pmaillst, pmutex, 'uid', pretmpmuid)
if i == (len(fileslst) - 1):
logaq('deleting mailmeta by uid(lastone)')
deldict_fromlist(pmaillst, pmutex, 'uid', tmpmuid)
#--end delete
pretmpmuid = tmpmuid
process_exceptmaillst(pmaillst, pmutex)
if not lexistnewfile:
setfilepgok(public.gmutex_filepg, False)
else:
time.sleep(3)
logaq(pmaillst)
logaq('while end', 'i')
except Exception as e:
logaq('mailss Error: %s' % e, 'e')
time.sleep(3)
finally:
logaq('while finally', 'i')
saveobject(pmaillst, 'maills4')
#refresh_ini()
finally:
try:
ms_serv.quit()
except Exception as e:
logaq('smtp exit exception: %s' % e, 'i')
lpypara = sys.argv[1]
if helps(lpypara):
exit()
try:
#1.init; get ini config parameter,init dirs,log,set loglevel by arg passed
refresh_ini()
init_dirs()
init_log(lpypara)
#2.refresh parameters of db
refresh_paras(gparaslst)
#3.prevent script runing again
if isrunning():
exit()
#4.create thread:mails analyzing&downloading
logaq('--starting--', 'i')
logaq('create thread Thread_mailsa', 'i')
t_mailsa = threading.Thread(target=maila_main,
args=(gmailslst, gparaslst, gmutex))
t_mailsa.setName('Thread_mailsa1')
t_mailsa.setDaemon(False)
t_mailsa.start()
#5.create thread:exportfiles tar&zip&put/get
logaq('create thread Thread_filepg')
t_filepg = threading.Thread(target=get_main,
args=(gmailslst, gparaslst, gmutex))
t_filepg.setName('Thread_filepg1')
t_filepg.setDaemon(False)
t_filepg.start()
def exeprocedure_bycxora(pconstr, pproname, pproparaowner, pproparaviewlike,
pdbconlst, pmutex):
#instead of execprocedure; solving BUG:mutithread call sqlplus hang
rst = 'begin exeprocedure_bycxora'
logaq('ConToDB constr:' + get2betw13(pconstr, '', '/') + '/******@' +
get2betw13(pconstr, '@', ''))
logaq('callproc:' + pproname + '(' + pproparaowner + ',' +
pproparaviewlike + ')')
try:
odb = cx_Oracle.Connection(pconstr)
dbcur = odb.cursor()
except Exception as e:
logaq('DB connect Error: %s' % e, 'e')
rst = "Error:DB connect Exception"
return rst
try:
add_dbcon(pdbconlst, odb, dbcur,
threading.currentThread().getName(), pmutex)
logaq('dblst size(AaddedBcall):' + str(len(pdbconlst)))
dbcur.callproc(pproname, [pproparaowner, pproparaviewlike])
logaq('remove dbfromlst')
remove_dbcon(pdbconlst, threading.currentThread().getName(), pmutex)
logaq('dblst size(After call&removed):' + str(len(pdbconlst)))
except Exception as e:
logaq('DB Error: %s' % e, 'e')
rst = S2_MSG5
logaq('remove dbfromlst')
remove_dbcon(pdbconlst, threading.currentThread().getName(), pmutex)
logaq('dblst size(After call&removed):' + str(len(pdbconlst)))
finally:
try:
dbcur.close()
except Exception as e:
logaq('dbcur close Error: %s' % e, 'e')
try:
odb.close()
except Exception as e:
logaq('dbcon close Error: %s' % e, 'e')
return rst
def execsfile(pconstr, pfile):
global s2_charset
logaq('NLS_LANG=' + os.environ["NLS_LANG"])
logaq(S2_SQLAPP + ' constr:' + get2betw13(pconstr, '', '/') + '/******@' +
get2betw13(pconstr, '@', '') + ' file:' + pfile) #/xlzhu@
procf = Popen([S2_SQLAPP, '-S', pconstr],
stdout=PIPE,
stdin=PIPE,
stderr=PIPE)
logaq('opensqlplus id:' + str(id(procf)))
logaq('begin create views...')
#procf.stdin.write(('@'+pfile).encode())#exist bug
try:
(outf,
errf) = procf.communicate(input=('@' + pfile).encode(s2_charset),
timeout=60)
logaq('create views end')
except TimeoutExpired:
logaq('sqlplus timeout', 'w')
procf.terminate()
(outf, errf) = procf.communicate()
except Exception as e:
logaq('sqlplus Exception: %s' % e, 'w')
(outf, errf) = procf.communicate()
if procf.returncode != 0:
returnmsgf = errf.decode()
logaq(
'returncode:' + str(procf.returncode) + 'err:' + returnmsgf +
'out:' + outf.decode(), 'e')
if len(returnmsgf) == 0:
returnmsgf = outf.decode()
if len(returnmsgf) == 0:
returnmsgf = S2_MSG5
#sys.exit(procf.returncode)
else:
returnmsgf = outf.decode()
return returnmsgf
def createviews(pmailslst, pparaslst, pmutex, ppath, pfilename):
#login to sqlplus exec sqlfile
rst = 'OK'
lsqlerr = ''
ltmpuid = get2betw13(pfilename, public.GPREFIX_FILEUID,
public.GSEPTOR_FILE)
lcon = getsqlpluscon(getdbtnsbyuid(ltmpuid, pmailslst),
getdbusrbyuid(ltmpuid, pmailslst), pparaslst)
if lcon:
lrst = execsfile(lcon, ppath)
try:
logaq('execsfile return')
except Exception as e:
logaq('log Error: %s' % e, 'e')
else:
logaq(pfilename + ':' + S2_MSG3, 'e')
if os.path.exists(public.gdir_history + pfilename):
os.remove(public.gdir_history + pfilename)
logaq('moving file to his')
shutil.move(ppath, public.gdir_history)
rst = ''
return rst
#logaq('judging returnstr error')
for lerr in S2_ERROR:
if lrst.lower().find(lerr.lower()) > -1:
lsqlerr = lrst
#logaq('judging end 20s')
time.sleep(20)
if lsqlerr: #sqlfile exec error
lsqlerr = clearblankchar(lsqlerr)
updateflagmsg_byuidflag(pmailslst, pmutex, ltmpuid,
public.GSTATUS_ATACHMT_DOWNLOADED,
public.GSTATUS_ERROR, lsqlerr)
logaq(pfilename + ':' + lsqlerr, 'e')
if os.path.exists(public.gdir_history + pfilename):
os.remove(public.gdir_history + pfilename)
logaq('moving file to his')
shutil.move(ppath, public.gdir_history)
rst = ''
return rst
def get_main(pmailslst, pparaslst, pmutex):
'''
#1.get all servers(tnslist) where login to download UID*.csv files
#2.ssh to server for tar&biz2&split all UID*.csv files
#3.sftp to server for downloading&removing all UID*.gz or UID*.gz.00(00 stand for two bit number)
#4.move local file TNS_UID0000000000.sql to his dir
#5.update pmailst status to sqldata_downloaded
'''
tmpfilelst = []
transport = None
while not breaktime():
while not gets2fileok(public.gmutex_s2file) and not breaktime():
logaq('waiting for s2file ' + str(public.gsleep) + 's...', 'i')
time.sleep(public.gsleep)
continue
strlstuid10 = ','
try:
for serv in gettnsset_inmailslst(pmailslst,
public.GSTATUS_SQLDATA_EXPORTED):
tmpip, tmpprt, tmpusr, tmppd = getoinforbytnsusr(
serv + getdbusrbydbtns(serv, pparaslst), pparaslst)
transport = paramiko.Transport((tmpip, int(tmpprt)))
transport.connect(username=tmpusr, password=tmppd)
logaq('server connect ok', 'i')
while strlstuid10:
strlstuid10 = getuidstrlst_inmailslst(
pmailslst, serv, public.GSTATUS_SQLDATA_EXPORTED,
strlstuid10)
if strlstuid10:
logaq('process uid:' + strlstuid10, 'i')
replaceline_infile(PG_CMD_LPATH, PG_CMD_TEMPLATE,
PG_CMD_EXESH, PG_CMD_KEYWORD,
strlstuid10)
put_shfile(transport, PG_CMD_LPATH + PG_CMD_EXESH,
PG_CMD_EXESH)
targzipsplit_servfiles(transport, strlstuid10)
del tmpfilelst[:]
getdel_servfiles(transport, './', public.gdir_atachmt_willsend,
tmpfilelst)
#process_localfiles(pmailslst,pmutex,tmpfilelst,public.gdir_atachmt_downloaded,public.gdir_history)
process_mailslst(pmailslst, pmutex, tmpfilelst, serv)
logaq('sleep 3s for nextloop...', 'i')
time.sleep(3)
logaq('filepg.get_main end', 'i')
if not existflag_inmailslst(pmailslst,
public.GSTATUS_SQLDATA_EXPORTED):
sets2fileok(public.gmutex_s2file, False)
setfilepgok(public.gmutex_filepg, True)
logaq(pmailslst)
except Exception as e:
logaq('filespg Error: %s' % e, 'e')
time.sleep(3)
finally:
if transport:
try:
transport.close()
except Exception as e:
logaq('transport close exception: %s' % e, 'i')
logaq('Dowload datafile of SQLExported over', 'i')
saveobject(pmailslst, 'maills3')
def execprocedure(pconstr, pprocname, pproclst, pmutexp):
'''
if procp.stdin:
procp.stdin.close()
if procp.stdout:
procp.stdout.close()
if procp.stderr:
procp.stderr.close()
'''
logaq(S2_SQLAPP + ' constr:' + pconstr + ' procedure:' + pprocname)
procp = Popen([S2_SQLAPP, '-S', pconstr],
stdout=PIPE,
stdin=PIPE,
stderr=PIPE)
logaq('opensqlplus id:' + str(id(procp)))
add_process(pproclst, procp, threading.currentThread().getName(), pmutexp)
logaq('sqlpluslst size(AaddedBexec):' + str(len(pproclst)))
#procp.stdin.write(pprocname.encode())#exist bug
try:
(outp, errp) = procp.communicate(input=pprocname.encode(),
timeout=public.gmmaxqtime)
except TimeoutExpired:
logaq('sqlplus timeout', 'w')
procp.terminate()
(outp, errp) = procp.communicate()
except Exception as e:
logaq('sqlplus Exception: %s' % e, 'w')
(outp, errp) = procp.communicate()
if procp.returncode != 0:
returnmsgp = errp.decode()
logaq(
'returncode:' + str(procp.returncode) + 'err:' + returnmsgp +
'out:' + outp.decode(), 'e')
if len(returnmsgp) == 0:
returnmsgp = outp.decode()
if len(returnmsgp) == 0:
returnmsgp = S2_MSG5
else:
returnmsgp = outp.decode()
logaq('remove sqlplusfromlst')
remove_process(pproclst, threading.currentThread().getName(), pmutexp)
logaq('sqlpluslst size(After exec&removed):' + str(len(pproclst)))
return returnmsgp
def maila_main(pmailslst, pparaslst, pmutex):
global ma_uid
global ma_from
global ma_to
global ma_cc
global ma_sub
global ma_flag
global ma_ftime
global ma_dbusr
global ma_dbtns
global ma_msg
if breaktime():
exit()
try:
lpretime = time.time()
ma_uid = ''
conn = imaplib.IMAP4_SSL(port=public.giprt, host=public.gihst)
logaq('connect ok', 'i')
conn.login(public.gmusr, getpdbyen(public.gmenpd))
logaq('login ok', 'i')
while not breaktime():
try:
global ma_chset
global ma_existatta
#del pmailslst[:]
conn = relogin_mail(lpretime, conn)
conn.select(public.giwkdir)
logaq('working in folder:%s' % public.giwkdir, 'i')
#type, data = conn.search(None, 'ALL', 'SUBJECT "*review1810221613"')#can not search mail like 'keyword'
type, data = conn.search(None, 'ALL')
mailbidx = data[0].split() #[b'1',b'2',b'3']
mailcnt = len(mailbidx)
logaq('mail cnt:' + str(mailcnt), 'i')
for num in range(
mailcnt - 1, -1, -1
): #process from new mail(if delete mail,Don't begin from old,else Error:Nontype for decode)
try:
#pmailslst.append(newmailmeta())
#get mailuid
clearmailmeta()
type, data = conn.fetch(mailbidx[num], 'UID')
ma_chset = chardet.detect(data[0])['encoding']
logaq('uidencode:' + ma_chset)
ma_uid = getmailuid(
str(
email.message_from_string(
data[0].decode(ma_chset))))
logaq('current mail uid:' + ma_uid, 'i')
#get maildetailinfo
type, data = conn.fetch(mailbidx[num], 'RFC822')
logaq('--MailBinaryInfo:')
logaq(data)
logaq('--EndBinaryInfo')
try:
ma_chset = chardet.detect(data[0][1])['encoding']
logaq('mailencode:' + ma_chset)
except Exception as e:
logaq(
str(ma_chset) +
' decode error(will default utf-8): %s' % e,
'e')
ma_chset = 'utf-8'
msg = email.message_from_string(
data[0][1].decode(ma_chset)) #.decode('gbk')
process_mailheader(msg)
if allow_from(ma_from, public.gmfromallow):
process_mailbody(pparaslst, msg, num)
if ma_msg:
ma_flag = public.GSTATUS_ERROR
else:
if not ma_existatta:
ma_msg = MS_MSG3
ma_flag = public.GSTATUS_ERROR
else:
ma_flag = public.GSTATUS_ATACHMT_DOWNLOADED
else:
logaq(
'mail will be deleted directly(sender not in allow list),uid:'
+ ma_uid, 'w')
ma_uid = ''
except Exception as e:
logaq('mailsa Forloop Error: %s' % e, 'e')
ma_flag = public.GSTATUS_ERROR
ma_msg = str(e)
time.sleep(3)
addmaillist(pmailslst, pmutex, ma_uid, ma_from, ma_to,
ma_cc, ma_sub, ma_flag, ma_ftime, ma_dbusr,
ma_dbtns, ma_msg)
#--testing please remark this codes
logaq('Deleting mail uid:' + ma_uid, 'i')
conn.store(mailbidx[num], '+FLAGS',
'\\Deleted') #delete mail
#--testing remark
logaq(pmailslst)
setmailsaok(public.gmutex_mailsa, True)
logaq('maila_main sleep ' + str(public.gsleep) +
's,refresh_ini...')
time.sleep(public.gsleep)
refresh_ini()
#break;#if delete mail,please mark break
except Exception as e:
logaq('mailsa while Error: %s' % e, 'e')
time.sleep(3)
finally:
try:
conn.close()
except Exception as e:
logaq('connect close exception: %s' % e, 'i')
conn.logout()
logaq('logout mailserver', 'i')
saveobject(pmailslst, 'maillst')
logaq('serializing maillist', 'i')
def s2file_main(pmailslst, pparaslst, pmutex, pdb, pproclst, pmutexp,
pdbconlst): #add pdbconlst
while not getmailsaok(public.gmutex_mailsa) and not breaktime():
logaq('waiting for mailsa 10s...', 'i')
time.sleep(10)
continue
preuid = ''
curuid = ''
try:
fileslst = os.listdir(public.gdir_atachmt_downloaded)
fileslst.sort()
logaq('FilesCNT:' + str(len(fileslst)))
for i in range(0, len(fileslst), 1):
try:
lexistnewfile = False
#only process file start with pdb
logaq('Process idx:' + str(i))
if get2betw13(fileslst[i], '', public.GPREFIX_FILEUID) == pdb:
logaq('Curfile:' + fileslst[i])
lpath = os.path.join(public.gdir_atachmt_downloaded,
fileslst[i])
curuid = get2betw13(fileslst[i], public.GPREFIX_FILEUID,
public.GSEPTOR_FILE)
#if exist one attachment error in same mail,will not process others
if getflagbyuid(curuid, pmailslst) == public.GSTATUS_ERROR:
if os.path.exists(public.gdir_history + fileslst[i]):
os.remove(public.gdir_history + fileslst[i])
logaq('moving file to his')
shutil.move(lpath, public.gdir_history)
continue
#if all attachment have been processed in same mail,update flag to GSTATUS_SQLDATA_EXPORTED
if preuid and preuid != curuid:
updateflagmsg_byuidflag(
pmailslst, pmutex, preuid,
public.GSTATUS_SQLDATA_BEFEXPORT,
public.GSTATUS_SQLDATA_EXPORTED, '')
if time.time() - os.path.getmtime(lpath) <= 1:
logaq('MTime<=1:' + lpath)
lexistnewfile = True
continue
if not checkfilestatus(pmailslst, lpath, fileslst[i]):
continue
else:
#process file context
#1.judging file charset
#2.changing select segment to create or replace view ..... as select segment
#3.judging whether exist risk factors
if not processfile(pmailslst, pmutex, lpath,
fileslst[i]):
continue
#end process file context
#login to sqlplus exec sqlfile
if not createviews(pmailslst, pparaslst, pmutex, lpath,
fileslst[i]):
continue
else:
callprocedure(pmailslst, pparaslst, pmutex, lpath,
fileslst[i], pproclst, pmutexp,
pdbconlst)
preuid = curuid
logaq('Curfile end')
except Exception as e:
logaq('s2file Forloop Error: %s' % e, 'e')
time.sleep(3)
updateflagmsg_byuidflag(pmailslst, pmutex, curuid,
public.GSTATUS_ATACHMT_DOWNLOADED,
public.GSTATUS_ERROR, str(e))
updateflagmsg_byuidflag(pmailslst, pmutex, curuid,
public.GSTATUS_SQLDATA_BEFEXPORT,
public.GSTATUS_ERROR, str(e))
updateflagmsg_byuidflag(pmailslst, pmutex, curuid,
public.GSTATUS_SQLDATA_BEFEXPORT,
public.GSTATUS_SQLDATA_EXPORTED, '')
if not lexistnewfile:
setmailsaok(public.gmutex_mailsa, False)
else:
time.sleep(3)
sets2fileok(public.gmutex_s2file, True)
logaq(pmailslst)
except Exception as e:
logaq('s2file Error: %s' % e, 'e')
finally:
#saveobject(pmailslst,'maills2')
logaq('s2file finally')
def process_mailbody(pparaslst, pmsg, pmidx):
isfirstpart = 1
idxattach = 0
global ma_existatta
global ma_chset
global ma_flag
global ma_ftime
global ma_dbusr
global ma_dbtns
global ma_msg
ma_existatta = False
for part in pmsg.walk():
if not part.is_multipart():
try:
ma_chset = chardet.detect(
part.get_payload(decode=True))['encoding']
logaq('bodyencode:' + str(ma_chset))
#ma_chset='utf-8'
part.get_payload(decode=True).decode(ma_chset) #mail content
except Exception as e:
logaq(
str(ma_chset) +
' decode error(will default utf-8): %s' % e, 'e')
ma_chset = 'utf-8'
part.get_payload(decode=True).decode(ma_chset)
if not ma_dbusr and isfirstpart:
tmpstr = getkeyword(
part.get_payload(decode=True).decode(ma_chset), 'username')
if tmpstr:
ma_dbusr = tmpstr
if not ma_dbtns and isfirstpart:
tmpstr = getkeyword(
part.get_payload(decode=True).decode(ma_chset), 'database')
logaq('database:' + tmpstr, 'i')
if tmpstr:
ma_dbtns = getdbtnsbymaildb(tmpstr, pparaslst, pmidx)
logaq('tns:' + ma_dbtns, 'i')
if not ma_dbtns:
ma_msg = MS_MSG1
ma_flag = public.GSTATUS_ERROR
logaq(MS_MSG2, 'e')
break
isfirstpart = 0
#################-----------downloading attachments
file_name = part.get_filename()
contType = part.get_content_type()
logaq('attachment:' + str(file_name))
if file_name:
ma_existatta = True
idxattach = idxattach + 1
'''remark for unifying attachment name format:TNS+'_UID'+UID+'_file'+sequence.sql
h = email.header.Header(file_name)
dh = email.header.decode_header(h)#decode attachment name
filename = dh[0][0]
logaq(filename)
logaq(dh[0][1])
if dh[0][1]:
filename = decode_str(str(filename,dh[0][1]))
logaq(filename)
'''
data = part.get_payload(decode=True) #begin download
lfilename = ma_dbtns + '_UID' + ma_uid + '_file' + str(
idxattach)
logaq('download data to file:' +
public.gdir_atachmt_downloaded + lfilename +
public.GSUFFIX_FILEMAIL)
att_file = open(
public.gdir_atachmt_downloaded + lfilename +
public.GSUFFIX_FILEMAIL, 'wb')
try:
att_file.write(data) #save attachment
finally:
att_file.close()
