def getAccountData(datestr, type='all', prov='999'):
logger = logging.getLogger("accountLog")
if type == 'all' or type == 'recharge':
logger.debug(u"开始导出充值相关数据，账期[%s]，省份[%s]", datestr, prov)
dataPath = getconfig.readcfg(
'path', 'exportDataPath') + "recharge/" + datestr + "/"
if prov == '999':
if os.path.exists(dataPath):
shutil.rmtree(dataPath)
os.makedirs(dataPath)
for provCode in commTools.getprovDict().keys(): # 遍历所有省份
connandexport('recharge', dataPath, datestr, provCode)
catandclear('recharge', dataPath, datestr, provCode)
else:
if not os.path.exists(dataPath): # 如果是单个文件下载，则不清除目录，还需要判断目录是否存在
os.makedirs(dataPath)
connandexport('recharge', dataPath, datestr, prov)
catandclear('recharge', dataPath, datestr, prov)
if type == 'all' or type == 'flow':
logger.debug(u"开始导出流量相关数据，账期[%s]，省份[%s]", datestr, prov)
dataPath = getconfig.readcfg(
'path', 'exportDataPath') + "flow/" + datestr + "/"
if prov == '999':
if os.path.exists(dataPath):
shutil.rmtree(dataPath)
os.makedirs(dataPath)
for provCode in commTools.getprovDict().keys(): # 遍历所有省份
connandexport('flow', dataPath, datestr, provCode)
catandclear('flow', dataPath, datestr, provCode)
else:
if not os.path.exists(dataPath): # 如果是单个文件下载，则不清除目录，还需要判断目录是否存在
os.makedirs(dataPath)
connandexport('flow', dataPath, datestr, prov)
catandclear('flow', dataPath, datestr, prov)
if type == 'all' or type == 'commodity':
logger.debug(u"开始导出商品相关数据，账期[%s]，省份[%s]", datestr, prov)
dataPath = getconfig.readcfg(
'path', 'exportDataPath') + "commodity/" + datestr + "/"
if os.path.exists(dataPath):
shutil.rmtree(dataPath)
os.makedirs(dataPath)
connandexport('commodity', dataPath, datestr, '9999999')
catandclear('commodity', dataPath, datestr, '9999999')
if type == 'all' or type == 'package':
logger.debug(u"开始导出套餐相关数据，账期[%s]，省份[%s]", datestr, prov)
dataPath = getconfig.readcfg(
'path', 'exportDataPath') + "package/" + datestr + "/"
if os.path.exists(dataPath):
shutil.rmtree(dataPath)
os.makedirs(dataPath)
connandexport('package', dataPath, datestr, '999')
catandclear('package', dataPath, datestr, '999')
def procalldiff(datestr, type="all", prov='999'):
logger = logging.getLogger("accountLog")
tmpPath = getconfig.readcfg('path', 'tmpcmpPath')
if (type == 'all' or type == 'recharge'):
dataPath = getconfig.readcfg(
'path', 'exportDataPath') + "recharge/" + datestr + "/"
if prov != "999":
couponsfile = dataPath + "account_rechargeDB_" + datestr + "_" + prov + "_001.dat"
difffile, resultfile, retbyoptfile = fileCmp.getresultfilename(
tmpPath, couponsfile)
logger.debug(u"开始处理差异文件[%s]!", difffile)
procdifffile(difffile, retbyoptfile)
else:
for provCode in commTools.getprovDict().keys(): # 遍历所有省份比对
couponsfile = dataPath + "account_rechargeDB_" + datestr + "_" + provCode + "_001.dat"
difffile, resultfile, retbyoptfile = fileCmp.getresultfilename(
tmpPath, couponsfile)
logger.debug(u"开始处理差异文件[%s]!", difffile)
procdifffile(difffile, retbyoptfile)
if (type == 'all' or type == 'flow'):
dataPath = getconfig.readcfg(
'path', 'exportDataPath') + "flow/" + datestr + "/"
if prov != "999":
couponsfile = dataPath + "account_flowDB_" + datestr + "_" + prov + "_001.dat"
difffile, resultfile, retbyoptfile = fileCmp.getresultfilename(
tmpPath, couponsfile)
logger.debug(u"开始处理差异文件[%s]!", difffile)
procdifffile(difffile, retbyoptfile)
else:
for provCode in commTools.getprovDict().keys(): # 遍历所有省份比对
couponsfile = dataPath + "account_flowDB_" + datestr + "_" + provCode + "_001.dat"
difffile, resultfile, retbyoptfile = fileCmp.getresultfilename(
tmpPath, couponsfile)
logger.debug(u"开始处理差异文件[%s]!", difffile)
procdifffile(difffile, retbyoptfile)
if (type == 'all' or type == 'commodity'):
dataPath = getconfig.readcfg(
'path', 'exportDataPath') + "commodity/" + datestr + "/"
couponsfile = dataPath + "account_commodityDB_" + datestr + "_" + "9999999" + "_001.dat"
difffile, resultfile, retbyoptfile = fileCmp.getresultfilename(
tmpPath, couponsfile)
logger.debug(u"开始处理差异文件[%s]!", difffile)
procdifffile(difffile, retbyoptfile)
if (type == 'all' or type == 'package'):
dataPath = getconfig.readcfg(
'path', 'exportDataPath') + "package/" + datestr + "/"
couponsfile = dataPath + "account_packageDB_" + datestr + "_" + "999" + "_001.dat"
difffile, resultfile, retbyoptfile = fileCmp.getresultfilename(
tmpPath, couponsfile)
logger.debug(u"开始处理差异文件[%s]!", difffile)
procdifffile(difffile, retbyoptfile)
def cmpsame(samebusifile, samecouponsfile):
logger = logging.getLogger("accountLog")
# 检查文件是否存在
if not filepathcheck(samebusifile):
return -1
if not filepathcheck(samecouponsfile):
return -2
# 检查文件名是否符合要求
if not checkbothfile(samebusifile, samecouponsfile):
return -3
# 获取差异文件名和结果文件名
tmpPath = getconfig.readcfg('path', 'tmpcmpPath')
if not os.path.exists(tmpPath):
logger.error("config tmpcmpPath [%s] do not exists!", tmpPath)
return -4
difffile, resultfile, retbyoptfile = getresultfilename(tmpPath, samecouponsfile)
try:
# 如果结果文件已经存在需要先删除，避免脏数据
if os.path.exists(resultfile):
os.remove(resultfile)
if os.path.exists(retbyoptfile):
os.remove(retbyoptfile)
busipw, busirecords = readfile(samebusifile)
couponspw, couponsrecords = readfile(samecouponsfile)
if len(busipw) != len(couponspw):
logger.error("error [%s] and [%s] password not same!", samebusifile, samecouponsfile)
return -5
# 按卡密遍历，逐个比对
for pw in couponspw:
if cmprecord(pw, busirecords[pw], couponsrecords[pw], difffile, resultfile, retbyoptfile) != 0:
logger.error("cmpdiff error [%s] and [%s]!", samebusifile, samecouponsfile)
except IOError, e:
logger.error("cmpdiff error [%s] and [%s], error [%s]!", samebusifile, samecouponsfile, e)
return -6
def cmpdiff(busifile, couponsfile):
logger = logging.getLogger("accountLog")
# 检查文件是否存在
if not filepathcheck(busifile):
return -1
if not filepathcheck(couponsfile):
return -2
# 检查文件名是否符合要求
if not checkbothfile(busifile, couponsfile):
return -3
#获取目录和待生成的文件名
tmpPath = getconfig.readcfg('path', 'tmpcmpPath')
if not os.path.exists(tmpPath):
logger.error("config tmpcmpPath [%s] do not exists!", tmpPath)
return -4
busiOnlyfile, couponsOnlyfile, samebusifile, samecouponsfile = commTools.getcmpfilename(tmpPath, couponsfile)
try:
busiPW, busiInfos = readfile(busifile)
couponsPW, couponsInfos = readfile(couponsfile)
# 业务系统比卡券中心多的，处理后形成004差异
busionly = list(set(busiPW).difference(set(couponsPW)))
logger.debug(u"第一次比对[%s] 与 [%s] ， 业务系统多[%d]条记录！",busifile, couponsfile, len(busionly))
fp = open(busiOnlyfile, "wb+")
for pcardpw in busionly:
for record in busiInfos[pcardpw]:
line = "|".join(str(i) for i in record) + "\n"
fp.write(line)
fp.close()
# 业务系统比卡券中心少的，005差异，需要业务系统确认后再处理
# 文件记录时前缀增加005标记
couponsonly = list(set(couponsPW).difference(set(busiPW)))
logger.debug(u"第一次比对[%s] 与 [%s] ， 卡券中心多[%d]条记录！", busifile, couponsfile, len(couponsonly))
fp = open(couponsOnlyfile, "wb+")
for pcardpw in couponsonly:
for record in couponsInfos[pcardpw]:
line = "005|" + "|".join(str(i) for i in record) + "\n"
fp.write(line)
fp.close()
# 双方一致的
samelist = list(set(busiPW) & set(couponsPW))
logger.debug(u"第一次比对[%s] 与 [%s] ， 双方一致[%d]条记录！", busifile, couponsfile, len(samelist))
fp1 = open(samebusifile, "wb+")
fp2 = open(samecouponsfile, "wb+")
for pcardpw in samelist:
for record in busiInfos[pcardpw]:
line = "|".join(str(i) for i in record) + "\n"
fp1.write(line)
for record in couponsInfos[pcardpw]:
line = "|".join(str(i) for i in record) + "\n"
fp2.write(line)
fp1.close()
fp2.close()
except IOError, e:
logger.error("cmpdiff error [%s] and [%s]!",busifile, couponsfile)
return -5
def procbusiOnlyfile(datestr, type = "all", prov = '999'):
logger = logging.getLogger("accountLog")
tmpPath = getconfig.readcfg('path', 'tmpcmpPath')
if (type == 'all' or type == 'recharge'):
dataPath = getconfig.readcfg('path', 'exportDataPath') + "recharge/" + datestr + "/"
if prov != "999":
couponsfile = dataPath + "account_rechargeDB_" + datestr + "_" + prov + "_001.dat"
busiOnlyfile, couponsOnlyfile, samebusifile, samecouponsfile = commTools.getcmpfilename(tmpPath, couponsfile)
logger.debug(u"开始扫描[%s]补充数据!", busiOnlyfile)
replenishdata(busiOnlyfile, couponsOnlyfile, samebusifile, samecouponsfile)
else:
for provCode in commTools.getprovDict().keys(): # 遍历所有省份比对
couponsfile = dataPath + "account_rechargeDB_" + datestr + "_" + provCode + "_001.dat"
busiOnlyfile, couponsOnlyfile, samebusifile, samecouponsfile = commTools.getcmpfilename(tmpPath, couponsfile)
logger.debug(u"开始扫描[%s]补充数据!", busiOnlyfile)
replenishdata(busiOnlyfile, couponsOnlyfile, samebusifile, samecouponsfile)
if (type == 'all' or type == 'flow'):
dataPath = getconfig.readcfg('path', 'exportDataPath') + "flow/" + datestr + "/"
if prov != "999":
couponsfile = dataPath + "account_flowDB_" + datestr + "_" + prov + "_001.dat"
busiOnlyfile, couponsOnlyfile, samebusifile, samecouponsfile = commTools.getcmpfilename(tmpPath, couponsfile)
logger.debug(u"开始扫描[%s]补充数据!", busiOnlyfile)
replenishdata(busiOnlyfile, couponsOnlyfile, samebusifile, samecouponsfile)
else:
for provCode in commTools.getprovDict().keys(): # 遍历所有省份比对
couponsfile = dataPath + "account_flowDB_" + datestr + "_" + provCode + "_001.dat"
busiOnlyfile, couponsOnlyfile, samebusifile, samecouponsfile = commTools.getcmpfilename(tmpPath, couponsfile)
logger.debug(u"开始扫描[%s]补充数据!", busiOnlyfile)
replenishdata(busiOnlyfile, couponsOnlyfile, samebusifile, samecouponsfile)
if (type == 'all' or type == 'commodity'):
dataPath = getconfig.readcfg('path', 'exportDataPath') + "commodity/" + datestr + "/"
couponsfile = dataPath + "account_commodityDB_" + datestr + "_" + "9999999" + "_001.dat"
busiOnlyfile, couponsOnlyfile, samebusifile, samecouponsfile = commTools.getcmpfilename(tmpPath, couponsfile)
logger.debug(u"开始扫描[%s]补充数据!", busiOnlyfile)
replenishdata(busiOnlyfile, couponsOnlyfile, samebusifile, samecouponsfile)
if (type == 'all' or type == 'package'):
dataPath = getconfig.readcfg('path', 'exportDataPath') + "package/" + datestr + "/"
couponsfile = dataPath + "account_packageDB_" + datestr + "_" + "999" + "_001.dat"
busiOnlyfile, couponsOnlyfile, samebusifile, samecouponsfile = commTools.getcmpfilename(tmpPath, couponsfile)
logger.debug(u"开始扫描[%s]补充数据!", busiOnlyfile)
replenishdata(busiOnlyfile, couponsOnlyfile, samebusifile, samecouponsfile)
def connandexport(type, path, datestr, prov):
mysql = getconfig.readcfg('mysql', 'execpath') # 获取mysql命令执行路径
for dbcfg in getconfig.readdblist(): # 遍历所有库
dbflag, dbinfo = dbcfg
host, port, db, usrname, pw = dbinfo.split(',')
if dbflag == "db0_u":
exportData(type, mysql, host, port, usrname, pw, db, path, dbflag,
"u", datestr, prov)
else:
for i in range(0, 10):
exportData(type, mysql, host, port, usrname, pw, db, path,
dbflag, str(i), datestr, prov)
def cmpallfist(datestr, type = "all", prov = '999'):
logger = logging.getLogger("accountLog")
if (type == 'all' or type == 'recharge'):
ftppath = getconfig.readcfg('recharge_sftp', 'localpath') + "recharge/" + datestr + "/"
dataPath = getconfig.readcfg('path', 'exportDataPath') + "recharge/" + datestr + "/" # 获取数据库导出数据路径
if prov != "999":
busifile = ftppath + "account_recharge2coupons_" + datestr + "_" + prov + "_001.dat"
couponsfile = dataPath + "account_rechargeDB_" + datestr + "_" + prov + "_001.dat"
logger.debug(u"开始比对充值中心和卡券中心文件，账期[%s]， 省份[%s], 文件[%s][%s]!", datestr, prov, busifile, couponsfile)
cmpdiff(busifile, couponsfile)
else:
for provCode in commTools.getprovDict().keys(): # 遍历所有省份比对
busifile = ftppath + "account_recharge2coupons_" + datestr + "_" + provCode + "_001.dat"
couponsfile = dataPath + "account_rechargeDB_" + datestr + "_" + provCode + "_001.dat"
logger.debug(u"开始比对充值中心和卡券中心文件，账期[%s]， 省份[%s], 文件[%s][%s]!", datestr, provCode, busifile, couponsfile)
cmpdiff(busifile, couponsfile)
if (type == 'all' or type == 'flow'):
ftppath = getconfig.readcfg('flow_sftp', 'localpath') + "flow/" + datestr + "/"
dataPath = getconfig.readcfg('path', 'exportDataPath') + "flow/" + datestr + "/"
if prov != "999":
busifile = ftppath + "account_flow2coupons_" + datestr + "_" + prov + "_001.dat"
couponsfile = dataPath + "account_flowDB_" + datestr + "_" + prov + "_001.dat"
logger.debug(u"开始比对流量中心和卡券中心文件，账期[%s]， 省份[%s], 文件[%s][%s]!", datestr, prov, busifile, couponsfile)
cmpdiff(busifile, couponsfile)
else:
for provCode in commTools.getprovDict().keys(): # 遍历所有省份比对
busifile = ftppath + "account_flow2coupons_" + datestr + "_" + provCode + "_001.dat"
couponsfile = dataPath + "account_flowDB_" + datestr + "_" + provCode + "_001.dat"
logger.debug(u"开始比对流量中心和卡券中心文件，账期[%s]， 省份[%s], 文件[%s][%s]!", datestr, provCode, busifile, couponsfile)
cmpdiff(busifile, couponsfile)
if (type == 'all' or type == 'commodity'):
ftppath = getconfig.readcfg('commodity_sftp', 'localpath') + "commodity/" + datestr + "/"
dataPath = getconfig.readcfg('path', 'exportDataPath') + "commodity/" + datestr + "/"
busifile = ftppath + "account_commodity2coupons_" + datestr + "_" + "9999999" + "_001.dat"
couponsfile = dataPath + "account_commodityDB_" + datestr + "_" + "9999999" + "_001.dat"
logger.debug(u"开始比对商品中心和卡券中心文件，账期[%s]， 省份[%s], 文件[%s][%s]!", datestr, "9999999", busifile, couponsfile)
cmpdiff(busifile, couponsfile)
if (type == 'all' or type == 'package'):
ftppath = getconfig.readcfg('package_sftp', 'localpath') + "package/" + datestr + "/"
dataPath = getconfig.readcfg('path', 'exportDataPath') + "package/" + datestr + "/"
busifile = ftppath + "account_package2coupons_" + datestr + "_" + "999" + "_001.dat"
couponsfile = dataPath + "account_packageDB_" + datestr + "_" + "999" + "_001.dat"
logger.debug(u"开始比对商品中心和卡券中心文件，账期[%s]， 省份[%s], 文件[%s][%s]!", datestr, "999", busifile, couponsfile)
cmpdiff(busifile, couponsfile)