def batchInfoFilter(batchDict, newBatchFile, scanDate=today):
filteredDict = {}
for batch, info in batchDict.items():
try:
termType = info["termType"]
obtainBeginTime = info["obtainBeginTime"]
obtainEndTime = info["obtainEndTime"]
if not termType.isdigit():
log.warn(batch + "'s termType is not a number")
continue
if termType == '1':
expireTimestr = info["expireTime"]
expireTime = myDate.toDate("%Y%m%d", expireTimestr)
expireDay = info["expireDay"]
if dayAdd(scanDate, expireDay) == expireTime:
filteredDict[batch] = batchDict[batch]
else:
pass
elif termType == '2':
# expireTime is like '7'
expireTime = info["expireTime"]
obtainBeginDate = myDate.toDate("%Y%m%d", obtainBeginTime)
obtainEndDate = myDate.toDate("%Y%m%d", obtainEndTime)
expireDay = info["expireDay"]
if dayAdd(obtainBeginDate, expireTime) <= dayAdd(scanDate, expireDay) <= dayAdd(obtainEndDate,
expireTime):
filteredDict[batch] = batchDict[batch]
else:
pass
elif termType == '3':
obtainBeginTime = info["obtainBeginTime"]
obtainEndTime = info["obtainEndTime"]
# expireTime is like '7'
expireTime = info["expireTime"]
theLastDayOBT = dayAdd(monthAdd(myDate.toDate("%Y%m01", obtainBeginTime), 1), -1)
theLastDayOET = dayAdd(monthAdd(myDate.toDate("%Y%m01", obtainEndTime), 1), -1)
expireDay = info["expireDay"]
if dayAdd(theLastDayOBT, expireTime) <= dayAdd(scanDate, expireDay) <= dayAdd(theLastDayOET,
expireTime):
filteredDict[batch] = batchDict[batch]
else:
pass
else:
log.error("[batchInfoFilter]termType is wrong:[termType: %s ]" % termType)
except Exception as e:
log.error(e.__str__() + " wrong")
continue
with open(newBatchFile, 'a') as newfile:
for batch, info in filteredDict.items():
newfile.write(batch + "|" + "|".join(info.values()) + "\n")
return filteredDict
def generatePhoneDataCmd(filteredDict):
q = Queue.Queue()
for batch, info in filteredDict.items():
try:
expireDay = info['expireDay']
expireTime = myDate.toDate('%Y-%m-%d', dayAdd(today, expireDay))
obtainBeginTime = info["obtainBeginTime"]
obtainBeginDate = myDate.toDate("%Y%m%d", obtainBeginTime)
expire_time = '$8'
lifecycle_st = '$12'
bind_no = '$19'
tmp = obtainBeginDate
a=myDate.yesMon
b=myDate.toYm(tmp)
while myDate.toYm(tmp) <= myDate.toYm(yesterday):
year = myDate.toDate("%Y", tmp)
month = myDate.toDate("%m", tmp)
fsPath = hadoop.path('card', '%s', '%s', '*') % (year, month, )
cmd = """hadoop fs -cat %s |grep %s |awk -F '|' '{if(substr(%s,0,10)=="%s"&&%s=="8")print %s"|"%s}'""" % (
fsPath, batch, expire_time, expireTime, lifecycle_st, batch, bind_no)
q.put(cmd)
# log.info(cmd)
# log.info("%s" % fsPath)
tmp = monthAdd(tmp, 1)
# log.info("%s----%s--%s-----%s"%(tmp,obtainBeginDate,obtainEndDate,yesterday))
except Exception as e:
log.error(e)
log.info("[hadoop cmd's Queue count: %s]" % q.qsize())
return q
def genPhoDatSqlList(filteredDict):
sqlList = []
for batch, info in filteredDict.items():
expireDay = info['expireDay']
expireTime = info['expireTime']
obtainBeginTime = info['obtainBeginTime']
# obtainBeginTime = time.strptime(info["obtainBeginTime"], "%Y-%m-%d")
termType = info["termType"]
if termType == '1':
card_expire_time = myDate.toDate('%Y-%m-%d', expireTime)
obd = myDate.toDate("%d", obtainBeginTime)
obm = myDate.toDate("%m", obtainBeginTime)
oby = myDate.toDate("%Y", obtainBeginTime)
sql = "select batch_id,bind_no from pods.coupons_v2_td_pcard_info " \
"where batch_id ='%s' and to_date(expire_time)='%s' and lifecycle_st ='8' " \
"and yearstr >='%s' and monthstr >='%s' and daystr >='%s'" % (batch, card_expire_time, oby, obm, obd)
sqlList.append(sql)
elif termType == '2':
card_expire_time = myDate.toDate('%Y-%m-%d',
dayAdd(today, expireDay))
card_bind_time = dayAdd(card_expire_time, -(int(expireTime)))
oby = myDate.toDate("%Y", card_bind_time)
obm = myDate.toDate("%m", card_bind_time)
obd = myDate.toDate("%d", card_bind_time)
sql = "select batch_id,bind_no from pods.coupons_v2_td_pcard_info " \
"where batch_id ='%s' and to_date(expire_time)='%s' and lifecycle_st ='8' " \
"and yearstr ='%s' and monthstr ='%s' and daystr ='%s'" % (batch, card_expire_time, oby, obm, obd)
sqlList.append(sql)
elif termType == '3':
card_expire_time = myDate.toDate('%Y-%m-%d',
dayAdd(today, expireDay))
card_bind_time = monthAdd(card_expire_time, -(int(expireTime)))
oby = myDate.toDate("%Y", card_bind_time)
obm = myDate.toDate("%m", card_bind_time)
# obd= myDate.toDate("%d", card_bind_time)
sql = "select batch_id,bind_no from pods.coupons_v2_td_pcard_info " \
"where batch_id ='%s' and to_date(expire_time)='%s' and lifecycle_st ='8' " \
"and yearstr ='%s' and monthstr ='%s' " % (batch, card_expire_time, oby, obm)
sqlList.append(sql)
else:
log.info("[genPhoDatSqlList]termType Error:" + termType)
continue
return sqlList