def batchInfoFilter(batchDict, newBatchFile, scanDate=today): filteredDict = {} for batch, info in batchDict.items(): try: termType = info["termType"] obtainBeginTime = info["obtainBeginTime"] obtainEndTime = info["obtainEndTime"] if not termType.isdigit(): log.warn(batch + "'s termType is not a number") continue if termType == '1': expireTimestr = info["expireTime"] expireTime = myDate.toDate("%Y%m%d", expireTimestr) expireDay = info["expireDay"] if dayAdd(scanDate, expireDay) == expireTime: filteredDict[batch] = batchDict[batch] else: pass elif termType == '2': # expireTime is like '7' expireTime = info["expireTime"] obtainBeginDate = myDate.toDate("%Y%m%d", obtainBeginTime) obtainEndDate = myDate.toDate("%Y%m%d", obtainEndTime) expireDay = info["expireDay"] if dayAdd(obtainBeginDate, expireTime) <= dayAdd(scanDate, expireDay) <= dayAdd(obtainEndDate, expireTime): filteredDict[batch] = batchDict[batch] else: pass elif termType == '3': obtainBeginTime = info["obtainBeginTime"] obtainEndTime = info["obtainEndTime"] # expireTime is like '7' expireTime = info["expireTime"] theLastDayOBT = dayAdd(monthAdd(myDate.toDate("%Y%m01", obtainBeginTime), 1), -1) theLastDayOET = dayAdd(monthAdd(myDate.toDate("%Y%m01", obtainEndTime), 1), -1) expireDay = info["expireDay"] if dayAdd(theLastDayOBT, expireTime) <= dayAdd(scanDate, expireDay) <= dayAdd(theLastDayOET, expireTime): filteredDict[batch] = batchDict[batch] else: pass else: log.error("[batchInfoFilter]termType is wrong:[termType: %s ]" % termType) except Exception as e: log.error(e.__str__() + " wrong") continue with open(newBatchFile, 'a') as newfile: for batch, info in filteredDict.items(): newfile.write(batch + "|" + "|".join(info.values()) + "\n") return filteredDict
def generatePhoneDataCmd(filteredDict): q = Queue.Queue() for batch, info in filteredDict.items(): try: expireDay = info['expireDay'] expireTime = myDate.toDate('%Y-%m-%d', dayAdd(today, expireDay)) obtainBeginTime = info["obtainBeginTime"] obtainBeginDate = myDate.toDate("%Y%m%d", obtainBeginTime) expire_time = '$8' lifecycle_st = '$12' bind_no = '$19' tmp = obtainBeginDate a=myDate.yesMon b=myDate.toYm(tmp) while myDate.toYm(tmp) <= myDate.toYm(yesterday): year = myDate.toDate("%Y", tmp) month = myDate.toDate("%m", tmp) fsPath = hadoop.path('card', '%s', '%s', '*') % (year, month, ) cmd = """hadoop fs -cat %s |grep %s |awk -F '|' '{if(substr(%s,0,10)=="%s"&&%s=="8")print %s"|"%s}'""" % ( fsPath, batch, expire_time, expireTime, lifecycle_st, batch, bind_no) q.put(cmd) # log.info(cmd) # log.info("%s" % fsPath) tmp = monthAdd(tmp, 1) # log.info("%s----%s--%s-----%s"%(tmp,obtainBeginDate,obtainEndDate,yesterday)) except Exception as e: log.error(e) log.info("[hadoop cmd's Queue count: %s]" % q.qsize()) return q
def genPhoDatSqlList(filteredDict): sqlList = [] for batch, info in filteredDict.items(): expireDay = info['expireDay'] expireTime = info['expireTime'] obtainBeginTime = info['obtainBeginTime'] # obtainBeginTime = time.strptime(info["obtainBeginTime"], "%Y-%m-%d") termType = info["termType"] if termType == '1': card_expire_time = myDate.toDate('%Y-%m-%d', expireTime) obd = myDate.toDate("%d", obtainBeginTime) obm = myDate.toDate("%m", obtainBeginTime) oby = myDate.toDate("%Y", obtainBeginTime) sql = "select batch_id,bind_no from pods.coupons_v2_td_pcard_info " \ "where batch_id ='%s' and to_date(expire_time)='%s' and lifecycle_st ='8' " \ "and yearstr >='%s' and monthstr >='%s' and daystr >='%s'" % (batch, card_expire_time, oby, obm, obd) sqlList.append(sql) elif termType == '2': card_expire_time = myDate.toDate('%Y-%m-%d', dayAdd(today, expireDay)) card_bind_time = dayAdd(card_expire_time, -(int(expireTime))) oby = myDate.toDate("%Y", card_bind_time) obm = myDate.toDate("%m", card_bind_time) obd = myDate.toDate("%d", card_bind_time) sql = "select batch_id,bind_no from pods.coupons_v2_td_pcard_info " \ "where batch_id ='%s' and to_date(expire_time)='%s' and lifecycle_st ='8' " \ "and yearstr ='%s' and monthstr ='%s' and daystr ='%s'" % (batch, card_expire_time, oby, obm, obd) sqlList.append(sql) elif termType == '3': card_expire_time = myDate.toDate('%Y-%m-%d', dayAdd(today, expireDay)) card_bind_time = monthAdd(card_expire_time, -(int(expireTime))) oby = myDate.toDate("%Y", card_bind_time) obm = myDate.toDate("%m", card_bind_time) # obd= myDate.toDate("%d", card_bind_time) sql = "select batch_id,bind_no from pods.coupons_v2_td_pcard_info " \ "where batch_id ='%s' and to_date(expire_time)='%s' and lifecycle_st ='8' " \ "and yearstr ='%s' and monthstr ='%s' " % (batch, card_expire_time, oby, obm) sqlList.append(sql) else: log.info("[genPhoDatSqlList]termType Error:" + termType) continue return sqlList