def getInputDaysStr(sortedDays, nowDay, baseDir, N=7): baseFiles = getFiles(baseDir) baseFileDict = {} for f in baseFiles: baseFileDict[f] = 1 if len(sortedDays) > 0: lastDay = TimeUtil.addDay(sortedDays[-1], 1) if not baseDir.endswith("/"): baseDir = baseDir + "/" inputDir = "" while lastDay <= nowDay: if len(inputDir) > 0: inputDir += "," newDir = baseDir + lastDay if newDir not in baseFileDict: sys.stderr.write(newDir + "is not exists!\n") continue inputDir += newDir lastDay = TimeUtil.addDay(lastDay, 1) else: inputDir = "" for i in range(N): if len(inputDir) > 0: inputDir += "," newDir = baseDir + datetime.strftime( date.today() - timedelta(days=i + 1), "%Y-%m-%d") if newDir not in baseFileDict: sys.stderr.write(newDir + "is not exists!\n") continue inputDir += newDir return inputDir
def getInputDaysStr(sortedDays,nowDay,baseDir,N=7): baseFiles=getFiles(baseDir) baseFileDict={} for f in baseFiles: baseFileDict[f]=1 if len(sortedDays) > 0: lastDay=TimeUtil.addDay(sortedDays[-1],1) if not baseDir.endswith("/"): baseDir=baseDir+"/" inputDir="" while lastDay <= nowDay: if len(inputDir) > 0: inputDir+="," newDir=baseDir+lastDay if newDir not in baseFileDict: sys.stderr.write(newDir+"is not exists!\n") continue inputDir+=newDir lastDay=TimeUtil.addDay(lastDay,1) else: inputDir="" for i in range(N): if len(inputDir) > 0: inputDir+="," newDir=baseDir+datetime.strftime(date.today()-timedelta(days=i+1),"%Y-%m-%d") if newDir not in baseFileDict: sys.stderr.write(newDir+"is not exists!\n") continue inputDir+=newDir return inputDir
def addDay(self, dayRet): ldiv = TimeUtil.daysDiv(self.lastDay, dayRet.day) sdiv = TimeUtil.daysDiv(self.firstDay, dayRet.day) if (len(self.days) > 0 and ldiv <= 0) or sdiv > MaxDiv: return self.days.append(dayRet) dayRet.div(self.firstDay, self.lastDay) self.lastDay = dayRet.day
def cueInSeconds(self, seconds: float, lambdaFunction=None, action=None): if lambdaFunction is not None: action = SimpleAction(lambdaFunction) action.triggerMicros = TimeUtil.getMicrosInFuture(seconds) self.queued.append(action) elif action is not None: action.triggerMicros = TimeUtil.getMicrosInFuture(seconds) self.queued.append(action)
def addDay(self,dayRet): ldiv=TimeUtil.daysDiv(self.lastDay,dayRet.day) sdiv=TimeUtil.daysDiv(self.firstDay,dayRet.day) if (len(self.days) > 0 and ldiv <= 0) or sdiv > MaxDiv: return self.days.append(dayRet) dayRet.div(self.firstDay,self.lastDay) self.lastDay=dayRet.day
def _writeMD5Csv(): md5_file = csv.reader( open('D:\git_repo\DesignerWork\designer\Excel\Csv\Md5.csv', 'r')) md5_list = __getMD5List() for item in md5_list: item[2] = TimeUtil._getCurrentTime() csv_write = csv.writer( open('D:\git_repo\DesignerWork\designer\Excel\Csv\Md5.csv', 'r+', newline='')) for item in md5_list: item[2] = TimeUtil._getCurrentTime() csv_write.writerow(item)
def getLatest(date): lastKey = "" lines = [] isLatest = False latestTime = TimeUtil.toIntTime(date, "%Y-%m-%d") for line in sys.stdin: cols = line.strip().split("\t") key = cols[0] if lastKey == "": lastKey = key if lastKey != key: if isLatest: for line in lines: print line lastKey = key lines = [] isLatest = False k2 = cols[2] if k2 not in StatisKeys: try: time = int(cols[4]) except: sys.stderr.write(line + "\n") if time > latestTime: isLatest = True lines.append(line.strip()) if lastKey != "": if isLatest: for line in lines: print line
def ds(uuidFile, today): regUserFile = "/home/zhangzhonghui/data/reg/regUser." + today uuids = {} for line in open(regUserFile): cols = line.strip().split() uuids[cols[0]] = [cols[1], ""] for line in open(uuidFile): cols = line.split() if len(cols) < 4: continue uuid = cols[0] if uuid not in uuids: continue day = cols[2] if uuids[uuid][1] == "": uuids[uuid][1] = day else: if uuids[uuid][1] > day: uuids[uuid][1] = day ds = {} for uuid in uuids: OK, lastDay = uuids[uuid] if OK not in ds: ds[OK] = {} div = TimeUtil.daysDiv(lastDay, today) if div not in ds[OK]: ds[OK][div] = 1 else: ds[OK][div] += 1 for OK in ds: print OK, DictUtil.sum(ds[OK]) print ds[OK]
def getInputStrPlus(hdfsDir,inputBaseDir="/backup/CDA39907/001/",N=7): sortedDays=getDays(hdfsDir) nowPath=getInputDaysStr(sortedDays,TimeUtil.getYesterdayStr(),inputBaseDir,N) if len(sortedDays) > 0: return hdfsDir+"/"+sortedDays[-1]+","+nowPath else: return nowPath
def getLatest(date): lastKey="" lines=[] isLatest=False latestTime=TimeUtil.toIntTime(date,"%Y-%m-%d") for line in sys.stdin: cols=line.strip().split("\t") key=cols[0] if lastKey == "": lastKey=key if lastKey != key: if isLatest: for line in lines: print line lastKey=key lines=[] isLatest=False k2=cols[2] if k2 not in StatisKeys: try: time=int(cols[4]) except: sys.stderr.write(line+"\n") if time > latestTime: isLatest=True lines.append(line.strip()) if lastKey != "": if isLatest: for line in lines: print line
def getLastDay(hdfsDir, N=7): sortedDays = getDays(hdfsDir) if len(sortedDays) > 0: lastDay = TimeUtil.addDay(sortedDays[-1], 1) return lastDay else: return datetime.strftime(date.today() - timedelta(days=N), "%Y-%m-%d")
def ds(uuidFile,today): regUserFile="/home/zhangzhonghui/data/reg/regUser."+today uuids={} for line in open(regUserFile): cols=line.strip().split() uuids[cols[0]]=[cols[1],""] for line in open(uuidFile): cols=line.split() if len(cols) < 4:continue uuid=cols[0] if uuid not in uuids:continue day=cols[2] if uuids[uuid][1] == "": uuids[uuid][1]=day else: if uuids[uuid][1] > day: uuids[uuid][1]=day ds={} for uuid in uuids: OK,lastDay=uuids[uuid] if OK not in ds: ds[OK]={} div=TimeUtil.daysDiv(lastDay,today) if div not in ds[OK]: ds[OK][div]=1 else: ds[OK][div]+=1 for OK in ds: print OK,DictUtil.sum(ds[OK]) print ds[OK]
def getLastDay(hdfsDir,N=7): sortedDays=getDays(hdfsDir) if len(sortedDays) > 0: lastDay=TimeUtil.addDay(sortedDays[-1],1) return lastDay else: return datetime.strftime(date.today()-timedelta(days=N),"%Y-%m-%d")
def outputMc(lastU,actions): mc1={} mc2={} for action in actions: method=action.method if action.method == "search.getlist": scene=column.getValue(action.para,"scene") if scene != "": method=method+"("+scene+")" try: date=TimeUtil.getDay(int(action.time)) except: sys.stderr.write(action.time+"\n") continue if date not in ds: continue if ds[date] == 1: if action.method in tops: if method not in mc1: mc1[method]=[0,0,0] mc1[method][0]+=1 sat=satisDegree(action) mc1[method][1]+=sat if sat > mc1[method][2]: mc1[method][2]=sat else: mc2[method]=1 addUser(mc1,mc2)
def __init__(self, durationSeconds, updateFunction, min, max): self.startTime = None self.endTime = None self.durationMicros = TimeUtil.toMicros(durationSeconds) self.updateFunction = updateFunction self.min = min self.max = max
def getInputStrPlus(hdfsDir, inputBaseDir="/backup/CDA39907/001/", N=7): sortedDays = getDays(hdfsDir) nowPath = getInputDaysStr(sortedDays, TimeUtil.getYesterdayStr(), inputBaseDir, N) if len(sortedDays) > 0: return hdfsDir + "/" + sortedDays[-1] + "," + nowPath else: return nowPath
def trigger(self): """Trigger this event, and update the time at which the next triggering should occur. """ self.triggerTime = self.findNextTriggerTime() log.msg("Cron: running %s, next run at %s" % (self.name, TimeUtil.formatLogDate(self.triggerTime))) defer.maybeDeferred(self.callable).addCallback(self.triggerFinished)
def pushHit(): import TimeUtil import os ds=TimeUtil.days("2014-07-01","2014-08-28") name="pushview" name="push_received" for d in ds: estr='echo '+'"'+d+'\t"$('+'hdfs dfs -text /online_logs/beijing/behaviour/'+name+'/'+d+'/'+name+'.log.'+d+".lzo 2> /dev/null | wc | awk '{print $1}')" #print estr os.system(estr)
def pushHit(): import TimeUtil import os ds = TimeUtil.days("2014-07-01", "2014-08-28") name = "pushview" name = "push_received" for d in ds: estr = 'echo ' + '"' + d + '\t"$(' + 'hdfs dfs -text /online_logs/beijing/behaviour/' + name + '/' + d + '/' + name + '.log.' + d + ".lzo 2> /dev/null | wc | awk '{print $1}')" #print estr os.system(estr)
def getOldDays(olds,statis): days=getTimeDict(statis,DaysKey) if len(days) > 0: return days for k2,(time,n) in olds.items(): day=TimeUtil.getDay(int(time)) if day not in days: days[day]=n else: days[day]+=n return days
def getOldDays(olds, statis): days = getTimeDict(statis, DaysKey) if len(days) > 0: return days for k2, (time, n) in olds.items(): day = TimeUtil.getDay(int(time)) if day not in days: days[day] = n else: days[day] += n return days
def number(text, keepInt=False): if text == "": raise InvalidXMLException("empty attribute value") if ":" in text: return float(TimeUtil.strdt(text)) text = text.strip() if not keepInt: return float(text) if (all(c in "0123456789" for c in text)): return int(text) else: return float(text)
def parseText(text): text = text.lower() if text == "": raise InvalidXMLException("empty attribute value") if text in ["true", "false"]: return XValue(text == "true") if ":" in text: return float(TimeUtil.strdt(text)) if (all(c in "0123456789" for c in text)): return XValue(int(text)) else: return XValue(float(text))
def parseText(text): text = text.lower() if text == "": raise InvalidXMLException("empty attribute value") if text in ["true", "false"]: return XValue(text == "true") if ":" in text: return float(TimeUtil.strdt(text)) if(all(c in "0123456789" for c in text)): return XValue(int(text)) else: return XValue(float(text))
def creatReport(reportList, urlName, fileName, reportName, cursor, db, takeTimeLog): createReportStartTime = time.time() createTimeStr = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) print('\n{} : creating {} {} reports....'.format(createTimeStr, len(reportList), reportName)) cashStatementUrl = createReportUrl(urlName) cashStatementFilePath = fileName + '/' getReportAndCreate(reportList, cashStatementUrl, reportName, cashStatementFilePath, cursor, db) createReportTakesTime = TimeUtil.takesTimeLog( round(time.time() - createReportStartTime), takeTimeLog) createReportEndTime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) print('{} : all {} reports have created! {}'.format( createReportEndTime, reportName, createReportTakesTime))
def acc(): lastKey = "" ds = {} for line in sys.stdin: cols = line.strip().split("\t") key = cols[0] if lastKey == "": lastKey = key if lastKey != key: output(lastKey, ds) lastKey = key ds = {} t = cols[1] action = cols[3] if t not in ds: ds[t] = {} if action not in ds[t]: ds[t][action] = [{}, {}, {}, {}] statis, olds, news, days = ds[t][action] k2 = cols[2] if k2 in StatisKeys: #历史统计数据 statis[k2] = cols[4:] elif len(cols) >= 6: #历史数据 time = int(cols[4]) n = int(cols[5]) olds[k2] = [time, n] else: #新增数据 time = int(cols[4]) day = TimeUtil.getDay(time) if day not in days: days[day] = 1 else: days[day] += 1 if k2 not in news: news[k2] = [time, 1] else: news[k2][1] += 1 if time > news[k2][0]: news[k2][0] = time if lastKey != "": output(lastKey, ds)
def count(): lastU = "" lastUser = None daysRet = {} userRet = None dayRet = None for line in sys.stdin: cols = line.strip().split("\t") if len(cols) < 5: continue u = cols[0] time = cols[1] if time == "-1": user = actionUserInfo.UserInfo.readUserInfo(cols) if lastU == u: lastUser.merge(user) if lastU == "": lastU = u lastUser = user if lastU != u: addUserRet(daysRet, userRet) lastU = u lastUser = user userRet = None if time == "-1": continue method = column.detailMethod(cols[3], cols[4]) #if method.startswith("search.getlist"): # sys.stderr.write(method+"\t"+cols[4]+"\n") if cols[2] == "0": day = TimeUtil.getDay(int(time)) if userRet == None: userRet = UserRet(lastUser, day) dayRet = DayRet(day) userRet.addDay(dayRet) if dayRet != None: dayRet.addMethod(method) if lastU != "": addUserRet(daysRet, userRet) for i in daysRet: print str(i) + "\t" + str(daysRet[i])
def count(): lastU="" lastUser=None daysRet={} userRet=None dayRet=None for line in sys.stdin: cols=line.strip().split("\t") if len(cols) < 5: continue u=cols[0] time=cols[1] if time == "-1": user=actionUserInfo.UserInfo.readUserInfo(cols) if lastU == u: lastUser.merge(user) if lastU == "": lastU=u lastUser=user if lastU != u: addUserRet(daysRet,userRet) lastU=u lastUser=user userRet=None if time == "-1": continue method=column.detailMethod(cols[3],cols[4]) #if method.startswith("search.getlist"): # sys.stderr.write(method+"\t"+cols[4]+"\n") if cols[2] == "0": day=TimeUtil.getDay(int(time)) if userRet == None: userRet=UserRet(lastUser,day) dayRet=DayRet(day) userRet.addDay(dayRet) if dayRet != None: dayRet.addMethod(method) if lastU != "": addUserRet(daysRet,userRet) for i in daysRet: print str(i)+"\t"+str(daysRet[i])
def acc(): lastKey="" ds={} for line in sys.stdin: cols=line.strip().split("\t") key=cols[0] if lastKey == "": lastKey=key if lastKey != key: output(lastKey,ds) lastKey=key ds={} t=cols[1] action=cols[3] if t not in ds: ds[t]={} if action not in ds[t]: ds[t][action]=[{},{},{},{}] statis,olds,news,days=ds[t][action] k2=cols[2] if k2 in StatisKeys: #历史统计数据 statis[k2]=cols[4:] elif len(cols) >= 6: #历史数据 time=int(cols[4]) n=int(cols[5]) olds[k2]=[time,n] else: #新增数据 time=int(cols[4]) day=TimeUtil.getDay(time) if day not in days: days[day]=1 else: days[day]+=1 if k2 not in news: news[k2]=[time,1] else: news[k2][1]+=1 if time > news[k2][0]: news[k2][0]=time if lastKey != "": output(lastKey,ds)
def start(self): self.startTime = TimeUtil.nowMicros() self.endTime = self.startTime + self.durationMicros
def div(self,firstDay,lastDay): self.lastDiv=TimeUtil.daysDiv(lastDay,self.day) #与上一次的访问间隔 self.firstDiv=TimeUtil.daysDiv(firstDay,self.day) #与第一次的访问间隔
def update(self): now = TimeUtil.nowMicros() value = MathUtil.map(now, self.startTime, self.endTime, self.min, self.max) self.updateFunction(value)
def isTriggered(self): return TimeUtil.isInThePast(self.triggerMicros)
def readMall(day,end): db=DB() db.connect("haodou_mall") cursor=db.execute("select og.UserId,og.GoodsId,max(ol.CreateTime),ol.OrderId from OrderBase as ob,OrderGoods as og, OrderLog as ol where og.OrderId=ob.OrderId and ob.OrderId=ol.OrderId and ob.OrderStatus=70 and ol.CreateTime >='"+day+" 00:00:00' and ol.CreateTime <= '"+end+" 23:59:59' group by og.UserId,og.GoodsId,ol.OrderId;") ret=cursor.fetchall() for r in ret: #sys.stdout.write(str(r[2])+"\n") nutPrint("uid-"+str(r[0]),"buy","goods-%d"%(r[1]),str(int(time.mktime(r[2].timetuple())))) if __name__=="__main__": if len(sys.argv) >= 4 and sys.argv[1] == "acc": hdfsDir=sys.argv[2] N=int(sys.argv[3]) sortedDays=hdfsFile.getDays(hdfsDir) if len(sortedDays) > 0: lastDay=TimeUtil.addDay(sortedDays[-1],1) else: lastDay=datetime.strftime(date.today()-timedelta(days=N),"%Y-%m-%d") end=datetime.strftime(date.today()-timedelta(days=1),"%Y-%m-%d") sys.stderr.write("lastDay for dbNut:"+lastDay+"\n") sys.stderr.write("end for dbNut:"+end+"\n") readComment(lastDay,end) readMall(lastDay,end) elif len(sys.argv) >= 3: readComment(sys.argv[1],sys.argv[2]) readMall(sys.argv[1],sys.argv[2]) elif len(sys.argv) >= 2: N=int(sys.argv[1]) end=datetime.strftime(date.today()-timedelta(days=1),"%Y-%m-%d") day=datetime.strftime(date.today()-timedelta(days=N),"%Y-%m-%d") print N,day,end
def isDone(self): return TimeUtil.nowMicros() > self.endTime
def getInputStr(hdfsDir,inputBaseDir="/backup/CDA39907/001/",N=7): sortedDays=getDays(hdfsDir) return getInputDaysStr(sortedDays,TimeUtil.getYesterdayStr(),inputBaseDir,N)
def number(text): if ":" in text: return float(TimeUtil.strdt(text)) else: return float(text)
import sys sys.path.append("../util") import TimeUtil print TimeUtil.addDay(sys.argv[1],int(sys.argv[2]))
def div(self, firstDay, lastDay): self.lastDiv = TimeUtil.daysDiv(lastDay, self.day) #与上一次的访问间隔 self.firstDiv = TimeUtil.daysDiv(firstDay, self.day) #与第一次的访问间隔
createAllStocksFileSimple(stockList, jsonPath) # createAllStocksFile(stockList, allStockInfoDateList, jsonPath) loadList = stockSymbols loadList = stockSymbols[0:1] # 测试,只要一个 stockLen = len(loadList) print('there are {} stockSymbols :'.format(stockLen)) creatReport(loadList, 'lrb', incomeFileName, 'income', cursor, db, takeTimeLog) time.sleep(1) creatReport(loadList, 'zcfzb', balanceSheetFolderName, 'balance', cursor, db, takeTimeLog) time.sleep(1) creatReport(loadList, 'xjllb', cashStatementFileName, 'cash', cursor, db, takeTimeLog) StockInfo.insertDate(allStockInfoDateList, cursor, db) allStockInfo = getAllStocksFileSimple(stockList) # jsonFileDic["allStockInfo"] = allStockInfo JsonCreater.createJson(jsonFileDic, jsonPath + "report_title.json") startTimeLocal = time.localtime(startTime) print('\napplication starts at ', time.strftime("%Y-%m-%d %H:%M:%S", startTimeLocal)) print('application ends at ', time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) print(TimeUtil.takesTimeLog(round(time.time() - startTime), takeTimeLog))
def getInputStr(hdfsDir, inputBaseDir="/backup/CDA39907/001/", N=7): sortedDays = getDays(hdfsDir) return getInputDaysStr(sortedDays, TimeUtil.getYesterdayStr(), inputBaseDir, N)
import sys sys.path.append("../util") import TimeUtil print TimeUtil.addDay(sys.argv[1], int(sys.argv[2]))