def getInputDaysStr(sortedDays, nowDay, baseDir, N=7):
baseFiles = getFiles(baseDir)
baseFileDict = {}
for f in baseFiles:
baseFileDict[f] = 1
if len(sortedDays) > 0:
lastDay = TimeUtil.addDay(sortedDays[-1], 1)
if not baseDir.endswith("/"):
baseDir = baseDir + "/"
inputDir = ""
while lastDay <= nowDay:
if len(inputDir) > 0:
inputDir += ","
newDir = baseDir + lastDay
if newDir not in baseFileDict:
sys.stderr.write(newDir + "is not exists!\n")
continue
inputDir += newDir
lastDay = TimeUtil.addDay(lastDay, 1)
else:
inputDir = ""
for i in range(N):
if len(inputDir) > 0:
inputDir += ","
newDir = baseDir + datetime.strftime(
date.today() - timedelta(days=i + 1), "%Y-%m-%d")
if newDir not in baseFileDict:
sys.stderr.write(newDir + "is not exists!\n")
continue
inputDir += newDir
return inputDir
def getInputDaysStr(sortedDays,nowDay,baseDir,N=7):
baseFiles=getFiles(baseDir)
baseFileDict={}
for f in baseFiles:
baseFileDict[f]=1
if len(sortedDays) > 0:
lastDay=TimeUtil.addDay(sortedDays[-1],1)
if not baseDir.endswith("/"):
baseDir=baseDir+"/"
inputDir=""
while lastDay <= nowDay:
if len(inputDir) > 0:
inputDir+=","
newDir=baseDir+lastDay
if newDir not in baseFileDict:
sys.stderr.write(newDir+"is not exists!\n")
continue
inputDir+=newDir
lastDay=TimeUtil.addDay(lastDay,1)
else:
inputDir=""
for i in range(N):
if len(inputDir) > 0:
inputDir+=","
newDir=baseDir+datetime.strftime(date.today()-timedelta(days=i+1),"%Y-%m-%d")
if newDir not in baseFileDict:
sys.stderr.write(newDir+"is not exists!\n")
continue
inputDir+=newDir
return inputDir
def addDay(self, dayRet):
ldiv = TimeUtil.daysDiv(self.lastDay, dayRet.day)
sdiv = TimeUtil.daysDiv(self.firstDay, dayRet.day)
if (len(self.days) > 0 and ldiv <= 0) or sdiv > MaxDiv:
return
self.days.append(dayRet)
dayRet.div(self.firstDay, self.lastDay)
self.lastDay = dayRet.day
def cueInSeconds(self, seconds: float, lambdaFunction=None, action=None):
if lambdaFunction is not None:
action = SimpleAction(lambdaFunction)
action.triggerMicros = TimeUtil.getMicrosInFuture(seconds)
self.queued.append(action)
elif action is not None:
action.triggerMicros = TimeUtil.getMicrosInFuture(seconds)
self.queued.append(action)
def addDay(self,dayRet): ldiv=TimeUtil.daysDiv(self.lastDay,dayRet.day) sdiv=TimeUtil.daysDiv(self.firstDay,dayRet.day) if (len(self.days) > 0 and ldiv <= 0) or sdiv > MaxDiv: return self.days.append(dayRet) dayRet.div(self.firstDay,self.lastDay) self.lastDay=dayRet.day
def _writeMD5Csv():
md5_file = csv.reader(
open('D:\git_repo\DesignerWork\designer\Excel\Csv\Md5.csv', 'r'))
md5_list = __getMD5List()
for item in md5_list:
item[2] = TimeUtil._getCurrentTime()
csv_write = csv.writer(
open('D:\git_repo\DesignerWork\designer\Excel\Csv\Md5.csv',
'r+',
newline=''))
for item in md5_list:
item[2] = TimeUtil._getCurrentTime()
csv_write.writerow(item)
def getLatest(date):
lastKey = ""
lines = []
isLatest = False
latestTime = TimeUtil.toIntTime(date, "%Y-%m-%d")
for line in sys.stdin:
cols = line.strip().split("\t")
key = cols[0]
if lastKey == "":
lastKey = key
if lastKey != key:
if isLatest:
for line in lines:
print line
lastKey = key
lines = []
isLatest = False
k2 = cols[2]
if k2 not in StatisKeys:
try:
time = int(cols[4])
except:
sys.stderr.write(line + "\n")
if time > latestTime:
isLatest = True
lines.append(line.strip())
if lastKey != "":
if isLatest:
for line in lines:
print line
def ds(uuidFile, today):
regUserFile = "/home/zhangzhonghui/data/reg/regUser." + today
uuids = {}
for line in open(regUserFile):
cols = line.strip().split()
uuids[cols[0]] = [cols[1], ""]
for line in open(uuidFile):
cols = line.split()
if len(cols) < 4: continue
uuid = cols[0]
if uuid not in uuids: continue
day = cols[2]
if uuids[uuid][1] == "":
uuids[uuid][1] = day
else:
if uuids[uuid][1] > day:
uuids[uuid][1] = day
ds = {}
for uuid in uuids:
OK, lastDay = uuids[uuid]
if OK not in ds:
ds[OK] = {}
div = TimeUtil.daysDiv(lastDay, today)
if div not in ds[OK]:
ds[OK][div] = 1
else:
ds[OK][div] += 1
for OK in ds:
print OK, DictUtil.sum(ds[OK])
print ds[OK]
def getInputStrPlus(hdfsDir,inputBaseDir="/backup/CDA39907/001/",N=7): sortedDays=getDays(hdfsDir) nowPath=getInputDaysStr(sortedDays,TimeUtil.getYesterdayStr(),inputBaseDir,N) if len(sortedDays) > 0: return hdfsDir+"/"+sortedDays[-1]+","+nowPath else: return nowPath
def getLatest(date):
lastKey=""
lines=[]
isLatest=False
latestTime=TimeUtil.toIntTime(date,"%Y-%m-%d")
for line in sys.stdin:
cols=line.strip().split("\t")
key=cols[0]
if lastKey == "":
lastKey=key
if lastKey != key:
if isLatest:
for line in lines:
print line
lastKey=key
lines=[]
isLatest=False
k2=cols[2]
if k2 not in StatisKeys:
try:
time=int(cols[4])
except:
sys.stderr.write(line+"\n")
if time > latestTime:
isLatest=True
lines.append(line.strip())
if lastKey != "":
if isLatest:
for line in lines:
print line
def getLastDay(hdfsDir, N=7):
sortedDays = getDays(hdfsDir)
if len(sortedDays) > 0:
lastDay = TimeUtil.addDay(sortedDays[-1], 1)
return lastDay
else:
return datetime.strftime(date.today() - timedelta(days=N), "%Y-%m-%d")
def ds(uuidFile,today):
regUserFile="/home/zhangzhonghui/data/reg/regUser."+today
uuids={}
for line in open(regUserFile):
cols=line.strip().split()
uuids[cols[0]]=[cols[1],""]
for line in open(uuidFile):
cols=line.split()
if len(cols) < 4:continue
uuid=cols[0]
if uuid not in uuids:continue
day=cols[2]
if uuids[uuid][1] == "":
uuids[uuid][1]=day
else:
if uuids[uuid][1] > day:
uuids[uuid][1]=day
ds={}
for uuid in uuids:
OK,lastDay=uuids[uuid]
if OK not in ds:
ds[OK]={}
div=TimeUtil.daysDiv(lastDay,today)
if div not in ds[OK]:
ds[OK][div]=1
else:
ds[OK][div]+=1
for OK in ds:
print OK,DictUtil.sum(ds[OK])
print ds[OK]
def getLastDay(hdfsDir,N=7): sortedDays=getDays(hdfsDir) if len(sortedDays) > 0: lastDay=TimeUtil.addDay(sortedDays[-1],1) return lastDay else: return datetime.strftime(date.today()-timedelta(days=N),"%Y-%m-%d")
def outputMc(lastU,actions):
mc1={}
mc2={}
for action in actions:
method=action.method
if action.method == "search.getlist":
scene=column.getValue(action.para,"scene")
if scene != "":
method=method+"("+scene+")"
try:
date=TimeUtil.getDay(int(action.time))
except:
sys.stderr.write(action.time+"\n")
continue
if date not in ds:
continue
if ds[date] == 1:
if action.method in tops:
if method not in mc1:
mc1[method]=[0,0,0]
mc1[method][0]+=1
sat=satisDegree(action)
mc1[method][1]+=sat
if sat > mc1[method][2]:
mc1[method][2]=sat
else:
mc2[method]=1
addUser(mc1,mc2)
def __init__(self, durationSeconds, updateFunction, min, max):
self.startTime = None
self.endTime = None
self.durationMicros = TimeUtil.toMicros(durationSeconds)
self.updateFunction = updateFunction
self.min = min
self.max = max
def getInputStrPlus(hdfsDir, inputBaseDir="/backup/CDA39907/001/", N=7):
sortedDays = getDays(hdfsDir)
nowPath = getInputDaysStr(sortedDays, TimeUtil.getYesterdayStr(),
inputBaseDir, N)
if len(sortedDays) > 0:
return hdfsDir + "/" + sortedDays[-1] + "," + nowPath
else:
return nowPath
def trigger(self):
"""Trigger this event, and update the time at which
the next triggering should occur.
"""
self.triggerTime = self.findNextTriggerTime()
log.msg("Cron: running %s, next run at %s" %
(self.name, TimeUtil.formatLogDate(self.triggerTime)))
defer.maybeDeferred(self.callable).addCallback(self.triggerFinished)
def trigger(self):
"""Trigger this event, and update the time at which
the next triggering should occur.
"""
self.triggerTime = self.findNextTriggerTime()
log.msg("Cron: running %s, next run at %s" %
(self.name, TimeUtil.formatLogDate(self.triggerTime)))
defer.maybeDeferred(self.callable).addCallback(self.triggerFinished)
def pushHit():
import TimeUtil
import os
ds=TimeUtil.days("2014-07-01","2014-08-28")
name="pushview"
name="push_received"
for d in ds:
estr='echo '+'"'+d+'\t"$('+'hdfs dfs -text /online_logs/beijing/behaviour/'+name+'/'+d+'/'+name+'.log.'+d+".lzo 2> /dev/null | wc | awk '{print $1}')"
#print estr
os.system(estr)
def pushHit():
import TimeUtil
import os
ds = TimeUtil.days("2014-07-01", "2014-08-28")
name = "pushview"
name = "push_received"
for d in ds:
estr = 'echo ' + '"' + d + '\t"$(' + 'hdfs dfs -text /online_logs/beijing/behaviour/' + name + '/' + d + '/' + name + '.log.' + d + ".lzo 2> /dev/null | wc | awk '{print $1}')"
#print estr
os.system(estr)
def getOldDays(olds,statis): days=getTimeDict(statis,DaysKey) if len(days) > 0: return days for k2,(time,n) in olds.items(): day=TimeUtil.getDay(int(time)) if day not in days: days[day]=n else: days[day]+=n return days
def getOldDays(olds, statis):
days = getTimeDict(statis, DaysKey)
if len(days) > 0:
return days
for k2, (time, n) in olds.items():
day = TimeUtil.getDay(int(time))
if day not in days:
days[day] = n
else:
days[day] += n
return days
def number(text, keepInt=False):
if text == "":
raise InvalidXMLException("empty attribute value")
if ":" in text:
return float(TimeUtil.strdt(text))
text = text.strip()
if not keepInt:
return float(text)
if (all(c in "0123456789" for c in text)):
return int(text)
else:
return float(text)
def parseText(text):
text = text.lower()
if text == "":
raise InvalidXMLException("empty attribute value")
if text in ["true", "false"]:
return XValue(text == "true")
if ":" in text:
return float(TimeUtil.strdt(text))
if (all(c in "0123456789" for c in text)):
return XValue(int(text))
else:
return XValue(float(text))
def parseText(text):
text = text.lower()
if text == "":
raise InvalidXMLException("empty attribute value")
if text in ["true", "false"]:
return XValue(text == "true")
if ":" in text:
return float(TimeUtil.strdt(text))
if(all(c in "0123456789" for c in text)):
return XValue(int(text))
else:
return XValue(float(text))
def creatReport(reportList, urlName, fileName, reportName, cursor, db,
takeTimeLog):
createReportStartTime = time.time()
createTimeStr = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
print('\n{} : creating {} {} reports....'.format(createTimeStr,
len(reportList),
reportName))
cashStatementUrl = createReportUrl(urlName)
cashStatementFilePath = fileName + '/'
getReportAndCreate(reportList, cashStatementUrl, reportName,
cashStatementFilePath, cursor, db)
createReportTakesTime = TimeUtil.takesTimeLog(
round(time.time() - createReportStartTime), takeTimeLog)
createReportEndTime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
print('{} : all {} reports have created! {}'.format(
createReportEndTime, reportName, createReportTakesTime))
def acc():
lastKey = ""
ds = {}
for line in sys.stdin:
cols = line.strip().split("\t")
key = cols[0]
if lastKey == "":
lastKey = key
if lastKey != key:
output(lastKey, ds)
lastKey = key
ds = {}
t = cols[1]
action = cols[3]
if t not in ds:
ds[t] = {}
if action not in ds[t]:
ds[t][action] = [{}, {}, {}, {}]
statis, olds, news, days = ds[t][action]
k2 = cols[2]
if k2 in StatisKeys: #历史统计数据
statis[k2] = cols[4:]
elif len(cols) >= 6: #历史数据
time = int(cols[4])
n = int(cols[5])
olds[k2] = [time, n]
else: #新增数据
time = int(cols[4])
day = TimeUtil.getDay(time)
if day not in days:
days[day] = 1
else:
days[day] += 1
if k2 not in news:
news[k2] = [time, 1]
else:
news[k2][1] += 1
if time > news[k2][0]:
news[k2][0] = time
if lastKey != "":
output(lastKey, ds)
def count():
lastU = ""
lastUser = None
daysRet = {}
userRet = None
dayRet = None
for line in sys.stdin:
cols = line.strip().split("\t")
if len(cols) < 5:
continue
u = cols[0]
time = cols[1]
if time == "-1":
user = actionUserInfo.UserInfo.readUserInfo(cols)
if lastU == u:
lastUser.merge(user)
if lastU == "":
lastU = u
lastUser = user
if lastU != u:
addUserRet(daysRet, userRet)
lastU = u
lastUser = user
userRet = None
if time == "-1":
continue
method = column.detailMethod(cols[3], cols[4])
#if method.startswith("search.getlist"):
# sys.stderr.write(method+"\t"+cols[4]+"\n")
if cols[2] == "0":
day = TimeUtil.getDay(int(time))
if userRet == None:
userRet = UserRet(lastUser, day)
dayRet = DayRet(day)
userRet.addDay(dayRet)
if dayRet != None:
dayRet.addMethod(method)
if lastU != "":
addUserRet(daysRet, userRet)
for i in daysRet:
print str(i) + "\t" + str(daysRet[i])
def count():
lastU=""
lastUser=None
daysRet={}
userRet=None
dayRet=None
for line in sys.stdin:
cols=line.strip().split("\t")
if len(cols) < 5:
continue
u=cols[0]
time=cols[1]
if time == "-1":
user=actionUserInfo.UserInfo.readUserInfo(cols)
if lastU == u:
lastUser.merge(user)
if lastU == "":
lastU=u
lastUser=user
if lastU != u:
addUserRet(daysRet,userRet)
lastU=u
lastUser=user
userRet=None
if time == "-1":
continue
method=column.detailMethod(cols[3],cols[4])
#if method.startswith("search.getlist"):
# sys.stderr.write(method+"\t"+cols[4]+"\n")
if cols[2] == "0":
day=TimeUtil.getDay(int(time))
if userRet == None:
userRet=UserRet(lastUser,day)
dayRet=DayRet(day)
userRet.addDay(dayRet)
if dayRet != None:
dayRet.addMethod(method)
if lastU != "":
addUserRet(daysRet,userRet)
for i in daysRet:
print str(i)+"\t"+str(daysRet[i])
def acc():
lastKey=""
ds={}
for line in sys.stdin:
cols=line.strip().split("\t")
key=cols[0]
if lastKey == "":
lastKey=key
if lastKey != key:
output(lastKey,ds)
lastKey=key
ds={}
t=cols[1]
action=cols[3]
if t not in ds:
ds[t]={}
if action not in ds[t]:
ds[t][action]=[{},{},{},{}]
statis,olds,news,days=ds[t][action]
k2=cols[2]
if k2 in StatisKeys: #历史统计数据
statis[k2]=cols[4:]
elif len(cols) >= 6: #历史数据
time=int(cols[4])
n=int(cols[5])
olds[k2]=[time,n]
else: #新增数据
time=int(cols[4])
day=TimeUtil.getDay(time)
if day not in days:
days[day]=1
else:
days[day]+=1
if k2 not in news:
news[k2]=[time,1]
else:
news[k2][1]+=1
if time > news[k2][0]:
news[k2][0]=time
if lastKey != "":
output(lastKey,ds)
def start(self):
self.startTime = TimeUtil.nowMicros()
self.endTime = self.startTime + self.durationMicros
def div(self,firstDay,lastDay): self.lastDiv=TimeUtil.daysDiv(lastDay,self.day) #与上一次的访问间隔 self.firstDiv=TimeUtil.daysDiv(firstDay,self.day) #与第一次的访问间隔
def update(self):
now = TimeUtil.nowMicros()
value = MathUtil.map(now, self.startTime, self.endTime, self.min,
self.max)
self.updateFunction(value)
def isTriggered(self):
return TimeUtil.isInThePast(self.triggerMicros)
def readMall(day,end):
db=DB()
db.connect("haodou_mall")
cursor=db.execute("select og.UserId,og.GoodsId,max(ol.CreateTime),ol.OrderId from OrderBase as ob,OrderGoods as og, OrderLog as ol where og.OrderId=ob.OrderId and ob.OrderId=ol.OrderId and ob.OrderStatus=70 and ol.CreateTime >='"+day+" 00:00:00' and ol.CreateTime <= '"+end+" 23:59:59' group by og.UserId,og.GoodsId,ol.OrderId;")
ret=cursor.fetchall()
for r in ret:
#sys.stdout.write(str(r[2])+"\n")
nutPrint("uid-"+str(r[0]),"buy","goods-%d"%(r[1]),str(int(time.mktime(r[2].timetuple()))))
if __name__=="__main__":
if len(sys.argv) >= 4 and sys.argv[1] == "acc":
hdfsDir=sys.argv[2]
N=int(sys.argv[3])
sortedDays=hdfsFile.getDays(hdfsDir)
if len(sortedDays) > 0:
lastDay=TimeUtil.addDay(sortedDays[-1],1)
else:
lastDay=datetime.strftime(date.today()-timedelta(days=N),"%Y-%m-%d")
end=datetime.strftime(date.today()-timedelta(days=1),"%Y-%m-%d")
sys.stderr.write("lastDay for dbNut:"+lastDay+"\n")
sys.stderr.write("end for dbNut:"+end+"\n")
readComment(lastDay,end)
readMall(lastDay,end)
elif len(sys.argv) >= 3:
readComment(sys.argv[1],sys.argv[2])
readMall(sys.argv[1],sys.argv[2])
elif len(sys.argv) >= 2:
N=int(sys.argv[1])
end=datetime.strftime(date.today()-timedelta(days=1),"%Y-%m-%d")
day=datetime.strftime(date.today()-timedelta(days=N),"%Y-%m-%d")
print N,day,end
def isDone(self):
return TimeUtil.nowMicros() > self.endTime
def getInputStr(hdfsDir,inputBaseDir="/backup/CDA39907/001/",N=7): sortedDays=getDays(hdfsDir) return getInputDaysStr(sortedDays,TimeUtil.getYesterdayStr(),inputBaseDir,N)
def number(text):
if ":" in text:
return float(TimeUtil.strdt(text))
else:
return float(text)
import sys
sys.path.append("../util")
import TimeUtil
print TimeUtil.addDay(sys.argv[1],int(sys.argv[2]))
def div(self, firstDay, lastDay):
self.lastDiv = TimeUtil.daysDiv(lastDay, self.day) #与上一次的访问间隔
self.firstDiv = TimeUtil.daysDiv(firstDay, self.day) #与第一次的访问间隔
def number(text):
if ":" in text:
return float(TimeUtil.strdt(text))
else:
return float(text)
createAllStocksFileSimple(stockList, jsonPath)
# createAllStocksFile(stockList, allStockInfoDateList, jsonPath)
loadList = stockSymbols
loadList = stockSymbols[0:1] # 测试，只要一个
stockLen = len(loadList)
print('there are {} stockSymbols :'.format(stockLen))
creatReport(loadList, 'lrb', incomeFileName, 'income', cursor, db, takeTimeLog)
time.sleep(1)
creatReport(loadList, 'zcfzb', balanceSheetFolderName, 'balance', cursor, db,
takeTimeLog)
time.sleep(1)
creatReport(loadList, 'xjllb', cashStatementFileName, 'cash', cursor, db,
takeTimeLog)
StockInfo.insertDate(allStockInfoDateList, cursor, db)
allStockInfo = getAllStocksFileSimple(stockList)
# jsonFileDic["allStockInfo"] = allStockInfo
JsonCreater.createJson(jsonFileDic, jsonPath + "report_title.json")
startTimeLocal = time.localtime(startTime)
print('\napplication starts at ',
time.strftime("%Y-%m-%d %H:%M:%S", startTimeLocal))
print('application ends at ',
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
print(TimeUtil.takesTimeLog(round(time.time() - startTime), takeTimeLog))
def getInputStr(hdfsDir, inputBaseDir="/backup/CDA39907/001/", N=7):
sortedDays = getDays(hdfsDir)
return getInputDaysStr(sortedDays, TimeUtil.getYesterdayStr(),
inputBaseDir, N)
import sys
sys.path.append("../util")
import TimeUtil
print TimeUtil.addDay(sys.argv[1], int(sys.argv[2]))
