def get_user_file(start, end): global ufCount ufCount = 0 formType = 'UserFile' dateStr = start.strftime('%Y-%m-%d') rc = checkWriteState(dateStr, formType) if rc: print 'already batch inserted' return print '开始存储 ' + dateStr + ' ' + formType + ' 的数据' #判断今天是不是已经搞定了 ufObj = Object.extend('UserFile') ufQuery = Query(ufObj) ufQuery.greater_than_or_equal_to('createdAt',start) ufQuery.less_than_or_equal_to('createdAt',end) def uf_call_back(lst): global ufCount writeList = [] for item in lst: ufMap = {} ufMap['openId'] = item.get('openId') ufMap['fileType'] = item.get('fileType') ufMap['srcId'] = item.id ufMap['createdAt'] = item.created_at ufMap['updatedAt'] = item.updated_at writeList.append(ufMap) ufCount = ufCount + len(writeList) if len(writeList) > 0: loadUserFile(writeList) map_query(ufQuery, 1000, uf_call_back) print '本次一共存储了' + str(ufCount) + '条纪录' #一共存储了多少 finishLoadData(dateStr, formType, ufCount)
def get_daily_checkin(start, end): global dcCount dcCount = 0 formType = 'DailyCheckIn' dateStr = start.strftime('%Y-%m-%d') rc = checkWriteState(dateStr, formType) if rc: print 'already batch inserted' return print '开始存储 ' + dateStr + ' ' + formType + ' 的数据' ckObj = Object.extend('DailyCheckIn') ckQuery = Query(ckObj) ckQuery.greater_than_or_equal_to('createdAt',start) ckQuery.less_than_or_equal_to('createdAt',end) def call_back(lst): global dcCount writeList = [] for item in lst: ufMap = {} ufMap['openId'] = item.get('openId') ufMap['checkTime'] = item.get('checkTime') ufMap['srcId'] = item.id ufMap['createdAt'] = item.created_at ufMap['updatedAt'] = item.updated_at writeList.append(ufMap) dcCount = dcCount + len(writeList) if len(writeList) > 0: loadDailyCheckIn(writeList) map_query(ckQuery, 1000, call_back) finishLoadData(dateStr, formType, dcCount)
def associate_event_and_activity(db_name='MergedUserContext'): application_event_dict = get_all_event() print 'already get the application_event_dict' print str(len(application_event_dict.keys())) DBTable = Object.extend(db_name) # print 'event_list: %s' %str(application_event_dict.values()) for application,event_dict in application_event_dict.items(): # print event_dict if event_dict: print 'application_event_dict values first count: %s' %str(event_dict.keys()) EventActivity = Object.extend('FakeEventActivity') for event_name,event_list in event_dict.items(): total_count = len(event_list) print 'event_list total_count: %s with event_name is: %s' %(str(total_count),event_name) print 'application id is: %s' %str(application.id) event_activity = EventActivity() relation = event_activity.relation('event') activity_dict = {} for index,event in enumerate(event_list): relation.add(event) query = Query(DBTable) query.equal_to('tracker',event.get('tracker')) query.less_than_or_equal_to('startTime',event.get('timestamp')) query.greater_than_or_equal_to('endTime',event.get('timestamp')) activity_list = query.find() if len(activity_list) == 1 or len(activity_list) == 2 : # for the convenience of adding the dimension of time to the analyzer event.set('activity',activity_list[0]) event.save() # activity = activity_list[0].get('eventType')[0] activity = activity_list[0].get('eventType')[0] if activity in activity_dict.keys(): activity_dict[activity]+=1 else: activity_dict[activity] =1 else: event.destroy() print 'length of activity_list: %s' %(str(len(activity_list))) print 'Seems to be an error,index: %s,user: %s; timestamp: %s \n' %(str(index),str(event.get('tracker').id ),str(event.get('timestamp'))) other_activity_total_count =total_count-sum(activity_dict.values()) if other_activity_total_count: activity_dict['others'] = other_activity_total_count # EventActivity = Object.extend('EventActivity') # event_activity = EventActivity() event_activity.set('application',application) event_activity.set('event_name',event_list[0].get('event_name')) event_activity.set('activity_dict',activity_dict) event_activity.save()
def get_account(start, end): global acCount acCount = 0 formType = 'Account' dateStr = '2016-05-30'#start.strftime('%Y-%m-%d') rc = checkWriteState(dateStr, formType) if rc: print 'already batch inserted' return print '开始存储 ' + dateStr + ' ' + formType + ' 的数据' accObj = Object.extend('Account') accQuery = Query(accObj) if(start): accQuery.greater_than_or_equal_to('updatedAt',start) if(end): accQuery.less_than_or_equal_to('updatedAt',end) def call_back(lst): global acCount; writeList = [] for item in lst: ufMap = {} ufMap['card'] = item.get('card') ufMap['clazzKey'] = item.get('clazzKey') ufMap['isPublishCard'] = item.get('isPublishCard') ufMap['openId'] = item.get('openId') ufMap['sex'] = item.get('sex') ufMap['isVerify'] = item.get('isVerify') ufMap['srcId'] = item.id ufMap['createdAt'] = item.created_at ufMap['updatedAt'] = item.updated_at writeList.append(ufMap) if len(writeList) > 0: loadaAccount(writeList) acCount = acCount + len(writeList) map_query(accQuery, 1000, call_back) finishLoadData(dateStr, formType, acCount)
from leancloud import init, Object, Query from datetime import datetime, date, timedelta Log = Object.extend("Log") UserLocation = Object.extend("UserLocation") if __name__ == '__main__': init(u'9ra69chz8rbbl77mlplnl4l2pxyaclm612khhytztl8b1f9o', u'1zohz2ihxp9dhqamhfpeaer8nh1ewqd9uephe9ztvkka544b') log_query = Query(Log) location_query = Query(UserLocation) today = datetime.strptime(date.today().strftime('%Y-%m-%d'), '%Y-%m-%d') tommorow = datetime.strptime((date.today() + timedelta(days=1)).strftime('%Y-%m-%d'), '%Y-%m-%d') log_query.equal_to('type', 'location') log_query.greater_than_or_equal_to("createdAt", today) log_query.less_than_or_equal_to("createdAt", tommorow) log_count = log_query.count() location_query.greater_than_or_equal_to("createdAt", today) location_query.less_than_or_equal_to("createdAt", tommorow) location_count = location_query.count() print "Today's Log of location: ", log_count print "Today's Location count: ", location_count print "Rate: ", location_count * 1.0 / log_count