def get_all_tracker(table_name=INSTALLATION_TABLE): DbTable = table_name query = Query(DbTable) query.less_than('createdAt',current_time) query.exists('objectId') total_count=query.count() query_times=(total_count+query_limit-1)/query_limit all_installation_list = [] for index in range(query_times): query = Query(DbTable) query.exists('objectId') query.less_than('createdAt',current_time) query.select(USER_FIELD) query.ascending('createdAt') query.limit(query_limit) query.skip(index*query_limit) all_installation_list.extend(query.find()) user_id_set = set() user_list = [] for installation in all_installation_list: user = installation.get(USER_FIELD) user_id = user.id if user_id not in user_id_set: user_id_set.add(user_id) user_list.append(user) else: pass print "Have gotten all the trackers,total number is: %s" %(str(len(user_list))) return user_list
def get_tracker_data(table_name=None,tracker_list=None,field_name=None): DBTable = Object.extend(table_name) tracker_data_dict = {} for index,tracker in enumerate(tracker_list): #这样处理是因为可能一个user的记录超过了一次可以读取的数量(1K条) query = Query(DBTable) query.equal_to(field_name,tracker) query.less_than('createdAt',currentTime) total_count=query.count() # print 'TotalCount %s' %str(total_count) query_times=(total_count+query_limit-1)/query_limit #如果想在这里按timestamp排序取出每个user的记录是不靠谱的,可能读取时还有插入,而且timestamp介于之间 for index in range(query_times): # print 'querying index: %s' %str(index) query = Query(DBTable) query.equal_to(field_name,tracker) query.less_than('createdAt',currentTime) query.ascending('createdAt') query.limit(query_limit) query.skip(index*query_limit) if tracker in userDataDict.keys(): tracker_data_dict.get(tracker).extend(query.find()) else : tracker_data_dict[tracker]=query.find() return tracker_data_dict
def device_status(): try: query =Query(DeviceKey) query.ascending('index') devices =query.find() except: jsonify(status='find error') return render_template('status.html',devices=devices)
def device_status(): try: query = Query(DeviceKey) query.ascending('index') devices = query.find() except: jsonify(status='find error') return render_template('status.html', devices=devices)
def get_all_real_applications(db_name='Application'): DbTable = db_name query = Query(DbTable) query.less_than('createdAt',current_time) query.exists('objectId') total_count=query.count() query_times=(total_count+query_limit-1)/query_limit result_list = [] for index in range(query_times): query = Query(DbTable) query.exists('objectId') query.less_than('createdAt',current_time) query.ascending('createdAt') query.limit(query_limit) query.skip(index*query_limit) result_list.extend(query.find()) return result_list
def get_all_tracker(table_name=None): DbTable = table_name query = Query(DbTable) query.less_than('createdAt',current_time) query.exists('objectId') total_count=query.count() query_times=(total_count+query_limit-1)/query_limit user_list = [] for index in range(query_times): query = Query(DbTable) query.exists('objectId') query.less_than('createdAt',current_time) query.ascending('createdAt') query.limit(query_limit) query.skip(index*query_limit) user_list.extend(query.find()) return user_list
def get_all_tracker(self): try: # 这里认为应用的数量少于1000 Tracker = Object.extend("Tracker") query = Query(Tracker) query.exists("objectId") query.ascending("createdAt") query.limit(1000) result_list = query.find() all_tracker_dict = {} if result_list: for result in result_list: all_tracker_dict[result.get("username")] = result.id self.all_tracker_dict = all_tracker_dict return 1 except LeanCloudError, e: print e return 0
def get_all_application(self): try: # 这里认为应用的数量少于1000 Application = Object.extend(APPLICATION_CLASS) query = Query(Application) query.equal_to("user", self.user.become(self.session_token)) query.ascending("createdAt") query.limit(1000) result_list = query.find() all_application_dict = {} if result_list: for result in result_list: all_application_dict[result.get("app_name")] = result.get("app_id") self.all_application_dict = all_application_dict return 1 except LeanCloudError, e: print e return 0
def get_all_tracker(self): try: # 这里不能认为终端用户的数量少于1000 Application = Object.extend("Application") query = Query(Application) query.equal_to("app_id", self.app_id) query.ascending("createdAt") query.limit(1000) result_list = query.find() all_tracker_dict = {} if result_list: for result in result_list: all_tracker_dict[result.get("username")] = result.id self.all_tracker_dict = all_tracker_dict return 1 except LeanCloudError, e: print e return 0
def query_data(page, user_id, cur_time): ''' 查询弹幕,暂未使用 :param page: :param user_id: :param cur_time: :return: 弹幕数据 ''' last_post_time = update_user_info(page, user_id, cur_time) data = list() curtain_object = Object.extend('CurtainObject') query = Query(curtain_object) query.equal_to('page', page) query.greater_than('time', last_post_time) query.less_than('time', cur_time) query.ascending('time') results = query.find() for result in results: data.append({'content': result.get('content'), 'timestamp': result.get('time')}) return data
def get_all_applications(db_name=APPLICATION_TABLE): ''' 功能:查询并返回所有的应用,这里包括demo应用,返回形式为list ''' DbTable = db_name query = Query(DbTable) query.equal_to('app_id', 'demo55bc5d8e00b0cb9c40dec37b') query.less_than('createdAt',current_time) query.exists('objectId') total_count=query.count() query_times=(total_count+query_limit-1)/query_limit result_list = [] for index in range(query_times): query = Query(DbTable) query.exists('objectId') query.less_than('createdAt',current_time) query.ascending('createdAt') query.limit(query_limit) query.skip(index*query_limit) result_list.extend(query.find()) return result_list
def get_all_trackers(db_name=INSTALLATION_TABLE): ''' 这里返回每个应用和Installation表中相应user列表的字典 ''' # all_application_list = get_all_demo_applications() # all_application_list.extend(get_all_applications()) # 现在 Demo应用和新建的应用在一个表里面 print 'begin get_all_applications' all_application_list = get_all_applications() print 'length of application_list is %s' %(str(len(all_application_list))) print 'end get_all_applications' application_tracker_dict = {} for app in all_application_list: Installation = Object.extend(db_name) query = Query(Installation) query.equal_to(APPLICATION_FIELD,app) query.less_than('createdAt',current_time) total_count=query.count() query_times=(total_count+query_limit-1)/query_limit installation_list = [] for index in range(query_times): query = Query(Installation) query.equal_to(APPLICATION_FIELD,app) query.select(USER_FIELD) query.less_than('createdAt',current_time) query.ascending('createdAt') query.limit(query_limit) query.skip(index*query_limit) installation_list.extend(query.find()) # relation = app.relation('tracker') # query = relation.query() # result_list = query.find() #如果这个application没有tracker,则直接忽略掉 if installation_list: application_tracker_dict[app] = [installation.get(USER_FIELD) for installation in installation_list] print 'this application-tracker_list length is %s' %(str(len(installation_list))) print 'the length of application_tracker_dict is %s' %(str(len(application_tracker_dict.keys()))) return application_tracker_dict
def get_all_event(table_name='FakeAppEvent'): # return the application : event_list dict application_list = get_all_demo_applications() # application_list.extend(get_all_applications()) print 'already get all application' DbTable = Object.extend(table_name) application_event_dict = {} for index,application in enumerate(application_list): print 'getting event in application index : %s' %(str(index)) query = Query(DbTable) query.equal_to('application',application) query.less_than('createdAt',current_time) total_count=query.count() query_times=(total_count+query_limit-1)/query_limit event_list = [] for index in range(query_times): query = Query(DbTable) query.equal_to('application',application) query.less_than('createdAt',current_time) query.ascending('createdAt') query.limit(query_limit) query.skip(index*query_limit) event_list.extend(query.find()) event_dict = {} if event_list: for event in event_list: event_name = event.get('event_name') if event_name in event_dict.keys(): # print event_dict event_dict[event_name].append(event) else: event_dict[event_name] = [event] application_event_dict[application] = event_dict else: pass return application_event_dict
def query_data(page, user_id, cur_time): ''' 查询弹幕,暂未使用 :param page: :param user_id: :param cur_time: :return: 弹幕数据 ''' last_post_time = update_user_info(page, user_id, cur_time) data = list() curtain_object = Object.extend('CurtainObject') query = Query(curtain_object) query.equal_to('page', page) query.greater_than('time', last_post_time) query.less_than('time', cur_time) query.ascending('time') results = query.find() for result in results: data.append({ 'content': result.get('content'), 'timestamp': result.get('time') }) return data
def reviews(self): query = Query(Review) reviews = query.equal_to('option', self) query.ascending('updateAt') return query.find()
def fake_custom_event(): tracker_list = get_all_tracker(table_name='Tracker') print 'The length of userList is %s' %str(len(tracker_list)) table_name = 'MergedUserContext' field_name = 'tracker' DBTable = Object.extend(table_name) userContextDataDict = {} all_application_list = get_all_demo_applications() all_application_list.extend(get_all_applications()) for index,tracker in enumerate(tracker_list): tracker_id = tracker.id #这样处理是因为可能一个user的记录超过了一次可以读取的数量(1K条) query = Query(DBTable) query.equal_to(field_name,tracker) query.less_than('createdAt',current_time) total_count=query.count() if not total_count: print 'tracker not found with tracker id: %s and index: %s\n' %(str(tracker_id),str(index)) continue else: print 'faking tracker index : ' +str(index) #这里只是造假数据,假设记录不会新增 query = Query(DBTable) query.equal_to(field_name,tracker) query.less_than('createdAt',current_time) query.ascending('startTime') query.limit(1) earliest_time = query.find()[0].get('startTime') query = Query(DBTable) query.equal_to(field_name,tracker) query.less_than('createdAt',current_time) query.descending('endTime') query.limit(1) latest_time = query.find()[0].get('endTime') fake_timestamp_list = range(earliest_time,latest_time,int((latest_time-earliest_time)/(total_count*6))) list_len = len(fake_timestamp_list) print 'earliest_time: %s; latest_time: %s; total_count: %s;' %(str(earliest_time),str(latest_time),str(total_count)) print 'the length of fake_timestamp_list: %s \n' %str(list_len) table_name_dest='FakeAppEvent' DBTable = Object.extend(table_name_dest) for index in range(list_len): random_timestamp = fake_timestamp_list.pop(np.random.randint(len(fake_timestamp_list))) # 随机选取一个event app_event_name = app_event_name_list[np.random.randint(len(app_event_name_list))] info = { 'event':app_event_name, 'tracker_id':tracker_id, 'timestamp':random_timestamp, 'sth':'something else' } dbTable = DBTable() dbTable.set('application',all_application_list[np.random.randint(len(all_application_list))]) dbTable.set('event_name',app_event_name) dbTable.set('info',info) dbTable.set('tracker',tracker) dbTable.set('timestamp',random_timestamp) dbTable.save()
query.less_than('createdAt',currentTime) query.exists(field_name) total_count=query.count() print 'TotalCount %s' %str(total_count) query_times=(total_count+query_limit-1)/query_limit userList=[] #注意这种分页的方法取数据时,如果user数量很大,而且读数据时user还在快速增长的话,是可能取到重复的user的,要怎么Unique呢? #但是如果采用的是ascending的排序方式的话,貌似倒是不会出现这个情况 for index in range(query_times): print 'querying index: %s' %str(index) query = Query(User) query.less_than('createdAt',currentTime) query.exists(field_name) query.ascending('createdAt') query.limit(query_limit) query.skip(index*query_limit) userList.extend(query.find()) print 'The length of userList is %s' %str(len(userList)) table_name = 'StaticInfo' field_name = 'user' DBTable = Object.extend(table_name) userDataDict = {} for user in userList: #这样处理是因为可能一个user的记录超过了一次可以读取的数量(1K条) query = Query(DBTable) query.equal_to(field_name,user) query.less_than('createdAt',currentTime)
total_count=query.count() if not total_count: print 'user not found with user name: %s and index: %s\n' %(str(tracker_id),str(index0)) continue else: print 'faking user index : ' +str(index0) # print 'TotalCount %s' %str(total_count) # query_times=(total_count+query_limit-1)/query_limit total_record_count = total_count #这里只是造假数据,假设记录不会新增 query = Query(DBTable) query.equal_to(field_name,tracker) query.less_than('createdAt',currentTime) query.ascending('startTime') query.limit(1) earliest_time = query.find()[0].get('startTime') query = Query(DBTable) query.equal_to(field_name,tracker) query.less_than('createdAt',currentTime) query.descending('endTime') query.limit(1) latest_time = query.find()[0].get('endTime') fake_timestamp_list = range(earliest_time,latest_time,int((latest_time-earliest_time)/(total_count*6))) list_len = len(fake_timestamp_list) print 'earliest_time: %s; latest_time: %s; total_count: %s;' %(str(earliest_time),str(latest_time),str(total_count)) print 'the length of fake_timestamp_list: %s \n' %str(list_len)
query.less_than("createdAt", currentTime) query.exists(field_name) total_count = query.count() print "TotalCount %s" % str(total_count) query_times = (total_count + query_limit - 1) / query_limit userList = [] # 注意这种分页的方法取数据时,如果user数量很大,而且读数据时user还在快速增长的话,是可能取到重复的user的,要怎么Unique呢? # 但是如果采用的是ascending的排序方式的话,貌似倒是不会出现这个情况 for index in range(query_times): print "querying index: %s" % str(index) query = Query(User) query.less_than("createdAt", currentTime) query.exists(field_name) query.ascending("createdAt") query.limit(query_limit) query.skip(index * query_limit) userList.extend(query.find()) print "The length of userList is %s" % str(len(userList)) table_name = "UserContext" field_name = "user" DBTable = Object.extend(table_name) userContextDataDict = {} for user in userList: # 这样处理是因为可能一个user的记录超过了一次可以读取的数量(1K条) query = Query(DBTable) query.equal_to(field_name, user) query.less_than("createdAt", currentTime)