def __init__(self): leancloud.init(settings.APP_ID, master_key=settings.MASTER_KEY) QuestionFollowee = Object.extend('Followee' + self.userDataId) questionFollowee = QuestionFollowee() # query = Query(QuestionFollowee) query.equal_to('followerLinkId',self.userLinkId) curTime = datetime.now() query.less_than('createdAt', curTime) followeeCount = query.count() print "followeeNumbers: %s" % str(questionNum) queryLimit = 500 queryTimes = int(followeeCount +queryLimit -1) / queryLimit self.urls = [] for index in range(queryTimes): query = Query(QuestionFollowee) query.less_than('createdAt', curTime) query.equal_to('followerLinkId',self.userLinkId) query.descending('createdAt') query.limit(queryLimit) query.skip(index * queryLimit) query.select('followeeDataId') quesRet = query.find() for ques in quesRet: self.followeeDataIdList.extend(ques.get('followeeDataId'))
def get_event_to_location_data(self, event_name=None, app_table=APPLICATION_CLASS, kind=None): # print app_table try: app = self.get_the_app(kind=kind) db_name = "FakeEventActivity" DbTable = Object.extend(db_name) query = Query(DbTable) # 这里只是测试知道是少于1K条的 query.equal_to("application", app) query.exists("location_dict") if event_name: query.equal_to("event_name", event_name) # query.equal_to('application_id',application_id) query.descending("createdAt") query.limit(1) result_list = query.find() if result_list: # event_name = result_list[0].get('event_name') # activity_statistics_dict = {result_list[0].get('event_name'):result_list[0].get('activity_dict')} statistics_dict = result_list[0].get("location_dict") else: statistics_dict = [] except LeanCloudError, e: raise e
def get_all_behavior_prediction(table_name='UserBehavior',field_name='prediction'): DBTable = Object.extend(table_name) query = Query(DBTable) query.exists(field_name) query.less_than('updatedAt',currentTime) total_count=query.count() print 'TotalCount %s' %str(total_count) query_times=(total_count+query_limit-1)/query_limit behavior_prediction_dict={} for index in range(query_times): print 'querying index: %s' %str(index) query = Query(DBTable) query.exists(field_name) query.less_than('updatedAt',currentTime) query.limit(query_limit) query.skip(index*query_limit) query.descending('updatedAt') result_list=query.find() for result in result_list: record_wrap ={} record_wrap['startTime'] = result.get('startTime') record_wrap['endTime'] = result.get('endTime') record_wrap['prediction'] = result.get(field_name) # record_wrap['UserBehavior'] = result # 这里是为了绑定userBehavior behavior_prediction_dict[result]=record_wrap #这里的键user由于是对象,所以即使是同一个user,他们也是不同的 return behavior_prediction_dict
def _latest_convs_by_creater(username): cq = Query("_Conversation") creater = user_by_username(username) cq.equal_to("c", creater.id) cq.descending("createdAt") convs = cq.find() return convs
def __init__(self): leancloud.init('mctfj249nwy7c1ymu3cps56lof26s17hevwq4jjqeqoloaey', master_key='ao6h5oezem93tumlalxggg039qehcbl3x3u8ofo7crw7atok') Users = Object.extend('Users') query = Query(Users) curTime = datetime.now() query.exists('userId') query.less_than('createdAt',curTime) userCount = query.count() print "userCounts: %s" %str(userCount) queryLimit = 500 queryTimes = userCount/queryLimit + 1 self.urls = [] for index in range(queryTimes): query = Query(Users) query.exists('userId') query.less_than('createdAt',curTime) query.descending('createdAt') query.limit(queryLimit) query.skip(index*queryLimit) query.select('userId') userRet = query.find() for user in userRet: self.urls.append(self.baseUrl + user.get('userId') +".json") pass
def many_users(many): print MyEnv.appid, MyEnv.appkey query = Query("_User") query.descending("createdAt") query.include("sioeyeInfo") query.limit(many) users = query.find() return users
def getssids(self): pdata = Object.extend(self.pass_word_data) pdataQuery = Query(pdata) pdataQuery.limit(10) pdataQuery.descending("updatedAt") result = pdataQuery.find() ssids = [x.get('ssid') for x in result] return ssids
def get_lastest_item_id(): query = Query('Reading') query.descending("item_id") query.limit(1) querys = query.find() if len(querys) == 0: return 0 else: return querys[0].get("item_id")
def get_readys(): ''' 返回当前准备好的人 ''' user_query = Query(User) user_query.equal_to('ready', True) # 使得顺序固定 user_query.descending('objectId') return user_query.find()
def _latest_live_by_creater_startswith_title(username, title): lq = Query("Live") caster = user_by_username(username) lq.equal_to("caster", caster) lq.startswith("keyword", title) lq.descending("createdAt") live = lq.first() showo_inDict(live) return live
def get_latest(key): try: query = Query(test_esp) query.equal_to('key', key) query.descending('createdAt') latest = query.first() return latest except: rasie
def get_latest(key): try: query =Query(test_esp) query.equal_to('key',key) query.descending('createdAt') latest = query.first() return latest except: rasie
def get_lastest_item_id(): query = Query('Reading') query.equal_to('category', 'composition') query.equal_to('source_name', '恒星英语') query.descending("item_id") query.limit(1) querys = query.find() itemId = querys[0].get("item_id") return itemId
def getModel(algo_type, model_tag, event_type): query = Query(Model) query.equal_to("algoType", algo_type) query.equal_to("tag", model_tag) query.equal_to("eventType", event_type) query.descending("timestamp") model_info = query.first() model_param = model_info.get("param") status_sets = model_info.get("statusSets") return {"modelParam": model_param, "statusSets": status_sets}
def get_lastest_item_id(): query = Query('Reading') query.equal_to('category', 'composition') query.equal_to('source_name', '水滴英语作文网') query.descending("item_id") query.limit(1) querys = query.find() if len(querys) == 0: return 0 else: return querys[0].get("item_id")
def min_pm(key): query =Query(test_esp) query.equal_to('key',key) query.descending('createdAt') pm=[] query.limit(60) results = query.find() for result in results: pm.append(result.get('pm')) pm.reverse() return pm
def min_noise(key): query =Query(test_esp) query.equal_to('key',key) query.descending('createdAt') noise=[] query.limit(60) results = query.find() for result in results: noise.append(result.get('noise')) noise.reverse() return noise
def get_lastest_item_id(): global source_name global category query = Query('Reading') query.descending("item_id") query.limit(1) querys = query.find() if len(querys) == 0: return 0 else: return querys[0].get("item_id")
def copyTask(count): global counter query = Query('Reading') query.descending('createdAt') query.skip(count + counter) querys = query.find() print len(querys) size = len(querys) if size == 0: return for data in querys: counter += 1 title = data.get('title') source_url = data.get('source_url') print(counter) print(title) print(data.id) print(source_url) if isUrlExit(source_url): print('-------rds already has this url--------') pass else: mReading = Reading() mReading.title = data.get('title') mReading.content = data.get('content') mReading.content_type = data.get('content_type') mReading.source_url = source_url mReading.source_name = data.get('source_name') mReading.createdAt = data.get('createdAt') mReading.publish_time = data.get('publish_time') mReading.updatedAt = data.get('updatedAt') mReading.uuid = data.id mReading.media_url = data.get('media_url') mReading.img_url = data.get('img_url') img_urls = None if data.get('img_urls'): img_urls = json.dumps(data.get('img_urls')) small_img = None if data.get('small_img'): small_img = json.dumps(data.get('small_img')) mReading.img_urls = img_urls mReading.small_img = small_img mReading.type = data.get('type') mReading.type_id = data.get('type_id') mReading.type_name = data.get('type_name') mReading.level = data.get('level') mReading.lrc_url = data.get('lrc_url') mReading.category_2 = data.get('category_2') mReading.category = data.get('category') print saveReadingItem(mReading) print saveSpiderDBUrl(source_url) print('save item')
def get_lastest_item_id(): query = Query('Reading') query.equal_to('category', 'shuangyu_reading') query.equal_to('source_name', '恒星英语') query.descending("item_id") query.limit(1) querys = query.find() if len(querys) == 0: return 0 else: return querys[0].get("item_id")
def min_pm(key): query = Query(test_esp) query.equal_to('key', key) query.descending('createdAt') pm = [] query.limit(60) results = query.find() for result in results: pm.append(result.get('pm')) pm.reverse() return pm
def min_noise(key): query = Query(test_esp) query.equal_to('key', key) query.descending('createdAt') noise = [] query.limit(60) results = query.find() for result in results: noise.append(result.get('noise')) noise.reverse() return noise
def average_pm(num, key): query = Query(test_esp) query.equal_to('key', key) query.descending('createdAt') pm = [] for i in range(1, 24): query.limit(60) results = query.find() pm.append(average(results, 60)) query.skip(60 * i) pm.reverse() print pm return pm
def get_lastest_item_id(): global source_name global category query = Query('Reading') query.equal_to('category', category) query.equal_to('source_name', source_name) query.descending("item_id") query.limit(1) querys = query.find() if len(querys) == 0: return 0 else: return querys[0].get("item_id")
def average_pm(num,key): query =Query(test_esp) query.equal_to('key',key) query.descending('createdAt') pm=[] for i in range(1,24): query.limit(60) results = query.find() pm.append(average(results,60)) query.skip(60*i) pm.reverse() print pm return pm
def find_plugin(identifier, array): version = None query = Query(Rule) query.equal_to('identifier', identifier) query.descending("version") query.limit(1) rules_result = query.find() if rules_result: rule = rules_result[0].dump() rule['file'] = rule['file']['url'] array.append(rule) version = rule['version'] return version
def getModel(algo_type, model_tag, event_type): query = Query(Model) query.equal_to("algoType", algo_type) query.equal_to("tag", model_tag) query.equal_to("eventType", event_type) query.descending("timestamp") model_info = query.first() model_param = model_info.get("param") status_sets = model_info.get("statusSets") return { "modelParam": model_param, "statusSets": status_sets }
def get_installationid_by_trackerid(tracker_id=None): print "tracker_id", tracker_id query = Query(Installation) user = { "__type": "Pointer", "className": "_User", "objectId": tracker_id } query.equal_to('user', user) query.descending('updatedAt') print "query.count", query.count() installation = query.find()[0] if query.count() else {} print "query Installaton", installation.id return installation.id, installation.get('deviceType')
def get_event_to_activity_data(self, application_id, event_name, db_name="EventActivity"): try: DbTable = Object.extend(db_name) query = Query(DbTable) # 这里只是测试知道是少于1K条的 query.equal_to("event_name", event_name) # query.equal_to('application_id',application_id) query.descending("createdAt") query.limit(1) result = query.find() activity_statistics_dict = result[0].get("activity_dict") except LeanCloudError, e: raise e
def get_all_res_list(call_back): global skip_num query = Query(Post) #query.descending('ID') query.descending('updatedAt') query.skip(skip_num*LIMIT_NUM) query.limit(LIMIT_NUM) res = query.find() call_back(res) skip_num += 1 if len(res) >= LIMIT_NUM: time.sleep(1) get_all_res_list(call_back)
def get(self, request): page = request.GET.get('page') try: query = Query(Card) query.not_equal_to('publish', True) query.not_equal_to('deleted', True) query.does_not_exist('user') #query.not_equal_to('publish',False) #query.does_not_exist('cid') if page is None: page = 0 count = query.count() query.skip(int(page) * 100) query.limit(100) query.include('user') cards = query.descending('likes').find() except LeanCloudError as e: if e.code == 101: cards = [] else: raise e return render(request, 'card_preview.html', { 'cards': cards, 'count': count })
def show(year, month): month = float(month) start, end = get_month_id(year, month), get_month_id(year, month + 0.5) if end is None: end = 1e10 if start is None: abort(404) chinese_num = dict(enumerate(u'一二三四五六七八九', start=1)) chinese_num.update({10: '十', 11: '十一', 12: '十二'}) chinese_month = chinese_num.get(int(month)) if float(month) > int(month): chinese_month += u'月份下' else: chinese_month += u'月份上' try: topics = Query(Topic).less_than("pid", end).greater_than('pid', start) top_topics = topics.descending('reply_num').limit(20).find() users = [ topic.get('author_name') for topic in topics.limit(800).find() ] users.extend( topic.get('author_name') for topic in topics.limit(800).skip(800).find()) counter = collections.Counter(users).most_common(20) except LeanCloudError, e: if e.code == 101: topics = [] else: raise e
def _get_rnnrbm_params(**conditions): ''' :param conditions: {"eventType":"dining_in_restaurant","tag":"latest"} :return: ''' query = Query(Rnnrbm) query.equal_to("eventType", conditions["eventType"]) #query.equal_to("note",conditions["note"]) #query.equal_to("tag","latest") query.descending("trainedAt") rnnrbm = query.first() print "rnnrbm", rnnrbm return dict(params=rnnrbm.get("params"))
def get_my_last_at(message): ''' todo: relate the origin chat id. ''' query = Query(AtMessage) query.descending('createdAt') query.equal_to('owner', message.from_user.username) query.equal_to('chat_id', message.chat.id) try: msg = query.first() except LeanCloudError as e: bot.sendMessage(chat_id=message.chat.id, reply_to_message_id=message.message_id, text='你在本群还没有任何 AT 消息。') return text = 'Here you are.' message_id = msg.get('mid') bot.sendMessage(chat_id=message.chat.id, reply_to_message_id=message_id, text=text)
def _get_rnnrbm_params(**conditions): ''' :param conditions: {"eventType":"dining_in_restaurant","tag":"latest"} :return: ''' query = Query(Rnnrbm) query.equal_to("eventType",conditions["eventType"]) #query.equal_to("note",conditions["note"]) #query.equal_to("tag","latest") query.descending("trainedAt") rnnrbm = query.first() print "rnnrbm",rnnrbm return dict(params=rnnrbm.get("params"))
def get_all_static_info(table_name='UserInfoLog',field_name='staticInfo'): DBTable = Object.extend(table_name) query = Query(DBTable) query.exists(field_name) query.less_than('updatedAt',current_time) total_count=query.count() print 'TotalCount %s' %str(total_count) query_times=(total_count+query_limit-1)/query_limit static_info_list=[] for index in range(query_times): print 'querying index: %s' %str(index) query = Query(DBTable) query.exists(field_name) query.less_than('updatedAt',current_time) query.limit(query_limit) query.skip(index*query_limit) query.descending('updatedAt') result_list=query.find() static_info_list.extend(result_list) return static_info_list
def get(self, request): try: query = Query(_User) query.exists('avatarUrl') count = query.count() query.limit(1000) users = query.descending('createdAt').find() except LeanCloudError as e: if e.code == 101: cards = [] else: raise e return render(request,'users.html',{'users':users,'count':count})
def get(self, request): try: query = Query(Card) query.not_equal_to('deleted',False) query.does_not_exist('cid') query.limit(1000) count = query.count() query.include('user') cards = query.descending('createdAt').find() except LeanCloudError as e: if e.code == 101: cards = [] else: raise e return render(request,'card_deleted.html',{'cards':cards,'count':count})
query = Query(User) query.less_than('updatedAt',currentTime) query.exists(field_name) total_count=query.count() print 'TotalCount %s' %str(total_count) query_times=(total_count+query_limit-1)/query_limit userList=[] for index in range(query_times): print 'querying index: %s' %str(index) query = Query(User) query.less_than('updatedAt',currentTime) query.exists(field_name) query.descending('updatedAt') query.limit(query_limit) query.skip(index*query_limit) userList.extend(query.find()) DBTable = Object.extend(table_name_dest) probList =[] recordCount = 10 userLen= len(userList) print 'userLength is : %s' %str(userLen) for index ,key in enumerate(range(recordCount)): # print 'saving index: %s' %str(index) dbTable = DBTable() # dbTable.set('userInfoLogSrc','fakeData') # dbTable.set('user','fakeData') start = time.time()*0.9+ index*1800
return False #--------Main------------# if __name__ == '__main__': print(''' ************************************* ** Welcome to Ted Python ** ** Created on 2015-06-25 ** ** @author: Ted ** *************************************''') ''' 更新最新的10条 ''' query = Query(allPictures) query.limit(10) query.descending("updatedAt") pictures = query.find() print pictures[0].get('image_src') oldQuery = Query(new10Picture) oldQuery.descending("updatedAt") oldPictures = oldQuery.find() print str(len(oldPictures)) for num in range(0, 10): oldPicture = oldPictures[num] newPicture = pictures[num] oldPicture.set('image_src', newPicture.get('image_src')) oldPicture.set('category', newPicture.get('category')) oldPicture.set('color', newPicture.get('color')) oldPicture.set('width', newPicture.get('width'))
Question = Object.extend('Question' + tableIndexStr) query = Query(Question) query.exists('questionId') #curTime = datetime.now() #query.less_than('createdAt',curTime) questionNum = query.count() totalQuestion =totalQuestion+questionNum #print "questionNums: %s" %str(questionNum) queryLimit = 700 queryTimes = questionNum/queryLimit + 1 for index in range(queryTimes): query = Query(Question) #query.less_than('createdAt',Question) query.exists('questionId') query.descending('createdAt') query.limit(queryLimit) query.skip(index*queryLimit) query.select('questionTimestamp') query.select('tableIndex') query.select('questionId') quesRet = query.find() for ques in quesRet: questionId = str(ques.get('questionId')) if client1.get(str(questionId)): ques.set("flag",3) try: ques.save()
query.less_than("updatedAt", currentTime) total_count = query.count() print "TotalCount %s" % str(total_count) query_times = (total_count + query_limit - 1) / query_limit staticInfoDict = {} for index in range(query_times): print "querying index: %s" % str(index) query = Query(DBTable) query.exists(field_name) query.less_than("updatedAt", currentTime) query.limit(query_limit) query.skip(index * query_limit) query.descending("updatedAt") result_list = query.find() for result in result_list: recordWrap = {} recordWrap["labels"] = result.get(field_name) recordWrap["UserInfoLog"] = result staticInfoDict[result.get("user")] = recordWrap # 注意此处的每个user是不同的 leancloud.init(APP_ID_DEST, APP_KEY_DEST) for index, key in enumerate(staticInfoDict.keys()): print "saving index: %s" % str(index) # print staticInfoDict[key] StaticInfo = Object.extend("StaticInfo") staticInfo = StaticInfo() staticInfo.set("user", key) staticInfo.set("userInfoLogSrc", staticInfoDict[key]["UserInfoLog"])
# print 'TotalCount %s' %str(total_count) # query_times=(total_count+query_limit-1)/query_limit total_record_count = total_count #这里只是造假数据,假设记录不会新增 query = Query(DBTable) query.equal_to(field_name,tracker) query.less_than('createdAt',currentTime) query.ascending('startTime') query.limit(1) earliest_time = query.find()[0].get('startTime') query = Query(DBTable) query.equal_to(field_name,tracker) query.less_than('createdAt',currentTime) query.descending('endTime') query.limit(1) latest_time = query.find()[0].get('endTime') fake_timestamp_list = range(earliest_time,latest_time,int((latest_time-earliest_time)/(total_count*6))) list_len = len(fake_timestamp_list) print 'earliest_time: %s; latest_time: %s; total_count: %s;' %(str(earliest_time),str(latest_time),str(total_count)) print 'the length of fake_timestamp_list: %s \n' %str(list_len) table_name_dest='FakeAppEvent' DBTable = Object.extend(table_name_dest) for index in range(list_len): random_timestamp = fake_timestamp_list.pop(np.random.randint(len(fake_timestamp_list))) app_event_name = app_event_name_list[np.random.randint(len(app_event_name_list))]
def paginate(self, page=None, per_page=None, error_out=True): if has_request_context(): if page is None: try: page = int(request.args.get('page', 1)) except (TypeError, ValueError): if error_out: abort(404) page = 1 if per_page is None: try: per_page = int(request.args.get('per_page', 20)) except (TypeError, ValueError): if error_out: abort(404) per_page = 1 else: if page is None: page = 1 if per_page is None: per_page = 20 if error_out and page < 1: abort(404) if self.query_class: items = Query(Object.extend(self.query_class)) else: return None if self.news_type: items = items.equal_to('news_type', self.news_type) if self.descend_key: items = items.descending(key=self.descend_key) if page: items = items.skip((page - 1) * per_page) if per_page: items = items.limit(per_page) # TODO select items = items.select('news_type', 'news_title', 'news_time') items = items.find() # # items = Query(Object.extend(self.query_class)).equal_to('news_type', self.news_type).\ # descending(key=self.descend_key).skip((page - 1) * per_page).limit(per_page).\ # select('news_title','news_time').find() if not items and page != 1 and error_out: abort(404) if page == 1 and len(items) < per_page: total = len(items) else: if self.news_type: total = Query(Object.extend(self.query_class)).equal_to( 'news_type', self.news_type).count() else: total = Query(Object.extend(self.query_class)).count() return Pagination(page, per_page, total, items)
def fake_custom_event(): tracker_list = get_all_tracker(table_name='Tracker') print 'The length of userList is %s' %str(len(tracker_list)) table_name = 'MergedUserContext' field_name = 'tracker' DBTable = Object.extend(table_name) userContextDataDict = {} all_application_list = get_all_demo_applications() all_application_list.extend(get_all_applications()) for index,tracker in enumerate(tracker_list): tracker_id = tracker.id #这样处理是因为可能一个user的记录超过了一次可以读取的数量(1K条) query = Query(DBTable) query.equal_to(field_name,tracker) query.less_than('createdAt',current_time) total_count=query.count() if not total_count: print 'tracker not found with tracker id: %s and index: %s\n' %(str(tracker_id),str(index)) continue else: print 'faking tracker index : ' +str(index) #这里只是造假数据,假设记录不会新增 query = Query(DBTable) query.equal_to(field_name,tracker) query.less_than('createdAt',current_time) query.ascending('startTime') query.limit(1) earliest_time = query.find()[0].get('startTime') query = Query(DBTable) query.equal_to(field_name,tracker) query.less_than('createdAt',current_time) query.descending('endTime') query.limit(1) latest_time = query.find()[0].get('endTime') fake_timestamp_list = range(earliest_time,latest_time,int((latest_time-earliest_time)/(total_count*6))) list_len = len(fake_timestamp_list) print 'earliest_time: %s; latest_time: %s; total_count: %s;' %(str(earliest_time),str(latest_time),str(total_count)) print 'the length of fake_timestamp_list: %s \n' %str(list_len) table_name_dest='FakeAppEvent' DBTable = Object.extend(table_name_dest) for index in range(list_len): random_timestamp = fake_timestamp_list.pop(np.random.randint(len(fake_timestamp_list))) # 随机选取一个event app_event_name = app_event_name_list[np.random.randint(len(app_event_name_list))] info = { 'event':app_event_name, 'tracker_id':tracker_id, 'timestamp':random_timestamp, 'sth':'something else' } dbTable = DBTable() dbTable.set('application',all_application_list[np.random.randint(len(all_application_list))]) dbTable.set('event_name',app_event_name) dbTable.set('info',info) dbTable.set('tracker',tracker) dbTable.set('timestamp',random_timestamp) dbTable.save()